From 621b0bfeab6ab305b9ba8d04dc4ae6c65642e70d Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Wed, 13 Mar 2019 17:53:46 +0100
Subject: [PATCH 01/26] WIP rr

---
 src/algo_rr.c | 139 ++++++++++++++++++++++++++++++++++++++++++++++++++
 src/packet.h  |   3 ++
 2 files changed, 142 insertions(+)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 6404eca..165ac81 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -1,6 +1,145 @@
+#include <sys/timerfd.h>
 #include "algo_skel.h"
 #include "algo_utils.h"
 
+int rr_on_tcp_co(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
+  int conn_sock1, conn_sock2;
+  struct sockaddr_in addr;
+  socklen_t in_len;
+  char url[1024], port[6];
+  struct evt_core_cat local_cat = {0};
+  struct evt_core_fdinfo to_fdinfo = {0};
+  to_fdinfo.cat = &local_cat;
+  to_fdinfo.url = url;
+
+  in_len = sizeof(addr);
+  conn_sock1 = accept(fdinfo->fd, (struct sockaddr*)&addr, &in_len);
+
+  if (conn_sock1 == -1) goto co_error;
+  conn_sock2 = dup(conn_sock1);
+  if (conn_sock2 == -1) goto co_error;
+  //printf("fd=%d accepts, creating fds=%d,%d\n", fd, conn_sock1, conn_sock2);
+
+  url_get_port(port, fdinfo->url);
+
+  to_fdinfo.fd = conn_sock1;
+  to_fdinfo.cat->name = "tcp-read";
+  sprintf(to_fdinfo.url, "tcp:read:127.0.0.1:%s", port);
+  evt_core_add_fd (ctx, &to_fdinfo);
+
+  to_fdinfo.fd = conn_sock2;
+  to_fdinfo.cat->name = "tcp-write";
+  sprintf(to_fdinfo.url, "tcp:write:127.0.0.1:%s", port);
+  evt_core_add_fd (ctx, &to_fdinfo);
+
+  return 1;
+
+co_error:
+  perror("Failed to handle new connection");
+  exit(EXIT_FAILURE);
+}
+
+void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec) {
+  struct timespec now;
+  struct itimerspec timer_config;
+  char url[1024];
+  struct evt_core_cat cat = {0};
+  struct evt_core_fdinfo fdinfo = {0};
+  fdinfo.cat = &cat;
+  fdinfo.url = url;
+
+  if (clock_gettime(CLOCK_REALTIME, &now) == -1) {
+    perror("clock_gettime");
+    exit(EXIT_FAILURE);
+  }
+
+  timer_config.it_value.tv_sec = now.tv_sec + micro_sec / 1000;
+  timer_config.it_value.tv_nsec = now.tv_nsec + micro_sec % 1000 * 1000000;
+  timer_config.it_interval.tv_sec = 60;
+  timer_config.it_interval.tv_nsec = 0;
+
+  fdinfo.fd = timerfd_create(CLOCK_REALTIME, 0);
+  if (fdinfo.fd == -1) {
+    perror("Unable to timerfd_create");
+    exit(EXIT_FAILURE);
+  }
+  if (timerfd_settime (fdinfo.fd, TFD_TIMER_ABSTIME, &timer_config, NULL) == -1) {
+    perror("Unable to timerfd_time");
+    exit(EXIT_FAILURE);
+  }
+  fdinfo.cat->name = "timeout";
+  fdinfo.other = NULL;
+  fdinfo.free_other = NULL;
+  sprintf(fdinfo.url, "timer:%ld:1", micro_sec);
+  evt_core_add_fd (evts, &fdinfo);
+}
+
+void rr_handle_recv(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct evt_core_fdinfo* fdinfo) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+
+  // 1. Update links I can used thanks to target feedback
+  if (bp->ip.ap.str.bitfield ^ app_ctx->my_bitfield) {
+    update_my_bitfield(bp, app_ctx);
+  }
+
+  // 2. If packet arrived too late
+  if (app_ctx->recv_id >  bp->ip.ap.str.id - 1) {
+    // Packet has already been delivered or dropped, we free the buffer
+    g_hash_table_remove (app_ctx->used_buffer, &(fdinfo->fd));
+    memset(bp, 0, sizeof(struct buffer_packet));
+    g_queue_push_tail(app_ctx->free_buffer, bp);
+  }
+
+  // 3. If packet arrived too early
+  else if (app_ctx->recv_id < bp->ip.ap.str.id - 1) {
+    int64_t timeout = app_ctx->mjit - (int64_t) bp->ip.ap.str.deltat;
+    if (timeout <= 0) timeout = 0;
+    //bp->ip.ap.str.id;
+    set_timeout(ctx, timeout);
+  }
+}
+
+int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
+  struct buffer_packet* bp;
+  struct evt_core_fdinfo *to_fdinfo = NULL;
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  int read_res = FDS_READY;
+  char url[255];
+
+  // 1. Get current read buffer OR a new read buffer OR subscribe to be notified later
+  if ((bp = get_read_buffer(app_ctx, fdinfo)) == NULL) return 1;
+
+  // 2. Try to read a whole packet in the buffer
+  while (bp->mode == BP_READING) {
+    read_res = read_packet_from_tcp (fdinfo->fd, bp);
+    if (read_res == FDS_ERR) goto co_error;
+    if (read_res == FDS_AGAIN) return 1;
+  }
+
+  // 3. Logic on packet
+  rr_handle_recv(bp, app_ctx, fdinfo);
+
+  /*
+  // 3. A whole packet has been read, we will find someone to write it
+  sprintf(url, "udp:write:127.0.0.1:%d", bp->ip.ap.str.port);
+  to_fdinfo = evt_core_get_from_url (ctx, url);
+  if (to_fdinfo == NULL) {
+    fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
+    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    return 1;
+  }
+  //printf("Pass packet from %s to %s\n", fdinfo->url, url);
+
+  // 4. We move the buffer and notify the target
+  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
+  on_udp_write(ctx, to_fdinfo);
+*/
+  return 0;
+co_error:
+  perror("Failed to TCP read");
+  exit(EXIT_FAILURE);
+}
+
 void algo_rr(struct algo_skel* as) {
   struct algo_ctx* ctx = malloc(sizeof(struct algo_ctx));
   if (ctx == NULL) goto init_err;
diff --git a/src/packet.h b/src/packet.h
index eb3dc29..7a0c72a 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -34,6 +34,9 @@ union abstract_packet {
     uint16_t size;
     uint16_t port;
     uint8_t id;
+    uint8_t bitfield;
+    uint8_t prevlink;
+    uint16_t deltat;
     char payload;
   } str;
 };

From cf9914e18a6f1653bb46257f89635ec12109c594 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Fri, 15 Mar 2019 09:39:23 +0100
Subject: [PATCH 02/26] Rework experiment logic

---
 scripts/xp1 | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/scripts/xp1 b/scripts/xp1
index e36125a..ab06b4f 100755
--- a/scripts/xp1
+++ b/scripts/xp1
@@ -2,8 +2,14 @@
 
 # Create workdir
 XP_FOLDER=`mktemp -d -p . xp1-XXXXXXXXXXXXXXXX`
-mkdir -p $XP_FOLDER
+XP_ID=`echo $XP_FOLDER|cut -c 3-`
 cd $XP_FOLDER
+COUNT=$1
+INTERVAL=$2
+SIZE=$3
+ALLOWED_BOOTSTRAP=190
+ALLOWED_TIME=`expr $(expr ${COUNT} * ${INTERVAL}) / 1000 + ${ALLOWED_BOOTSTRAP}`
+echo "Will run for ${ALLOWED_TIME} (count=$1, interval=$2)"
 
 # Create folders
 docker run \
@@ -17,23 +23,27 @@ docker run \
 docker run \
   --privileged \
   --rm \
+  --name "$XP_ID-server" \
   -v `pwd`/shared:/home/donar/shared \
   -v `pwd`/log:/home/donar/log \
   registry.gitlab.inria.fr/qdufour/donar \
   xp1-server &
 
-sleep 30
+sleep 10
 
 # Run client
 docker run \
   --privileged \
   --rm \
+  --name "$XP_ID-client" \
   -v `pwd`/res:/home/donar/res \
   -v `pwd`/shared:/home/donar/shared \
   -v `pwd`/log:/home/donar/log \
   registry.gitlab.inria.fr/qdufour/donar \
-  xp1-client $1 $2 $3
+  xp1-client $COUNT $INTERVAL $SIZE &
 
-# Kill server
-kill %1
+sleep $ALLOWED_TIME
 
+# Kill
+docker kill "$XP_ID-client" || true
+docker kill "$XP_ID-server" || true

From b04283b59ca2272eabf6746607bcc9f1be4d100f Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Fri, 15 Mar 2019 16:44:47 +0100
Subject: [PATCH 03/26] Huge WIP

---
 src/algo_rr.c    | 32 ++++++++++++++++++++++++++++++--
 src/algo_utils.h | 15 ++++++++++-----
 src/packet.h     |  6 ++++++
 3 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 165ac81..93b2ba6 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -68,7 +68,7 @@ void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec) {
     exit(EXIT_FAILURE);
   }
   fdinfo.cat->name = "timeout";
-  fdinfo.other = NULL;
+  fdinfo.other = NULL; // Should put the duration, the file descriptor, the packet id
   fdinfo.free_other = NULL;
   sprintf(fdinfo.url, "timer:%ld:1", micro_sec);
   evt_core_add_fd (evts, &fdinfo);
@@ -94,8 +94,22 @@ void rr_handle_recv(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct e
   else if (app_ctx->recv_id < bp->ip.ap.str.id - 1) {
     int64_t timeout = app_ctx->mjit - (int64_t) bp->ip.ap.str.deltat;
     if (timeout <= 0) timeout = 0;
-    //bp->ip.ap.str.id;
     set_timeout(ctx, timeout);
+    app_ctx->waiting[bp->ip.ap.str.id % LINK_COUNT] = bp; // should store more than that
+  }
+
+  // 4. If we were waiting this packet
+  else {
+    if (bp->ip.ap.str.flags & PKT_TIMEOUT) broken_rlink(fdinfo);
+    else if (bp->ip.ap.str.flags & PKT_CONTROL) working_rlink(fdinfo);
+    else deliver(bp);
+    app_ctx->recv_id = bp->ip.ap.str.id;
+    int next = app_ctx->recv_id+1;
+    if (app_ctx->waiting[next % LINK_COUNT] != NULL) {
+      bp = app_ctx->waiting[next % LINK_COUNT];
+      app_ctx->waiting[next % LINK_COUNT] = NULL;
+      rr_handle_recv(ctx,bp,NULL);
+    }
   }
 }
 
@@ -140,6 +154,19 @@ co_error:
   exit(EXIT_FAILURE);
 }
 
+struct deferred_pkt {
+  int link_fd;
+  struct buffer_packet* bp;
+};
+
+struct rr_ctx {
+  uint16_t mjit;
+  uint16_t recv_id;
+  uint16_t sent_id;
+  struct deferred_pkt real[LINK_COUNT];
+  struct deferred_pkt stub[LINK_COUNT];
+};
+
 void algo_rr(struct algo_skel* as) {
   struct algo_ctx* ctx = malloc(sizeof(struct algo_ctx));
   if (ctx == NULL) goto init_err;
@@ -148,6 +175,7 @@ void algo_rr(struct algo_skel* as) {
   ctx->read_waiting = g_queue_new ();
   ctx->used_buffer = g_hash_table_new(g_int_hash, g_int_equal);
   ctx->write_waiting = g_hash_table_new_full (g_int_hash, g_int_equal, NULL, naive_free_simple);
+  ctx->misc = malloc(sizeof(struct rr_ctx));
   for (int i = 0; i < sizeof(ctx->bps) / sizeof(ctx->bps[0]); i++) {
     g_queue_push_tail(ctx->free_buffer, &(ctx->bps[i]));
   }
diff --git a/src/algo_utils.h b/src/algo_utils.h
index 8c5c634..5165af1 100644
--- a/src/algo_utils.h
+++ b/src/algo_utils.h
@@ -3,14 +3,19 @@
 #include <glib-2.0/glib.h>
 #include <glib-2.0/gmodule.h>
 #include <glib-2.0/glib-object.h>
+#define LINK_COUNT 10
+
+typedef void (*algo_ctx_free_misc)(void*);
 
 struct algo_ctx {
   int ref_count;
-  struct buffer_packet bps[10];
-  GQueue* free_buffer; // Available buffers
-  GHashTable* used_buffer; // Buffers used for reading or writing
-  GQueue* read_waiting; // Who wait to be notified for a read
-  GHashTable* write_waiting; // Structure to track packets waiting to be written
+  struct buffer_packet bps[LINK_COUNT];
+  GQueue* free_buffer;          // Available buffers
+  GHashTable* used_buffer;      // Buffers used for reading or writing
+  GQueue* read_waiting;         // Who wait to be notified for a read
+  GHashTable* write_waiting;    // Structure to track packets waiting to be written
+  void* misc;                   // Additional structures
+  algo_ctx_free_misc free_misc; // Fx ptr to free misc
 };
 
 void mv_buffer_rtow(struct algo_ctx* app_ctx,
diff --git a/src/packet.h b/src/packet.h
index 7a0c72a..29e13b8 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -28,6 +28,11 @@ enum BP_MODE {
   BP_WRITING
 };
 
+enum PKT_FLAGS {
+  PKT_TIMEOUT = 1 << 0,
+  PKT_CONTROL = 1 << 1
+};
+
 union abstract_packet {
   char raw;
   struct {
@@ -37,6 +42,7 @@ union abstract_packet {
     uint8_t bitfield;
     uint8_t prevlink;
     uint16_t deltat;
+    uint8_t flags;
     char payload;
   } str;
 };

From 02a3507ef5f664e591f5a2adc6861a9c4eeba10c Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Mon, 18 Mar 2019 10:26:02 +0100
Subject: [PATCH 04/26] Global structures for RR algo

---
 src/algo_rr.c | 73 +++++++++++++++++++++++++++++++--------------------
 1 file changed, 44 insertions(+), 29 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 93b2ba6..72c500e 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -2,6 +2,24 @@
 #include "algo_skel.h"
 #include "algo_utils.h"
 
+
+struct deferred_pkt {
+  int link_fd;
+  struct buffer_packet* bp;
+};
+
+struct rr_ctx {
+  uint8_t my_links;
+  uint16_t my_links_ver;
+  uint8_t remote_links;
+  uint16_t mjit;
+  uint16_t recv_id;
+  uint16_t sent_id;
+  struct deferred_pkt real[LINK_COUNT];
+  struct deferred_pkt stub[LINK_COUNT];
+};
+
+
 int rr_on_tcp_co(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   int conn_sock1, conn_sock2;
   struct sockaddr_in addr;
@@ -76,14 +94,15 @@ void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec) {
 
 void rr_handle_recv(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct evt_core_fdinfo* fdinfo) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct rr_ctx* rr = app_ctx->misc;
 
-  // 1. Update links I can used thanks to target feedback
-  if (bp->ip.ap.str.bitfield ^ app_ctx->my_bitfield) {
-    update_my_bitfield(bp, app_ctx);
+  // 1. Update links I can use thanks to target feedback
+  if (bp->ip.ap.str.bitfield ^ rr->my_links) {
+    update_my_bitfield(bp, rr);
   }
 
   // 2. If packet arrived too late
-  if (app_ctx->recv_id >  bp->ip.ap.str.id - 1) {
+  if (rr->recv_id >  bp->ip.ap.str.id - 1) {
     // Packet has already been delivered or dropped, we free the buffer
     g_hash_table_remove (app_ctx->used_buffer, &(fdinfo->fd));
     memset(bp, 0, sizeof(struct buffer_packet));
@@ -91,24 +110,26 @@ void rr_handle_recv(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct e
   }
 
   // 3. If packet arrived too early
-  else if (app_ctx->recv_id < bp->ip.ap.str.id - 1) {
-    int64_t timeout = app_ctx->mjit - (int64_t) bp->ip.ap.str.deltat;
+  else if (rr->recv_id < bp->ip.ap.str.id - 1) {
+    int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.str.deltat;
     if (timeout <= 0) timeout = 0;
     set_timeout(ctx, timeout);
-    app_ctx->waiting[bp->ip.ap.str.id % LINK_COUNT] = bp; // should store more than that
+    int idx = bp->ip.ap.str.id % LINK_COUNT;
+    rr->real[idx].bp = bp;
+    rr->real[idx].link_fd = fdinfo->fd;
   }
 
   // 4. If we were waiting this packet
   else {
-    if (bp->ip.ap.str.flags & PKT_TIMEOUT) broken_rlink(fdinfo);
-    else if (bp->ip.ap.str.flags & PKT_CONTROL) working_rlink(fdinfo);
+    if (bp->ip.ap.str.flags & PKT_TIMEOUT) broken_rlink(rr, fdinfo);
+    else if (bp->ip.ap.str.flags & PKT_CONTROL) working_rlink(rr, fdinfo);
     else deliver(bp);
-    app_ctx->recv_id = bp->ip.ap.str.id;
-    int next = app_ctx->recv_id+1;
-    if (app_ctx->waiting[next % LINK_COUNT] != NULL) {
-      bp = app_ctx->waiting[next % LINK_COUNT];
-      app_ctx->waiting[next % LINK_COUNT] = NULL;
-      rr_handle_recv(ctx,bp,NULL);
+    rr->recv_id = bp->ip.ap.str.id;
+    struct deferred_pkt* def = &rr->real[rr->recv_id+1];
+    if (def->bp != NULL) {
+      struct evt_core_fdinfo* next = evt_core_get_from_fd (ctx, def->link_fd);
+      if (next == NULL) return;
+      rr_handle_recv(ctx, def->bp, next);
     }
   }
 }
@@ -131,7 +152,7 @@ int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   }
 
   // 3. Logic on packet
-  rr_handle_recv(bp, app_ctx, fdinfo);
+  rr_handle_recv(ctx, bp, fdinfo);
 
   /*
   // 3. A whole packet has been read, we will find someone to write it
@@ -154,18 +175,6 @@ co_error:
   exit(EXIT_FAILURE);
 }
 
-struct deferred_pkt {
-  int link_fd;
-  struct buffer_packet* bp;
-};
-
-struct rr_ctx {
-  uint16_t mjit;
-  uint16_t recv_id;
-  uint16_t sent_id;
-  struct deferred_pkt real[LINK_COUNT];
-  struct deferred_pkt stub[LINK_COUNT];
-};
 
 void algo_rr(struct algo_skel* as) {
   struct algo_ctx* ctx = malloc(sizeof(struct algo_ctx));
@@ -175,7 +184,13 @@ void algo_rr(struct algo_skel* as) {
   ctx->read_waiting = g_queue_new ();
   ctx->used_buffer = g_hash_table_new(g_int_hash, g_int_equal);
   ctx->write_waiting = g_hash_table_new_full (g_int_hash, g_int_equal, NULL, naive_free_simple);
-  ctx->misc = malloc(sizeof(struct rr_ctx));
+  struct rr_ctx* rr = malloc(sizeof(struct rr_ctx));
+  if (rr == NULL) goto init_err;
+  memset(rr, 0, sizeof(struct rr_ctx));
+  rr->mjit = 200;
+  rr->my_links = 0xff;
+  rr->remote_links = 0xff;
+  ctx->misc = rr;
   for (int i = 0; i < sizeof(ctx->bps) / sizeof(ctx->bps[0]); i++) {
     g_queue_push_tail(ctx->free_buffer, &(ctx->bps[i]));
   }

From 588219f0f0754c10e005709a500f0accdd4f1435 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Mon, 18 Mar 2019 17:58:40 +0100
Subject: [PATCH 05/26] WIP algo rr, TCP read logic

---
 src/algo_rr.c | 128 ++++++++++++++++++++++++++++++++++++--------------
 src/utils.c   |  20 ++++++++
 src/utils.h   |   5 ++
 3 files changed, 118 insertions(+), 35 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 72c500e..cdc62c6 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -1,7 +1,7 @@
 #include <sys/timerfd.h>
 #include "algo_skel.h"
 #include "algo_utils.h"
-
+#include "utils.h"
 
 struct deferred_pkt {
   int link_fd;
@@ -15,10 +15,15 @@ struct rr_ctx {
   uint16_t mjit;
   uint16_t recv_id;
   uint16_t sent_id;
+  uint8_t current_link;
   struct deferred_pkt real[LINK_COUNT];
   struct deferred_pkt stub[LINK_COUNT];
 };
 
+int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo);
+int rr_on_tcp_write(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo);
+int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo);
+int rr_on_udp_write(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo);
 
 int rr_on_tcp_co(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   int conn_sock1, conn_sock2;
@@ -92,54 +97,99 @@ void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec) {
   evt_core_add_fd (evts, &fdinfo);
 }
 
-void rr_handle_recv(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct evt_core_fdinfo* fdinfo) {
+int rr_update_states(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct evt_core_fdinfo* fdinfo) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
+  char buffer[16];
+  url_get_port (buffer, fdinfo->url);
+  int link_num = atoi(buffer) - 7500; // @FIXME Hardcoded
 
   // 1. Update links I can use thanks to target feedback
-  if (bp->ip.ap.str.bitfield ^ rr->my_links) {
-    update_my_bitfield(bp, rr);
+  if (bp->ip.ap.str.id > rr->my_links_ver) {
+    rr->my_links = bp->ip.ap.str.bitfield;
+    rr->my_links_ver = bp->ip.ap.str.id;
   }
 
   // 2. If packet arrived too late
-  if (rr->recv_id >  bp->ip.ap.str.id - 1) {
+  if (ring_gt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     // Packet has already been delivered or dropped, we free the buffer
     g_hash_table_remove (app_ctx->used_buffer, &(fdinfo->fd));
     memset(bp, 0, sizeof(struct buffer_packet));
     g_queue_push_tail(app_ctx->free_buffer, bp);
+    return 0;
   }
 
   // 3. If packet arrived too early
-  else if (rr->recv_id < bp->ip.ap.str.id - 1) {
+  if (ring_lt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.str.deltat;
     if (timeout <= 0) timeout = 0;
     set_timeout(ctx, timeout);
+    // Add a buffer to stub too
+    // Bitfield can be anything as a greater packet has triggered the bitfield update before
     int idx = bp->ip.ap.str.id % LINK_COUNT;
     rr->real[idx].bp = bp;
     rr->real[idx].link_fd = fdinfo->fd;
+    return 0;
   }
 
   // 4. If we were waiting this packet
-  else {
-    if (bp->ip.ap.str.flags & PKT_TIMEOUT) broken_rlink(rr, fdinfo);
-    else if (bp->ip.ap.str.flags & PKT_CONTROL) working_rlink(rr, fdinfo);
-    else deliver(bp);
-    rr->recv_id = bp->ip.ap.str.id;
-    struct deferred_pkt* def = &rr->real[rr->recv_id+1];
-    if (def->bp != NULL) {
-      struct evt_core_fdinfo* next = evt_core_get_from_fd (ctx, def->link_fd);
-      if (next == NULL) return;
-      rr_handle_recv(ctx, def->bp, next);
-    }
+  rr->recv_id = bp->ip.ap.str.id;
+  // 4.1 This is a timeout packet, we set the link as dead
+  if (bp->ip.ap.str.flags & PKT_TIMEOUT) {
+    rr->remote_links &= (1 << link_num) ^ UINT16_MAX;
+    return 0;
   }
+
+  // 4.2 This is a control packet, we set the link as alive
+  if (bp->ip.ap.str.flags & PKT_CONTROL) {
+    rr->remote_links |= 1 << link_num;
+    return 0;
+  }
+
+  return 1;
+}
+
+void rr_deliver(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct evt_core_fdinfo* fdinfo) {
+  struct evt_core_fdinfo *to_fdinfo = NULL;
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  char url[255];
+
+  // 1. A whole packet has been read, we will find its target
+  sprintf(url, "udp:write:127.0.0.1:%d", bp->ip.ap.str.port);
+  to_fdinfo = evt_core_get_from_url (ctx, url);
+  if (to_fdinfo == NULL) {
+    fprintf(stderr, "No fd for URL %s in udp:write for tcp-read. Dropping packet :( \n", url);
+    mv_buffer_wtor (app_ctx, fdinfo, bp);
+  }
+
+  // 2. We move the buffer and notify the target
+  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
+  rr_on_udp_write(ctx, to_fdinfo);
+}
+
+void rr_pkt_recv(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct rr_ctx* rr = app_ctx->misc;
+
+  do {
+    if(rr_update_states(ctx, bp, fdinfo)) rr_deliver(ctx, bp, fdinfo);
+    do {
+      struct deferred_pkt* def = &rr->real[rr->recv_id+1 % LINK_COUNT];
+      if (def->bp == NULL) break;
+      def->bp = NULL;
+      struct evt_core_fdinfo* fdinfo = evt_core_get_from_fd (ctx, def->link_fd);
+      if (fdinfo == NULL) {
+        fprintf(stderr, "An error occured as the link seems to be closed for the requested link\n");
+        rr->recv_id++;
+      }
+    } while (fdinfo == NULL);
+  } while(1);
 }
 
 int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct buffer_packet* bp;
-  struct evt_core_fdinfo *to_fdinfo = NULL;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   int read_res = FDS_READY;
-  char url[255];
 
   // 1. Get current read buffer OR a new read buffer OR subscribe to be notified later
   if ((bp = get_read_buffer(app_ctx, fdinfo)) == NULL) return 1;
@@ -152,29 +202,37 @@ int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   }
 
   // 3. Logic on packet
-  rr_handle_recv(ctx, bp, fdinfo);
+  rr_pkt_recv (ctx, fdinfo, bp);
 
-  /*
-  // 3. A whole packet has been read, we will find someone to write it
-  sprintf(url, "udp:write:127.0.0.1:%d", bp->ip.ap.str.port);
-  to_fdinfo = evt_core_get_from_url (ctx, url);
-  if (to_fdinfo == NULL) {
-    fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
-    mv_buffer_wtor (app_ctx, fdinfo, bp);
-    return 1;
-  }
-  //printf("Pass packet from %s to %s\n", fdinfo->url, url);
-
-  // 4. We move the buffer and notify the target
-  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
-  on_udp_write(ctx, to_fdinfo);
-*/
   return 0;
 co_error:
   perror("Failed to TCP read");
   exit(EXIT_FAILURE);
 }
 
+int rr_on_udp_write (struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
+  struct buffer_packet* bp;
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  int write_res = FDS_READY;
+
+  // 1. Get current write buffer OR a buffer from the waiting queue OR leave
+  if ((bp = get_write_buffer(app_ctx, fdinfo)) == NULL) return 1;
+
+  // 2. Write buffer
+  write_res = write_packet_to_udp(fdinfo->fd, bp, fdinfo->other);
+  if (write_res == FDS_ERR) goto co_error;
+  if (write_res == FDS_AGAIN) return 1;
+
+  // 3. A whole packet has been written
+  // Release the buffer and notify
+  mv_buffer_wtor(app_ctx, fdinfo, bp);
+  notify_read(ctx, app_ctx);
+
+  return 0;
+co_error:
+  perror("Failed to UDP write");
+  exit(EXIT_FAILURE);
+}
 
 void algo_rr(struct algo_skel* as) {
   struct algo_ctx* ctx = malloc(sizeof(struct algo_ctx));
diff --git a/src/utils.c b/src/utils.c
index 57ddc11..281238c 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -60,3 +60,23 @@ int ring_buffer_free_space(struct ring_buffer* rb) {
 int ring_buffer_used_space(struct ring_buffer* rb) {
   return RING_BUFFER_SIZE - ring_buffer_free_space (rb);
 }
+
+int ring_gt(uint16_t v1, uint16_t v2) {
+  int32_t vv1 = (int32_t) v1, vv2 = (int32_t) v2;
+  return v1 != v2 && (vv1 - vv2) % UINT16_MAX <= UINT16_MAX / 2;
+}
+
+int ring_ge(uint16_t v1, uint16_t v2) {
+  int32_t vv1 = (int32_t) v1, vv2 = (int32_t) v2;
+  return (vv1 - vv2) % UINT16_MAX <= UINT16_MAX / 2;
+}
+
+int ring_lt(uint16_t v1, uint16_t v2) {
+  int32_t vv1 = (int32_t) v1, vv2 = (int32_t) v2;
+  return v1 != v2 && (vv1 - vv2) % UINT16_MAX > UINT16_MAX / 2;
+}
+
+int ring_le(uint16_t v1, uint16_t v2) {
+  int32_t vv1 = (int32_t) v1, vv2 = (int32_t) v2;
+  return (vv1 - vv2) % UINT16_MAX > UINT16_MAX / 2;
+}
diff --git a/src/utils.h b/src/utils.h
index d30250f..4aba838 100644
--- a/src/utils.h
+++ b/src/utils.h
@@ -18,3 +18,8 @@ void ring_buffer_ack_read(struct ring_buffer* rb, int size);
 int ring_buffer_write(struct ring_buffer* rb, char* source, int size);
 int ring_buffer_free_space(struct ring_buffer* rb);
 int ring_buffer_used_space(struct ring_buffer* rb);
+
+int ring_gt(uint16_t v1, uint16_t v2);
+int ring_ge(uint16_t v1, uint16_t v2);
+int ring_lt(uint16_t v1, uint16_t v2);
+int ring_le(uint16_t v1, uint16_t v2);

From a43be46cb7e741716ece5145e0e03c7294c06801 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Tue, 19 Mar 2019 10:00:03 +0100
Subject: [PATCH 06/26] WIP timer

---
 src/algo_naive.c |  2 +-
 src/algo_rr.c    | 53 +++++++++++++++++++++++++++++++++++++++---------
 src/algo_skel.c  |  4 ++--
 src/algo_skel.h  |  6 +++---
 src/donar.c      |  3 ++-
 5 files changed, 51 insertions(+), 17 deletions(-)

diff --git a/src/algo_naive.c b/src/algo_naive.c
index 22fe3ca..cc8538a 100644
--- a/src/algo_naive.c
+++ b/src/algo_naive.c
@@ -193,7 +193,7 @@ int on_err(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   return 0;
 }
 
-void algo_naive(struct algo_skel* as) {
+void algo_naive(struct evt_core_ctx* evt, struct algo_skel* as) {
   struct algo_ctx* ctx = malloc(sizeof(struct algo_ctx));
   if (ctx == NULL) goto init_err;
   memset(ctx, 0, sizeof(struct algo_ctx));
diff --git a/src/algo_rr.c b/src/algo_rr.c
index cdc62c6..990b30b 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -18,6 +18,7 @@ struct rr_ctx {
   uint8_t current_link;
   struct deferred_pkt real[LINK_COUNT];
   struct deferred_pkt stub[LINK_COUNT];
+  struct buffer_packet stub_pool[LINK_COUNT];
 };
 
 int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo);
@@ -179,7 +180,7 @@ void rr_pkt_recv(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struc
       def->bp = NULL;
       struct evt_core_fdinfo* fdinfo = evt_core_get_from_fd (ctx, def->link_fd);
       if (fdinfo == NULL) {
-        fprintf(stderr, "An error occured as the link seems to be closed for the requested link\n");
+        fprintf(stderr, "An error occured as the link seems to be closed for the requested fd\n");
         rr->recv_id++;
       }
     } while (fdinfo == NULL);
@@ -234,7 +235,33 @@ co_error:
   exit(EXIT_FAILURE);
 }
 
-void algo_rr(struct algo_skel* as) {
+int rr_on_err(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct buffer_packet* bp;
+
+  // 1. If has a "used" buffer, remove it
+  bp = g_hash_table_lookup (app_ctx->used_buffer, &(fdinfo->fd));
+  if (bp != NULL) {
+    g_hash_table_remove (app_ctx->used_buffer, &(fdinfo->fd));
+    memset(bp, 0, sizeof(struct buffer_packet));
+    g_queue_push_tail(app_ctx->free_buffer, bp);
+  }
+
+  // 2. If appears in the write waiting queue, remove it
+  GQueue* writew = g_hash_table_lookup (app_ctx->write_waiting, &(fdinfo->fd));
+  while (writew != NULL && (bp = g_queue_pop_head (writew)) != NULL) {
+    memset(bp, 0, sizeof(struct buffer_packet));
+    g_queue_push_tail(app_ctx->free_buffer, bp);
+  }
+  g_hash_table_remove (app_ctx->write_waiting, &(fdinfo->fd));
+
+  // 3. If appears in the read waiting queue, remove it
+  g_queue_remove_all (app_ctx->read_waiting, &(fdinfo->fd));
+
+  return 0;
+}
+
+void algo_rr(struct evt_core_ctx* evt, struct algo_skel* as) {
   struct algo_ctx* ctx = malloc(sizeof(struct algo_ctx));
   if (ctx == NULL) goto init_err;
   memset(ctx, 0, sizeof(struct algo_ctx));
@@ -253,20 +280,20 @@ void algo_rr(struct algo_skel* as) {
     g_queue_push_tail(ctx->free_buffer, &(ctx->bps[i]));
   }
 
-/*
+
   as->on_tcp_co.name = "tcp-listen";
   as->on_tcp_co.flags = EPOLLIN;
   as->on_tcp_co.free_app_ctx = free_nothing;
-  as->on_tcp_co.cb = on_tcp_co;
+  as->on_tcp_co.cb = rr_on_tcp_co;
 
   as->on_tcp_read.name = "tcp-read";
   as->on_tcp_read.flags = EPOLLIN | EPOLLET | EPOLLRDHUP;
   as->on_tcp_read.app_ctx = ctx;
   as->on_tcp_read.free_app_ctx = free_naive;
-  as->on_tcp_read.cb = on_tcp_read;
-  as->on_tcp_read.err_cb = on_err;
+  as->on_tcp_read.cb = rr_on_tcp_read;
+  as->on_tcp_read.err_cb = rr_on_err;
   ctx->ref_count++;
-
+/*
   as->on_udp_read.name = "udp-read";
   as->on_udp_read.flags = EPOLLIN | EPOLLET;
   as->on_udp_read.app_ctx = ctx;
@@ -282,15 +309,21 @@ void algo_rr(struct algo_skel* as) {
   as->on_tcp_write.cb = on_tcp_write;
   as->on_tcp_write.err_cb = on_err;
   ctx->ref_count++;
+*/
 
   as->on_udp_write.name = "udp-write";
   as->on_udp_write.flags = EPOLLOUT | EPOLLET;
   as->on_udp_write.app_ctx = ctx;
   as->on_udp_write.free_app_ctx = free_naive;
-  as->on_udp_write.cb = on_udp_write;
-  as->on_udp_write.err_cb = on_err;
+  as->on_udp_write.cb = rr_on_udp_write;
+  as->on_udp_write.err_cb = rr_on_err;
   ctx->ref_count++;
-*/
+
+  struct evt_core_cat tcat = {
+
+  };
+  evt_core_add_cat(evt, &tcat);
+
   return;
 init_err:
   fprintf(stderr, "Failed to init algo naive\n");
diff --git a/src/algo_skel.c b/src/algo_skel.c
index 9fa5226..b3fa20b 100644
--- a/src/algo_skel.c
+++ b/src/algo_skel.c
@@ -1,9 +1,9 @@
 #include "algo_skel.h"
 
-void init_algo(struct algo_skel* as, char* name) {
+void init_algo(struct evt_core_ctx* ctx, struct algo_skel* as, char* name) {
   for (int i = 0; i < sizeof(available_algo) / sizeof(available_algo[0]); i++) {
     if (strcmp(available_algo[i].name, name) == 0) {
-      available_algo[i].init(as);
+      available_algo[i].init(ctx, as);
       return;
     }
   }
diff --git a/src/algo_skel.h b/src/algo_skel.h
index a8e6b28..7c51b8e 100644
--- a/src/algo_skel.h
+++ b/src/algo_skel.h
@@ -16,10 +16,10 @@ struct algo_skel {
   struct evt_core_cat on_tcp_co;
 };
 
-typedef void (*algo_init)(struct algo_skel* as);
+typedef void (*algo_init)(struct evt_core_ctx* ctx, struct algo_skel* as);
 
-void init_algo(struct algo_skel* as, char* name);
-void algo_naive(struct algo_skel* as);
+void init_algo(struct evt_core_ctx* ctx, struct algo_skel* as, char* name);
+void algo_naive(struct evt_core_ctx* ctx, struct algo_skel* as);
 
 struct algo_desc {
   algo_init init;
diff --git a/src/donar.c b/src/donar.c
index e098bdc..fe5dc53 100644
--- a/src/donar.c
+++ b/src/donar.c
@@ -59,14 +59,15 @@ int main(int argc, char** argv) {
     if (algo == NULL) goto in_error;
 
     struct algo_skel as = {0};
-    init_algo(&as, algo);
 
     if (is_server) {
       struct donar_server_ctx ctx;
+      init_algo(&ctx.evts, &as, algo);
       if (exposed_ports->len < 1 && remote_ports->len < 1) goto in_error;
       donar_server(&ctx, &as, exposed_ports, remote_ports);
     } else if (is_client) {
       struct donar_client_ctx ctx;
+      init_algo(&ctx.evts, &as, algo);
       if ((exposed_ports->len < 1 && remote_ports->len < 1) || onion_file == NULL) goto in_error;
       donar_client(&ctx, &as, onion_file, exposed_ports, remote_ports);
     }

From d89b7151c92ea88f5bb8f0e7466cf75a7d7f5c17 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Tue, 19 Mar 2019 13:50:38 +0100
Subject: [PATCH 07/26] We do not use LINK_COUNT anymore

---
 src/algo_rr.c    | 57 ++++++++++++++++++++++++++++++++++++++----------
 src/algo_utils.h |  4 ++--
 2 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 990b30b..7e9ad51 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -3,6 +3,8 @@
 #include "algo_utils.h"
 #include "utils.h"
 
+#define WAITING 10
+
 struct deferred_pkt {
   int link_fd;
   struct buffer_packet* bp;
@@ -16,9 +18,8 @@ struct rr_ctx {
   uint16_t recv_id;
   uint16_t sent_id;
   uint8_t current_link;
-  struct deferred_pkt real[LINK_COUNT];
-  struct deferred_pkt stub[LINK_COUNT];
-  struct buffer_packet stub_pool[LINK_COUNT];
+  struct deferred_pkt real[WAITING];
+  struct buffer_packet stub_bp;
 };
 
 int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo);
@@ -63,7 +64,7 @@ co_error:
   exit(EXIT_FAILURE);
 }
 
-void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec) {
+void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec, int64_t link) {
   struct timespec now;
   struct itimerspec timer_config;
   char url[1024];
@@ -92,7 +93,7 @@ void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec) {
     exit(EXIT_FAILURE);
   }
   fdinfo.cat->name = "timeout";
-  fdinfo.other = NULL; // Should put the duration, the file descriptor, the packet id
+  fdinfo.other = (void*)link; // Should put the link number and the id
   fdinfo.free_other = NULL;
   sprintf(fdinfo.url, "timer:%ld:1", micro_sec);
   evt_core_add_fd (evts, &fdinfo);
@@ -101,9 +102,6 @@ void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec) {
 int rr_update_states(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct evt_core_fdinfo* fdinfo) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
-  char buffer[16];
-  url_get_port (buffer, fdinfo->url);
-  int link_num = atoi(buffer) - 7500; // @FIXME Hardcoded
 
   // 1. Update links I can use thanks to target feedback
   if (bp->ip.ap.str.id > rr->my_links_ver) {
@@ -113,6 +111,7 @@ int rr_update_states(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct
 
   // 2. If packet arrived too late
   if (ring_gt(rr->recv_id, bp->ip.ap.str.id - 1)) {
+    if (bp->ip.ap.str.flags & PKT_TIMEOUT) return 0; // We don't use real buffer pkt for PKT_TIMEOUT
     // Packet has already been delivered or dropped, we free the buffer
     g_hash_table_remove (app_ctx->used_buffer, &(fdinfo->fd));
     memset(bp, 0, sizeof(struct buffer_packet));
@@ -124,10 +123,10 @@ int rr_update_states(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct
   if (ring_lt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.str.deltat;
     if (timeout <= 0) timeout = 0;
-    set_timeout(ctx, timeout);
+    set_timeout(ctx, timeout, bp->ip.ap.str.prevlink);
     // Add a buffer to stub too
     // Bitfield can be anything as a greater packet has triggered the bitfield update before
-    int idx = bp->ip.ap.str.id % LINK_COUNT;
+    int idx = bp->ip.ap.str.id % PACKET_BUFFER_SIZE;
     rr->real[idx].bp = bp;
     rr->real[idx].link_fd = fdinfo->fd;
     return 0;
@@ -135,6 +134,11 @@ int rr_update_states(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct
 
   // 4. If we were waiting this packet
   rr->recv_id = bp->ip.ap.str.id;
+
+  char buffer[16];
+  url_get_port (buffer, fdinfo->url);
+  int link_num = atoi(buffer) - 7500; // @FIXME Hardcoded
+
   // 4.1 This is a timeout packet, we set the link as dead
   if (bp->ip.ap.str.flags & PKT_TIMEOUT) {
     rr->remote_links &= (1 << link_num) ^ UINT16_MAX;
@@ -175,7 +179,7 @@ void rr_pkt_recv(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struc
   do {
     if(rr_update_states(ctx, bp, fdinfo)) rr_deliver(ctx, bp, fdinfo);
     do {
-      struct deferred_pkt* def = &rr->real[rr->recv_id+1 % LINK_COUNT];
+      struct deferred_pkt* def = &rr->real[(rr->recv_id+1) % PACKET_BUFFER_SIZE];
       if (def->bp == NULL) break;
       def->bp = NULL;
       struct evt_core_fdinfo* fdinfo = evt_core_get_from_fd (ctx, def->link_fd);
@@ -235,6 +239,28 @@ co_error:
   exit(EXIT_FAILURE);
 }
 
+int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct rr_ctx* rr = app_ctx->misc;
+  uint16_t id;
+
+  evt_core_rm_fd (ctx, fdinfo->fd); // We use the timeout only once
+  int fd = (int)fdinfo->other;
+  struct evt_core_fdinfo* tfdinfo = evt_core_get_from_fd (ctx, fd);
+  if (tfdinfo == NULL) {
+    fprintf(stderr, "An error occured as the link seems to be closed for the requested fd\n");
+    rr->recv_id++;
+    return 1;
+  }
+
+  rr->stub_bp.ip.ap.str.id = 0; //?
+  rr->stub_bp.ip.ap.str.flags = PKT_TIMEOUT;
+  rr->stub_bp.ip.ap.str.deltat = rr->mjit;
+  rr->stub_bp.ip.ap.str.prevlink = UINT8_MAX;
+  rr_pkt_recv (ctx, tfdinfo, &rr->stub_bp);
+  return 1;
+}
+
 int rr_on_err(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct buffer_packet* bp;
@@ -293,6 +319,7 @@ void algo_rr(struct evt_core_ctx* evt, struct algo_skel* as) {
   as->on_tcp_read.cb = rr_on_tcp_read;
   as->on_tcp_read.err_cb = rr_on_err;
   ctx->ref_count++;
+
 /*
   as->on_udp_read.name = "udp-read";
   as->on_udp_read.flags = EPOLLIN | EPOLLET;
@@ -320,8 +347,14 @@ void algo_rr(struct evt_core_ctx* evt, struct algo_skel* as) {
   ctx->ref_count++;
 
   struct evt_core_cat tcat = {
-
+    .name = "timeout",
+    .flags = EPOLLIN | EPOLLET,
+    .app_ctx = ctx,
+    .free_app_ctx = free_naive,
+    .cb = rr_on_timer,
+    .err_cb = NULL
   };
+  ctx->ref_count++;
   evt_core_add_cat(evt, &tcat);
 
   return;
diff --git a/src/algo_utils.h b/src/algo_utils.h
index 5165af1..0c4d93e 100644
--- a/src/algo_utils.h
+++ b/src/algo_utils.h
@@ -3,13 +3,13 @@
 #include <glib-2.0/glib.h>
 #include <glib-2.0/gmodule.h>
 #include <glib-2.0/glib-object.h>
-#define LINK_COUNT 10
+#define PACKET_BUFFER_SIZE 10
 
 typedef void (*algo_ctx_free_misc)(void*);
 
 struct algo_ctx {
   int ref_count;
-  struct buffer_packet bps[LINK_COUNT];
+  struct buffer_packet bps[PACKET_BUFFER_SIZE];
   GQueue* free_buffer;          // Available buffers
   GHashTable* used_buffer;      // Buffers used for reading or writing
   GQueue* read_waiting;         // Who wait to be notified for a read

From 5853e99e7b10251f812dc227efb7aa6b1955c202 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Tue, 19 Mar 2019 15:39:05 +0100
Subject: [PATCH 08/26] Rework algorithm

---
 src/algo_rr.c    | 137 +++++++++++++++++++++++------------------------
 src/algo_utils.h |   2 +-
 src/packet.h     |   3 +-
 3 files changed, 70 insertions(+), 72 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 7e9ad51..1c613a0 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -3,7 +3,12 @@
 #include "algo_utils.h"
 #include "utils.h"
 
-#define WAITING 10
+struct waited_pkt {
+  uint16_t id;
+  int link_num;
+  int timer_fd;
+  uint8_t on;
+};
 
 struct deferred_pkt {
   int link_fd;
@@ -18,8 +23,8 @@ struct rr_ctx {
   uint16_t recv_id;
   uint16_t sent_id;
   uint8_t current_link;
-  struct deferred_pkt real[WAITING];
-  struct buffer_packet stub_bp;
+  struct deferred_pkt real[PACKET_BUFFER_SIZE];
+  struct waited_pkt wait[PACKET_BUFFER_SIZE];
 };
 
 int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo);
@@ -64,7 +69,7 @@ co_error:
   exit(EXIT_FAILURE);
 }
 
-void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec, int64_t link) {
+int set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec, struct waited_pkt* wpkt) {
   struct timespec now;
   struct itimerspec timer_config;
   char url[1024];
@@ -93,13 +98,14 @@ void set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec, int64_t link) {
     exit(EXIT_FAILURE);
   }
   fdinfo.cat->name = "timeout";
-  fdinfo.other = (void*)link; // Should put the link number and the id
+  fdinfo.other = wpkt; // Should put the link number and the id
   fdinfo.free_other = NULL;
   sprintf(fdinfo.url, "timer:%ld:1", micro_sec);
   evt_core_add_fd (evts, &fdinfo);
+  return fdinfo.fd;
 }
 
-int rr_update_states(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct evt_core_fdinfo* fdinfo) {
+void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
 
@@ -109,57 +115,59 @@ int rr_update_states(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct
     rr->my_links_ver = bp->ip.ap.str.id;
   }
 
-  // 2. If packet arrived too late
+  // 2. If packet arrived too late, we discard it
   if (ring_gt(rr->recv_id, bp->ip.ap.str.id - 1)) {
-    if (bp->ip.ap.str.flags & PKT_TIMEOUT) return 0; // We don't use real buffer pkt for PKT_TIMEOUT
     // Packet has already been delivered or dropped, we free the buffer
-    g_hash_table_remove (app_ctx->used_buffer, &(fdinfo->fd));
-    memset(bp, 0, sizeof(struct buffer_packet));
-    g_queue_push_tail(app_ctx->free_buffer, bp);
-    return 0;
+    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    return;
   }
 
-  // 3. If packet arrived too early
+  // 3. If packet arrived too early, we register a timer
   if (ring_lt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.str.deltat;
     if (timeout <= 0) timeout = 0;
-    set_timeout(ctx, timeout, bp->ip.ap.str.prevlink);
-    // Add a buffer to stub too
-    // Bitfield can be anything as a greater packet has triggered the bitfield update before
-    int idx = bp->ip.ap.str.id % PACKET_BUFFER_SIZE;
-    rr->real[idx].bp = bp;
-    rr->real[idx].link_fd = fdinfo->fd;
-    return 0;
+    int idx_waited = (bp->ip.ap.str.id - 1) % PACKET_BUFFER_SIZE;
+    rr->wait[idx_waited].on = 1;
+    rr->wait[idx_waited].id = bp->ip.ap.str.id;
+    rr->wait[idx_waited].link_num = bp->ip.ap.str.prevlink;
+    set_timeout(ctx, timeout, &rr->wait[idx_waited]);
   }
 
-  // 4. If we were waiting this packet
-  rr->recv_id = bp->ip.ap.str.id;
+  // 4. We queue the packet
+  int idx_real = bp->ip.ap.str.id % PACKET_BUFFER_SIZE;
+  rr->real[idx_real].bp = bp;
+  rr->real[idx_real].link_fd = fdinfo->fd;
 
+  // 5. We make sure that the remote link is set to up
   char buffer[16];
   url_get_port (buffer, fdinfo->url);
   int link_num = atoi(buffer) - 7500; // @FIXME Hardcoded
-
-  // 4.1 This is a timeout packet, we set the link as dead
-  if (bp->ip.ap.str.flags & PKT_TIMEOUT) {
-    rr->remote_links &= (1 << link_num) ^ UINT16_MAX;
-    return 0;
-  }
-
-  // 4.2 This is a control packet, we set the link as alive
-  if (bp->ip.ap.str.flags & PKT_CONTROL) {
-    rr->remote_links |= 1 << link_num;
-    return 0;
-  }
-
-  return 1;
+  rr->remote_links |= 1 << link_num; // Make sure that the link is marked as working
 }
 
-void rr_deliver(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct evt_core_fdinfo* fdinfo) {
+void rr_deliver(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   struct evt_core_fdinfo *to_fdinfo = NULL;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct rr_ctx* rr = app_ctx->misc;
   char url[255];
 
-  // 1. A whole packet has been read, we will find its target
+  // 0. We update our cursor
+  rr->recv_id = bp->ip.ap.str.id;
+
+  // 1. We check that we don't have a running timeout
+  int idx_real = bp->ip.ap.str.id % PACKET_BUFFER_SIZE;
+  if (rr->wait[idx_real].on) {
+    rr->wait[idx_real].on = 0;
+    evt_core_rm_fd (ctx, rr->wait[idx_real].timer_fd);
+  }
+
+  // 2. We free the buffer if it's a control packet and quit
+  if (bp->ip.ap.str.flags & PKT_CONTROL) {
+    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    return;
+  }
+
+  // 3. A whole packet has been read, we will find its target
   sprintf(url, "udp:write:127.0.0.1:%d", bp->ip.ap.str.port);
   to_fdinfo = evt_core_get_from_url (ctx, url);
   if (to_fdinfo == NULL) {
@@ -167,33 +175,36 @@ void rr_deliver(struct evt_core_ctx* ctx, struct buffer_packet* bp, struct evt_c
     mv_buffer_wtor (app_ctx, fdinfo, bp);
   }
 
-  // 2. We move the buffer and notify the target
+  // 4. We move the buffer and notify the target
   mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
   rr_on_udp_write(ctx, to_fdinfo);
 }
 
-void rr_pkt_recv(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
-  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+void rr_pkt_unroll(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx) {
   struct rr_ctx* rr = app_ctx->misc;
+  struct evt_core_fdinfo* fdinfo = NULL;
+  struct buffer_packet* bp = NULL;
 
-  do {
-    if(rr_update_states(ctx, bp, fdinfo)) rr_deliver(ctx, bp, fdinfo);
-    do {
-      struct deferred_pkt* def = &rr->real[(rr->recv_id+1) % PACKET_BUFFER_SIZE];
-      if (def->bp == NULL) break;
-      def->bp = NULL;
-      struct evt_core_fdinfo* fdinfo = evt_core_get_from_fd (ctx, def->link_fd);
-      if (fdinfo == NULL) {
-        fprintf(stderr, "An error occured as the link seems to be closed for the requested fd\n");
-        rr->recv_id++;
-      }
-    } while (fdinfo == NULL);
-  } while(1);
+  while(1) {
+    struct deferred_pkt* def = &rr->real[(rr->recv_id+1) % PACKET_BUFFER_SIZE];
+    if (def->bp == NULL) break;
+    bp = def->bp;
+    def->bp = NULL;
+    fdinfo = evt_core_get_from_fd (ctx, def->link_fd);
+    if (fdinfo == NULL) {
+      fprintf(stderr, "An error occured as the link seems to be closed for the requested fd\n");
+      rr->recv_id++;
+      continue;
+    }
+
+    rr_deliver(ctx, fdinfo, bp);
+  }
 }
 
 int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct buffer_packet* bp;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct rr_ctx* rr = app_ctx->misc;
   int read_res = FDS_READY;
 
   // 1. Get current read buffer OR a new read buffer OR subscribe to be notified later
@@ -207,7 +218,8 @@ int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   }
 
   // 3. Logic on packet
-  rr_pkt_recv (ctx, fdinfo, bp);
+  rr_pkt_register(ctx, fdinfo, bp);
+  rr_pkt_unroll (ctx, app_ctx);
 
   return 0;
 co_error:
@@ -242,22 +254,9 @@ co_error:
 int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
-  uint16_t id;
 
-  evt_core_rm_fd (ctx, fdinfo->fd); // We use the timeout only once
-  int fd = (int)fdinfo->other;
-  struct evt_core_fdinfo* tfdinfo = evt_core_get_from_fd (ctx, fd);
-  if (tfdinfo == NULL) {
-    fprintf(stderr, "An error occured as the link seems to be closed for the requested fd\n");
-    rr->recv_id++;
-    return 1;
-  }
 
-  rr->stub_bp.ip.ap.str.id = 0; //?
-  rr->stub_bp.ip.ap.str.flags = PKT_TIMEOUT;
-  rr->stub_bp.ip.ap.str.deltat = rr->mjit;
-  rr->stub_bp.ip.ap.str.prevlink = UINT8_MAX;
-  rr_pkt_recv (ctx, tfdinfo, &rr->stub_bp);
+
   return 1;
 }
 
diff --git a/src/algo_utils.h b/src/algo_utils.h
index 0c4d93e..820dfef 100644
--- a/src/algo_utils.h
+++ b/src/algo_utils.h
@@ -3,7 +3,7 @@
 #include <glib-2.0/glib.h>
 #include <glib-2.0/gmodule.h>
 #include <glib-2.0/glib-object.h>
-#define PACKET_BUFFER_SIZE 10
+#define PACKET_BUFFER_SIZE 100
 
 typedef void (*algo_ctx_free_misc)(void*);
 
diff --git a/src/packet.h b/src/packet.h
index 29e13b8..2928836 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -29,8 +29,7 @@ enum BP_MODE {
 };
 
 enum PKT_FLAGS {
-  PKT_TIMEOUT = 1 << 0,
-  PKT_CONTROL = 1 << 1
+  PKT_CONTROL = 1 << 0
 };
 
 union abstract_packet {

From 298e385dae24fe8f3550927be3835f2f209d7275 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Tue, 19 Mar 2019 15:54:31 +0100
Subject: [PATCH 09/26] Handle timeout

---
 src/algo_rr.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 1c613a0..b748948 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -6,8 +6,8 @@
 struct waited_pkt {
   uint16_t id;
   int link_num;
-  int timer_fd;
   uint8_t on;
+  int timer_fd;
 };
 
 struct deferred_pkt {
@@ -102,6 +102,7 @@ int set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec, struct waited_pkt
   fdinfo.free_other = NULL;
   sprintf(fdinfo.url, "timer:%ld:1", micro_sec);
   evt_core_add_fd (evts, &fdinfo);
+
   return fdinfo.fd;
 }
 
@@ -130,7 +131,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
     rr->wait[idx_waited].on = 1;
     rr->wait[idx_waited].id = bp->ip.ap.str.id;
     rr->wait[idx_waited].link_num = bp->ip.ap.str.prevlink;
-    set_timeout(ctx, timeout, &rr->wait[idx_waited]);
+    rr->wait[idx_waited].timer_fd = set_timeout(ctx, timeout, &rr->wait[idx_waited]);
   }
 
   // 4. We queue the packet
@@ -255,7 +256,14 @@ int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
 
+  struct waited_pkt* pkt = fdinfo->other;
+  pkt->on = 0;
+  evt_core_rm_fd(ctx, fdinfo->fd);
 
+  rr->remote_links &= 0xffff ^ 1 << pkt->link_num;
+  rr->recv_id = pkt->id;
+
+  rr_pkt_unroll (ctx, app_ctx);
 
   return 1;
 }

From d48c5753fb9c669503eacaa4ac2e90cb8f707ed8 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Tue, 19 Mar 2019 16:35:11 +0100
Subject: [PATCH 10/26] All the pieces should be here

---
 src/algo_rr.c | 105 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 99 insertions(+), 6 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index b748948..d54e165 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -23,6 +23,7 @@ struct rr_ctx {
   uint16_t recv_id;
   uint16_t sent_id;
   uint8_t current_link;
+  struct timespec emit_time;
   struct deferred_pkt real[PACKET_BUFFER_SIZE];
   struct waited_pkt wait[PACKET_BUFFER_SIZE];
 };
@@ -202,6 +203,8 @@ void rr_pkt_unroll(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx) {
   }
 }
 
+//------
+
 int rr_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct buffer_packet* bp;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
@@ -252,6 +255,97 @@ co_error:
   exit(EXIT_FAILURE);
 }
 
+int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
+  struct buffer_packet* bp;
+  struct evt_core_fdinfo *to_fdinfo;
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct rr_ctx* rr = app_ctx->misc;
+  int read_res = FDS_READY;
+  char url[255];
+
+  // 1. Get current read buffer OR a new read buffer OR subscribe to be notified later
+  if ((bp = get_read_buffer(app_ctx, fdinfo)) == NULL) return 1;
+
+  // 2. Read packet from socket
+  bp->ip.ap.str.port = url_get_port_int (fdinfo->url);
+  read_res = read_packet_from_udp (fdinfo->fd, bp, fdinfo->other);
+  if (read_res == FDS_ERR) goto co_error;
+  if (read_res == FDS_AGAIN) return 1;
+
+  // 3. Prepare RR state and packet values
+  struct timespec curr;
+  int secs, nsecs;
+  uint64_t micro_sec;
+
+  if (clock_gettime(CLOCK_MONOTONIC, &curr) == -1){
+    perror("clock_gettime error");
+    exit(EXIT_FAILURE);
+  }
+
+  secs = curr.tv_sec - rr->emit_time.tv_sec;
+  nsecs = curr.tv_nsec - rr->emit_time.tv_nsec;
+  micro_sec = secs * 1000000 + nsecs / 1000;
+  if (micro_sec > rr->mjit) micro_sec = rr->mjit;
+
+  bp->ip.ap.str.id = rr->sent_id;
+  bp->ip.ap.str.flags = 0;
+  bp->ip.ap.str.deltat = micro_sec;
+  bp->ip.ap.str.bitfield = rr->remote_links;
+  bp->ip.ap.str.prevlink = rr->current_link;
+
+  do {
+    rr->current_link = (rr->current_link) + 1 % 10;
+  } while (!(rr->my_links & (1 << rr->current_link)));
+  rr->emit_time = curr;
+  rr->sent_id++;
+
+  // 4. A whole packet has been read, we will find someone to write it
+  sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + rr->current_link); //@FIXME Hardcoded
+  to_fdinfo = evt_core_get_from_url (ctx, url);
+  if (to_fdinfo == NULL) {
+    fprintf(stderr, "No fd for URL %s in udp-read. Dropping packet :( \n", url);
+    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    return 1;
+  }
+  //printf("Pass packet from %s to %s\n", fdinfo->url, url);
+
+  // 5. We move the buffer and notify the target
+  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
+  rr_on_tcp_write(ctx, to_fdinfo);
+
+  return 0;
+
+co_error:
+  perror("Failed to UDP read");
+  exit(EXIT_FAILURE);
+}
+
+int rr_on_tcp_write(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
+  struct buffer_packet* bp;
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  int write_res = FDS_READY;
+
+  // 1. Get current write buffer OR a buffer from the waiting queue OR leave
+  if ((bp = get_write_buffer(app_ctx, fdinfo)) == NULL) return 1;
+
+  // 2. Write data from the buffer to the socket
+  while (bp->mode == BP_WRITING) {
+    write_res = write_packet_to_tcp(fdinfo->fd, bp);
+    if (write_res == FDS_ERR) goto co_error;
+    if (write_res == FDS_AGAIN) return 1;
+  }
+
+  // 3. A whole packet has been written
+  // Release the buffer and notify
+  mv_buffer_wtor(app_ctx, fdinfo, bp);
+  notify_read(ctx, app_ctx);
+
+  return 0;
+co_error:
+  perror("Failed to TCP write");
+  exit(EXIT_FAILURE);
+}
+
 int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
@@ -327,23 +421,22 @@ void algo_rr(struct evt_core_ctx* evt, struct algo_skel* as) {
   as->on_tcp_read.err_cb = rr_on_err;
   ctx->ref_count++;
 
-/*
+
   as->on_udp_read.name = "udp-read";
   as->on_udp_read.flags = EPOLLIN | EPOLLET;
   as->on_udp_read.app_ctx = ctx;
   as->on_udp_read.free_app_ctx = free_naive;
-  as->on_udp_read.cb = on_udp_read;
-  as->on_udp_read.err_cb = on_err;
+  as->on_udp_read.cb = rr_on_udp_read;
+  as->on_udp_read.err_cb = rr_on_err;
   ctx->ref_count++;
 
   as->on_tcp_write.name = "tcp-write";
   as->on_tcp_write.flags = EPOLLOUT | EPOLLET | EPOLLRDHUP;
   as->on_tcp_write.app_ctx = ctx;
   as->on_tcp_write.free_app_ctx = free_naive;
-  as->on_tcp_write.cb = on_tcp_write;
-  as->on_tcp_write.err_cb = on_err;
+  as->on_tcp_write.cb = rr_on_tcp_write;
+  as->on_tcp_write.err_cb = rr_on_err;
   ctx->ref_count++;
-*/
 
   as->on_udp_write.name = "udp-write";
   as->on_udp_write.flags = EPOLLOUT | EPOLLET;

From ef49d07dd9a1114486169dcb01c4ad393986026f Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Tue, 19 Mar 2019 16:42:54 +0100
Subject: [PATCH 11/26] Better loop to choose links

---
 src/algo_rr.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index d54e165..df665c2 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -293,17 +293,21 @@ int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   bp->ip.ap.str.bitfield = rr->remote_links;
   bp->ip.ap.str.prevlink = rr->current_link;
 
-  do {
+  int max = 10;
+  while(1) {
+    if (max-- < 0) break;
     rr->current_link = (rr->current_link) + 1 % 10;
-  } while (!(rr->my_links & (1 << rr->current_link)));
+    if (!(rr->my_links & (1 << rr->current_link))) continue;
+    sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + rr->current_link); //@FIXME Hardcoded
+    to_fdinfo = evt_core_get_from_url (ctx, url);
+    if (to_fdinfo != NULL) break;
+  }
   rr->emit_time = curr;
   rr->sent_id++;
 
   // 4. A whole packet has been read, we will find someone to write it
-  sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + rr->current_link); //@FIXME Hardcoded
-  to_fdinfo = evt_core_get_from_url (ctx, url);
   if (to_fdinfo == NULL) {
-    fprintf(stderr, "No fd for URL %s in udp-read. Dropping packet :( \n", url);
+    fprintf(stderr, "No fd for URL %s in udp-read. Dropping packet :( \n", fdinfo->url);
     mv_buffer_wtor (app_ctx, fdinfo, bp);
     return 1;
   }

From 32931978606ac8c70cf5aa8b3a0e49fc2595070b Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Tue, 19 Mar 2019 17:09:42 +0100
Subject: [PATCH 12/26] Fix init error

---
 src/algo_skel.h    |  5 +++++
 src/donar.c        |  8 ++------
 src/donar_client.c | 15 +++++++++------
 src/donar_client.h |  2 +-
 src/donar_server.c | 15 +++++++++------
 src/donar_server.h |  2 +-
 6 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/src/algo_skel.h b/src/algo_skel.h
index 7c51b8e..40f140f 100644
--- a/src/algo_skel.h
+++ b/src/algo_skel.h
@@ -20,6 +20,7 @@ typedef void (*algo_init)(struct evt_core_ctx* ctx, struct algo_skel* as);
 
 void init_algo(struct evt_core_ctx* ctx, struct algo_skel* as, char* name);
 void algo_naive(struct evt_core_ctx* ctx, struct algo_skel* as);
+void algo_rr(struct evt_core_ctx* ctx, struct algo_skel* as);
 
 struct algo_desc {
   algo_init init;
@@ -30,5 +31,9 @@ static struct algo_desc available_algo[] = {
   {
     .init = algo_naive,
     .name = "naive"
+  },
+  {
+    .init = algo_rr,
+    .name = "rr"
   }
 };
diff --git a/src/donar.c b/src/donar.c
index fe5dc53..9dfc185 100644
--- a/src/donar.c
+++ b/src/donar.c
@@ -58,18 +58,14 @@ int main(int argc, char** argv) {
     if (!(is_server ^ is_client)) goto in_error;
     if (algo == NULL) goto in_error;
 
-    struct algo_skel as = {0};
-
     if (is_server) {
       struct donar_server_ctx ctx;
-      init_algo(&ctx.evts, &as, algo);
       if (exposed_ports->len < 1 && remote_ports->len < 1) goto in_error;
-      donar_server(&ctx, &as, exposed_ports, remote_ports);
+      donar_server(&ctx, algo, exposed_ports, remote_ports);
     } else if (is_client) {
       struct donar_client_ctx ctx;
-      init_algo(&ctx.evts, &as, algo);
       if ((exposed_ports->len < 1 && remote_ports->len < 1) || onion_file == NULL) goto in_error;
-      donar_client(&ctx, &as, onion_file, exposed_ports, remote_ports);
+      donar_client(&ctx, algo, onion_file, exposed_ports, remote_ports);
     }
     goto terminate;
 
diff --git a/src/donar_client.c b/src/donar_client.c
index 2a0bd21..bc93276 100644
--- a/src/donar_client.c
+++ b/src/donar_client.c
@@ -122,9 +122,12 @@ on_socks5_err:
     return 1;
 }
 
-void donar_client(struct donar_client_ctx* ctx, struct algo_skel* algo,
+void donar_client(struct donar_client_ctx* ctx, char* algoname,
                   char* onion_file,  GPtrArray* exposed_ports, GPtrArray* remote_ports) {
+  struct algo_skel algo = {0};
+
   evt_core_init (&(ctx->evts));
+  init_algo(&ctx->evts, &algo, algoname);
   struct evt_core_cat init_socks5 = {
     .app_ctx = ctx,
     .free_app_ctx = NULL,
@@ -135,11 +138,11 @@ void donar_client(struct donar_client_ctx* ctx, struct algo_skel* algo,
     .socklist = NULL
   };
   evt_core_add_cat (&(ctx->evts), &init_socks5);
-  evt_core_add_cat (&(ctx->evts), &(algo->on_tcp_co));
-  evt_core_add_cat (&(ctx->evts), &(algo->on_udp_read));
-  evt_core_add_cat (&(ctx->evts), &(algo->on_tcp_read));
-  evt_core_add_cat (&(ctx->evts), &(algo->on_udp_write));
-  evt_core_add_cat (&(ctx->evts), &(algo->on_tcp_write));
+  evt_core_add_cat (&(ctx->evts), &(algo.on_tcp_co));
+  evt_core_add_cat (&(ctx->evts), &(algo.on_udp_read));
+  evt_core_add_cat (&(ctx->evts), &(algo.on_tcp_read));
+  evt_core_add_cat (&(ctx->evts), &(algo.on_udp_write));
+  evt_core_add_cat (&(ctx->evts), &(algo.on_tcp_write));
   printf("--- Categories created\n");
 
   load_onion_services (ctx, onion_file, CLIENT_PORT_SIZE);
diff --git a/src/donar_client.h b/src/donar_client.h
index 248bdb1..497259d 100644
--- a/src/donar_client.h
+++ b/src/donar_client.h
@@ -19,5 +19,5 @@ struct donar_client_ctx {
   } client_sock[CLIENT_PORT_SIZE];
 };
 
-void donar_client(struct donar_client_ctx* ctx, struct algo_skel* as,
+void donar_client(struct donar_client_ctx* ctx, char* algoname,
                   char* onion_file, GPtrArray* exposed_ports, GPtrArray* remote_ports);
diff --git a/src/donar_server.c b/src/donar_server.c
index e5f8ffb..f557261 100644
--- a/src/donar_server.c
+++ b/src/donar_server.c
@@ -51,14 +51,17 @@ socket_create_err:
   exit(EXIT_FAILURE);
 }
 
-void donar_server(struct donar_server_ctx* ctx, struct algo_skel* algo,
+void donar_server(struct donar_server_ctx* ctx, char* algoname,
                   GPtrArray* exposed_ports, GPtrArray* remote_ports) {
+  struct algo_skel algo = {0};
+
   evt_core_init (&(ctx->evts));
-  evt_core_add_cat (&(ctx->evts), &(algo->on_tcp_co));
-  evt_core_add_cat (&(ctx->evts), &(algo->on_udp_read));
-  evt_core_add_cat (&(ctx->evts), &(algo->on_tcp_read));
-  evt_core_add_cat (&(ctx->evts), &(algo->on_udp_write));
-  evt_core_add_cat (&(ctx->evts), &(algo->on_tcp_write));
+  init_algo(&ctx->evts, &algo, algoname);
+  evt_core_add_cat (&(ctx->evts), &(algo.on_tcp_co));
+  evt_core_add_cat (&(ctx->evts), &(algo.on_udp_read));
+  evt_core_add_cat (&(ctx->evts), &(algo.on_tcp_read));
+  evt_core_add_cat (&(ctx->evts), &(algo.on_udp_write));
+  evt_core_add_cat (&(ctx->evts), &(algo.on_tcp_write));
 
   printf("--- Categories created\n");
 
diff --git a/src/donar_server.h b/src/donar_server.h
index d021d8f..bc4875b 100644
--- a/src/donar_server.h
+++ b/src/donar_server.h
@@ -20,5 +20,5 @@ struct donar_server_ctx {
   uint16_t ports[PORT_SIZE];
 };
 
-void donar_server(struct donar_server_ctx* ctx, struct algo_skel* algo,
+void donar_server(struct donar_server_ctx* ctx, char* algoname,
                   GPtrArray* exposed_ports, GPtrArray* remote_ports);

From 9a0c78ebbc7e281e9e68978eefecc064f818ba87 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Tue, 19 Mar 2019 17:16:49 +0100
Subject: [PATCH 13/26] Some log on algo rr

---
 src/algo_rr.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index df665c2..5eda380 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -257,7 +257,7 @@ co_error:
 
 int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct buffer_packet* bp;
-  struct evt_core_fdinfo *to_fdinfo;
+  struct evt_core_fdinfo *to_fdinfo = NULL;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
   int read_res = FDS_READY;
@@ -300,7 +300,10 @@ int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
     if (!(rr->my_links & (1 << rr->current_link))) continue;
     sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + rr->current_link); //@FIXME Hardcoded
     to_fdinfo = evt_core_get_from_url (ctx, url);
-    if (to_fdinfo != NULL) break;
+    if (to_fdinfo != NULL) {
+      printf("Selected url %s\n", url);
+      break;
+    }
   }
   rr->emit_time = curr;
   rr->sent_id++;

From d028638c95baad53575ea4069af693ced3667d58 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@dufour.io>
Date: Tue, 19 Mar 2019 17:54:28 +0100
Subject: [PATCH 14/26] Fix small bug + debug

---
 src/algo_rr.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 5eda380..336c9b0 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -111,6 +111,8 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
 
+    printf("Selected url %s for pkt %d to be queued for delivery\n", fdinfo->url, bp->ip.ap.str.id);
+
   // 1. Update links I can use thanks to target feedback
   if (bp->ip.ap.str.id > rr->my_links_ver) {
     rr->my_links = bp->ip.ap.str.bitfield;
@@ -153,6 +155,8 @@ void rr_deliver(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct
   struct rr_ctx* rr = app_ctx->misc;
   char url[255];
 
+  printf("Selected url %s for pkt %d to be delivered\n", fdinfo->url, bp->ip.ap.str.id);
+
   // 0. We update our cursor
   rr->recv_id = bp->ip.ap.str.id;
 
@@ -296,12 +300,13 @@ int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   int max = 10;
   while(1) {
     if (max-- < 0) break;
-    rr->current_link = (rr->current_link) + 1 % 10;
+    rr->current_link = (rr->current_link + 1) % 10;
     if (!(rr->my_links & (1 << rr->current_link))) continue;
     sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + rr->current_link); //@FIXME Hardcoded
+    printf("-- Trying %s\n", url);
     to_fdinfo = evt_core_get_from_url (ctx, url);
     if (to_fdinfo != NULL) {
-      printf("Selected url %s\n", url);
+      printf("Selected url %s for pkt %d to be sent on Tor\n", url, bp->ip.ap.str.id);
       break;
     }
   }
@@ -409,6 +414,7 @@ void algo_rr(struct evt_core_ctx* evt, struct algo_skel* as) {
   rr->mjit = 200;
   rr->my_links = 0xff;
   rr->remote_links = 0xff;
+  rr->sent_id = 1;
   ctx->misc = rr;
   for (int i = 0; i < sizeof(ctx->bps) / sizeof(ctx->bps[0]); i++) {
     g_queue_push_tail(ctx->free_buffer, &(ctx->bps[i]));

From 372aae6f6134f8ad395620fde4772523f2f9d46b Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@dufour.io>
Date: Tue, 19 Mar 2019 18:22:23 +0100
Subject: [PATCH 15/26] We fixed the comparison operation

---
 src/algo_rr.c |  5 +++++
 src/utils.c   | 24 ++++++++++++------------
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 336c9b0..b7c69fe 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -79,6 +79,7 @@ int set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec, struct waited_pkt
   fdinfo.cat = &cat;
   fdinfo.url = url;
 
+  printf("Will add a timeout of %ld ms\n", micro_sec);
   if (clock_gettime(CLOCK_REALTIME, &now) == -1) {
     perror("clock_gettime");
     exit(EXIT_FAILURE);
@@ -120,6 +121,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   }
 
   // 2. If packet arrived too late, we discard it
+  printf("== %d > %d ? %d\n", rr->recv_id, bp->ip.ap.str.id - 1, ring_gt(rr->recv_id, bp->ip.ap.str.id - 1));
   if (ring_gt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     // Packet has already been delivered or dropped, we free the buffer
     mv_buffer_wtor (app_ctx, fdinfo, bp);
@@ -127,6 +129,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   }
 
   // 3. If packet arrived too early, we register a timer
+  printf("== %d < %d ? %d\n", rr->recv_id, bp->ip.ap.str.id - 1, ring_lt(rr->recv_id, bp->ip.ap.str.id - 1));
   if (ring_lt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.str.deltat;
     if (timeout <= 0) timeout = 0;
@@ -362,6 +365,8 @@ int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
 
+  printf("Timer has been triggered\n");
+
   struct waited_pkt* pkt = fdinfo->other;
   pkt->on = 0;
   evt_core_rm_fd(ctx, fdinfo->fd);
diff --git a/src/utils.c b/src/utils.c
index 281238c..24e98b8 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -61,22 +61,22 @@ int ring_buffer_used_space(struct ring_buffer* rb) {
   return RING_BUFFER_SIZE - ring_buffer_free_space (rb);
 }
 
-int ring_gt(uint16_t v1, uint16_t v2) {
-  int32_t vv1 = (int32_t) v1, vv2 = (int32_t) v2;
-  return v1 != v2 && (vv1 - vv2) % UINT16_MAX <= UINT16_MAX / 2;
-}
-
+// Why we are using modulo, plus and modulo again:
+// https://stackoverflow.com/a/1907585
 int ring_ge(uint16_t v1, uint16_t v2) {
-  int32_t vv1 = (int32_t) v1, vv2 = (int32_t) v2;
-  return (vv1 - vv2) % UINT16_MAX <= UINT16_MAX / 2;
+  uint32_t vv1 = (uint32_t) v1, vv2 = (uint32_t) v2;
+  return (((vv1 - vv2) % UINT16_MAX) + UINT16_MAX) % UINT16_MAX <= UINT16_MAX / 2;
 }
 
-int ring_lt(uint16_t v1, uint16_t v2) {
-  int32_t vv1 = (int32_t) v1, vv2 = (int32_t) v2;
-  return v1 != v2 && (vv1 - vv2) % UINT16_MAX > UINT16_MAX / 2;
+int ring_gt(uint16_t v1, uint16_t v2) {
+  if (v1 == v2) return 0;
+  return ring_ge(v1,v2);
 }
 
 int ring_le(uint16_t v1, uint16_t v2) {
-  int32_t vv1 = (int32_t) v1, vv2 = (int32_t) v2;
-  return (vv1 - vv2) % UINT16_MAX > UINT16_MAX / 2;
+  return ring_gt(v2, v1);
+}
+
+int ring_lt(uint16_t v1, uint16_t v2) {
+  return ring_ge(v2, v1);
 }

From c58112f0b73363d8024e3e5284a67ac3ea7cd5cf Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@dufour.io>
Date: Tue, 19 Mar 2019 18:27:14 +0100
Subject: [PATCH 16/26] Fix a free bug

---
 src/evt_core.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/evt_core.c b/src/evt_core.c
index 7e3ee92..ffe547a 100644
--- a/src/evt_core.c
+++ b/src/evt_core.c
@@ -4,7 +4,7 @@ void free_fdinfo(void* v) {
   struct evt_core_fdinfo* fdinfo = (struct evt_core_fdinfo*)v;
   close(fdinfo->fd); // We close the file descriptor here
   if (fdinfo->url != NULL) free(fdinfo->url); // We free the URL here;
-  if (fdinfo->other != NULL) fdinfo->free_other(fdinfo->other);
+  if (fdinfo->free_other != NULL) fdinfo->free_other(fdinfo->other);
   free(v);
 }
 

From 5a6685e7c2908d2e987766bdbd1e265529cebd4d Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@dufour.io>
Date: Tue, 19 Mar 2019 18:50:56 +0100
Subject: [PATCH 17/26] WIP debugging prog

---
 src/algo_rr.c | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index b7c69fe..a07ce6a 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -19,7 +19,7 @@ struct rr_ctx {
   uint8_t my_links;
   uint16_t my_links_ver;
   uint8_t remote_links;
-  uint16_t mjit;
+  int64_t mjit;
   uint16_t recv_id;
   uint16_t sent_id;
   uint8_t current_link;
@@ -70,7 +70,7 @@ co_error:
   exit(EXIT_FAILURE);
 }
 
-int set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec, struct waited_pkt* wpkt) {
+int set_timeout(struct evt_core_ctx* evts, uint64_t milli_sec, struct waited_pkt* wpkt) {
   struct timespec now;
   struct itimerspec timer_config;
   char url[1024];
@@ -79,14 +79,15 @@ int set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec, struct waited_pkt
   fdinfo.cat = &cat;
   fdinfo.url = url;
 
-  printf("Will add a timeout of %ld ms\n", micro_sec);
+  printf("Will add a timeout of %ld ms\n", milli_sec);
   if (clock_gettime(CLOCK_REALTIME, &now) == -1) {
     perror("clock_gettime");
     exit(EXIT_FAILURE);
   }
 
-  timer_config.it_value.tv_sec = now.tv_sec + micro_sec / 1000;
-  timer_config.it_value.tv_nsec = now.tv_nsec + micro_sec % 1000 * 1000000;
+  uint64_t ns = now.tv_nsec + (milli_sec % 1000) * 1000000;
+  timer_config.it_value.tv_sec = now.tv_sec + milli_sec / 1000 + ns / 1000000000;
+  timer_config.it_value.tv_nsec = ns % 1000000000;
   timer_config.it_interval.tv_sec = 60;
   timer_config.it_interval.tv_nsec = 0;
 
@@ -96,13 +97,13 @@ int set_timeout(struct evt_core_ctx* evts, uint64_t micro_sec, struct waited_pkt
     exit(EXIT_FAILURE);
   }
   if (timerfd_settime (fdinfo.fd, TFD_TIMER_ABSTIME, &timer_config, NULL) == -1) {
-    perror("Unable to timerfd_time");
+    perror("Unable to timerfd_settime");
     exit(EXIT_FAILURE);
   }
   fdinfo.cat->name = "timeout";
   fdinfo.other = wpkt; // Should put the link number and the id
   fdinfo.free_other = NULL;
-  sprintf(fdinfo.url, "timer:%ld:1", micro_sec);
+  sprintf(fdinfo.url, "timer:%ld:1", milli_sec);
   evt_core_add_fd (evts, &fdinfo);
 
   return fdinfo.fd;
@@ -121,7 +122,6 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   }
 
   // 2. If packet arrived too late, we discard it
-  printf("== %d > %d ? %d\n", rr->recv_id, bp->ip.ap.str.id - 1, ring_gt(rr->recv_id, bp->ip.ap.str.id - 1));
   if (ring_gt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     // Packet has already been delivered or dropped, we free the buffer
     mv_buffer_wtor (app_ctx, fdinfo, bp);
@@ -129,15 +129,16 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   }
 
   // 3. If packet arrived too early, we register a timer
-  printf("== %d < %d ? %d\n", rr->recv_id, bp->ip.ap.str.id - 1, ring_lt(rr->recv_id, bp->ip.ap.str.id - 1));
   if (ring_lt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.str.deltat;
+    printf("%ld - %ld = %ld\n", rr->mjit, (int64_t) bp->ip.ap.str.deltat, timeout);
     if (timeout <= 0) timeout = 0;
     int idx_waited = (bp->ip.ap.str.id - 1) % PACKET_BUFFER_SIZE;
     rr->wait[idx_waited].on = 1;
     rr->wait[idx_waited].id = bp->ip.ap.str.id;
     rr->wait[idx_waited].link_num = bp->ip.ap.str.prevlink;
     rr->wait[idx_waited].timer_fd = set_timeout(ctx, timeout, &rr->wait[idx_waited]);
+    g_hash_table_remove(app_ctx->used_buffer, &fdinfo->fd); // We remove the packet from the reading buffer
   }
 
   // 4. We queue the packet
@@ -282,7 +283,7 @@ int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   // 3. Prepare RR state and packet values
   struct timespec curr;
   int secs, nsecs;
-  uint64_t micro_sec;
+  uint64_t mili_sec;
 
   if (clock_gettime(CLOCK_MONOTONIC, &curr) == -1){
     perror("clock_gettime error");
@@ -291,12 +292,12 @@ int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
 
   secs = curr.tv_sec - rr->emit_time.tv_sec;
   nsecs = curr.tv_nsec - rr->emit_time.tv_nsec;
-  micro_sec = secs * 1000000 + nsecs / 1000;
-  if (micro_sec > rr->mjit) micro_sec = rr->mjit;
+  mili_sec = secs * 1000 + nsecs / 1000000;
+  if (mili_sec > rr->mjit) mili_sec = rr->mjit;
 
   bp->ip.ap.str.id = rr->sent_id;
   bp->ip.ap.str.flags = 0;
-  bp->ip.ap.str.deltat = micro_sec;
+  bp->ip.ap.str.deltat = mili_sec;
   bp->ip.ap.str.bitfield = rr->remote_links;
   bp->ip.ap.str.prevlink = rr->current_link;
 

From f312b5fe18f690a648be5959856d20ee1a5df786 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@dufour.io>
Date: Tue, 19 Mar 2019 20:48:07 +0100
Subject: [PATCH 18/26] Fix a ring comparison bug

---
 src/algo_rr.c | 15 +++++++--------
 src/packet.h  |  2 +-
 src/utils.c   |  2 +-
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index a07ce6a..10fa76e 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -79,7 +79,7 @@ int set_timeout(struct evt_core_ctx* evts, uint64_t milli_sec, struct waited_pkt
   fdinfo.cat = &cat;
   fdinfo.url = url;
 
-  printf("Will add a timeout of %ld ms\n", milli_sec);
+  //printf("Will add a timeout of %ld ms\n", milli_sec);
   if (clock_gettime(CLOCK_REALTIME, &now) == -1) {
     perror("clock_gettime");
     exit(EXIT_FAILURE);
@@ -113,7 +113,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
 
-    printf("Selected url %s for pkt %d to be queued for delivery\n", fdinfo->url, bp->ip.ap.str.id);
+  printf("Selected url %s for pkt %d to be queued for delivery\n", fdinfo->url, bp->ip.ap.str.id);
 
   // 1. Update links I can use thanks to target feedback
   if (bp->ip.ap.str.id > rr->my_links_ver) {
@@ -124,6 +124,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   // 2. If packet arrived too late, we discard it
   if (ring_gt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     // Packet has already been delivered or dropped, we free the buffer
+    fprintf(stderr, "Packet %d arrived too late (current: %d)\n", bp->ip.ap.str.id, rr->recv_id);
     mv_buffer_wtor (app_ctx, fdinfo, bp);
     return;
   }
@@ -131,20 +132,20 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   // 3. If packet arrived too early, we register a timer
   if (ring_lt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.str.deltat;
-    printf("%ld - %ld = %ld\n", rr->mjit, (int64_t) bp->ip.ap.str.deltat, timeout);
+    //printf("%ld - %ld = %ld\n", rr->mjit, (int64_t) bp->ip.ap.str.deltat, timeout);
     if (timeout <= 0) timeout = 0;
     int idx_waited = (bp->ip.ap.str.id - 1) % PACKET_BUFFER_SIZE;
     rr->wait[idx_waited].on = 1;
     rr->wait[idx_waited].id = bp->ip.ap.str.id;
     rr->wait[idx_waited].link_num = bp->ip.ap.str.prevlink;
     rr->wait[idx_waited].timer_fd = set_timeout(ctx, timeout, &rr->wait[idx_waited]);
-    g_hash_table_remove(app_ctx->used_buffer, &fdinfo->fd); // We remove the packet from the reading buffer
   }
 
   // 4. We queue the packet
   int idx_real = bp->ip.ap.str.id % PACKET_BUFFER_SIZE;
   rr->real[idx_real].bp = bp;
   rr->real[idx_real].link_fd = fdinfo->fd;
+  g_hash_table_remove(app_ctx->used_buffer, &fdinfo->fd); // We remove the packet from the reading buffer
 
   // 5. We make sure that the remote link is set to up
   char buffer[16];
@@ -307,7 +308,7 @@ int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
     rr->current_link = (rr->current_link + 1) % 10;
     if (!(rr->my_links & (1 << rr->current_link))) continue;
     sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + rr->current_link); //@FIXME Hardcoded
-    printf("-- Trying %s\n", url);
+    //printf("-- Trying %s\n", url);
     to_fdinfo = evt_core_get_from_url (ctx, url);
     if (to_fdinfo != NULL) {
       printf("Selected url %s for pkt %d to be sent on Tor\n", url, bp->ip.ap.str.id);
@@ -366,13 +367,11 @@ int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
 
-  printf("Timer has been triggered\n");
-
   struct waited_pkt* pkt = fdinfo->other;
   pkt->on = 0;
   evt_core_rm_fd(ctx, fdinfo->fd);
 
-  rr->remote_links &= 0xffff ^ 1 << pkt->link_num;
+  //rr->remote_links &= 0xffff ^ 1 << pkt->link_num;
   rr->recv_id = pkt->id;
 
   rr_pkt_unroll (ctx, app_ctx);
diff --git a/src/packet.h b/src/packet.h
index 2928836..e97821b 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -37,7 +37,7 @@ union abstract_packet {
   struct {
     uint16_t size;
     uint16_t port;
-    uint8_t id;
+    uint16_t id;
     uint8_t bitfield;
     uint8_t prevlink;
     uint16_t deltat;
diff --git a/src/utils.c b/src/utils.c
index 24e98b8..ad49cf9 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -64,7 +64,7 @@ int ring_buffer_used_space(struct ring_buffer* rb) {
 // Why we are using modulo, plus and modulo again:
 // https://stackoverflow.com/a/1907585
 int ring_ge(uint16_t v1, uint16_t v2) {
-  uint32_t vv1 = (uint32_t) v1, vv2 = (uint32_t) v2;
+  int64_t vv1 = (int64_t) v1, vv2 = (int64_t) v2;
   return (((vv1 - vv2) % UINT16_MAX) + UINT16_MAX) % UINT16_MAX <= UINT16_MAX / 2;
 }
 

From 94e3ae872f381c8f90628858339645cd4fee2264 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@dufour.io>
Date: Tue, 19 Mar 2019 21:06:01 +0100
Subject: [PATCH 19/26] Add a check

---
 src/algo_rr.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 10fa76e..dbef6d8 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -170,6 +170,7 @@ void rr_deliver(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct
   if (rr->wait[idx_real].on) {
     rr->wait[idx_real].on = 0;
     evt_core_rm_fd (ctx, rr->wait[idx_real].timer_fd);
+    printf("Removed timer for packet %d\n",bp->ip.ap.str.id);
   }
 
   // 2. We free the buffer if it's a control packet and quit
@@ -368,8 +369,11 @@ int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   struct rr_ctx* rr = app_ctx->misc;
 
   struct waited_pkt* pkt = fdinfo->other;
-  pkt->on = 0;
   evt_core_rm_fd(ctx, fdinfo->fd);
+  if (ring_lt(pkt->id, rr->recv_id)) return 1;
+
+  printf("Timer reached for packet %d\n", pkt->id);
+  pkt->on = 0;
 
   //rr->remote_links &= 0xffff ^ 1 << pkt->link_num;
   rr->recv_id = pkt->id;

From aed1bea991a16a0cda0ef6cfc2c92399ea23b033 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@dufour.io>
Date: Tue, 19 Mar 2019 21:15:51 +0100
Subject: [PATCH 20/26] Fix a bug in identifier waited

---
 src/algo_rr.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index dbef6d8..2f7cc9a 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -136,7 +136,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
     if (timeout <= 0) timeout = 0;
     int idx_waited = (bp->ip.ap.str.id - 1) % PACKET_BUFFER_SIZE;
     rr->wait[idx_waited].on = 1;
-    rr->wait[idx_waited].id = bp->ip.ap.str.id;
+    rr->wait[idx_waited].id = bp->ip.ap.str.id - 1;
     rr->wait[idx_waited].link_num = bp->ip.ap.str.prevlink;
     rr->wait[idx_waited].timer_fd = set_timeout(ctx, timeout, &rr->wait[idx_waited]);
   }

From cd7f12aee09c8680e712b6feaf083831999fe4d8 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@dufour.io>
Date: Tue, 19 Mar 2019 21:21:00 +0100
Subject: [PATCH 21/26] Try to fix timers

---
 src/algo_rr.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 2f7cc9a..f2ef452 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -370,10 +370,10 @@ int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
 
   struct waited_pkt* pkt = fdinfo->other;
   evt_core_rm_fd(ctx, fdinfo->fd);
+  pkt->on = 0;
   if (ring_lt(pkt->id, rr->recv_id)) return 1;
 
   printf("Timer reached for packet %d\n", pkt->id);
-  pkt->on = 0;
 
   //rr->remote_links &= 0xffff ^ 1 << pkt->link_num;
   rr->recv_id = pkt->id;

From 1530209b2297c8fb86315b84ff1addd503a3d9fa Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Wed, 20 Mar 2019 08:51:40 +0100
Subject: [PATCH 22/26] Fix another bug of comparison

---
 src/algo_rr.c | 1 +
 src/utils.c   | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index f2ef452..058661a 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -130,6 +130,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   }
 
   // 3. If packet arrived too early, we register a timer
+  printf("%d < %d = %d\n", rr->recv_id, bp->ip.ap.str.id - 1, ring_lt(rr->recv_id, bp->ip.ap.str.id - 1));
   if (ring_lt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.str.deltat;
     //printf("%ld - %ld = %ld\n", rr->mjit, (int64_t) bp->ip.ap.str.deltat, timeout);
diff --git a/src/utils.c b/src/utils.c
index ad49cf9..287d820 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -74,9 +74,9 @@ int ring_gt(uint16_t v1, uint16_t v2) {
 }
 
 int ring_le(uint16_t v1, uint16_t v2) {
-  return ring_gt(v2, v1);
+  return ring_ge(v2, v1);
 }
 
 int ring_lt(uint16_t v1, uint16_t v2) {
-  return ring_ge(v2, v1);
+  return ring_gt(v2, v1);
 }

From cbf7e0883a6f0376f5fee83c458e91ed3d900225 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Wed, 20 Mar 2019 09:08:32 +0100
Subject: [PATCH 23/26] Fix timer logic

---
 src/algo_rr.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 058661a..1994790 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -377,9 +377,10 @@ int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   printf("Timer reached for packet %d\n", pkt->id);
 
   //rr->remote_links &= 0xffff ^ 1 << pkt->link_num;
-  rr->recv_id = pkt->id;
-
-  rr_pkt_unroll (ctx, app_ctx);
+  while (ring_lt(rr->recv_id, pkt->id)) {
+    rr->recv_id++;
+    rr_pkt_unroll (ctx, app_ctx);
+  }
 
   return 1;
 }

From e543c446742e4a90faad34cbb490535170e3bc9a Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Wed, 20 Mar 2019 10:28:52 +0100
Subject: [PATCH 24/26] Show buffer states

---
 src/algo_rr.c    | 10 ++++++----
 src/algo_utils.c | 29 +++++++++++++++++++++++++----
 src/algo_utils.h |  2 +-
 3 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/src/algo_rr.c b/src/algo_rr.c
index 1994790..ae6dc8e 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -113,7 +113,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
 
-  printf("Selected url %s for pkt %d to be queued for delivery\n", fdinfo->url, bp->ip.ap.str.id);
+  //printf("Selected url %s for pkt %d to be queued for delivery\n", fdinfo->url, bp->ip.ap.str.id);
 
   // 1. Update links I can use thanks to target feedback
   if (bp->ip.ap.str.id > rr->my_links_ver) {
@@ -130,7 +130,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   }
 
   // 3. If packet arrived too early, we register a timer
-  printf("%d < %d = %d\n", rr->recv_id, bp->ip.ap.str.id - 1, ring_lt(rr->recv_id, bp->ip.ap.str.id - 1));
+  //printf("%d < %d = %d\n", rr->recv_id, bp->ip.ap.str.id - 1, ring_lt(rr->recv_id, bp->ip.ap.str.id - 1));
   if (ring_lt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.str.deltat;
     //printf("%ld - %ld = %ld\n", rr->mjit, (int64_t) bp->ip.ap.str.deltat, timeout);
@@ -161,7 +161,7 @@ void rr_deliver(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct
   struct rr_ctx* rr = app_ctx->misc;
   char url[255];
 
-  printf("Selected url %s for pkt %d to be delivered\n", fdinfo->url, bp->ip.ap.str.id);
+  //printf("Selected url %s for pkt %d to be delivered\n", fdinfo->url, bp->ip.ap.str.id);
 
   // 0. We update our cursor
   rr->recv_id = bp->ip.ap.str.id;
@@ -313,7 +313,7 @@ int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
     //printf("-- Trying %s\n", url);
     to_fdinfo = evt_core_get_from_url (ctx, url);
     if (to_fdinfo != NULL) {
-      printf("Selected url %s for pkt %d to be sent on Tor\n", url, bp->ip.ap.str.id);
+      //printf("Selected url %s for pkt %d to be sent on Tor\n", url, bp->ip.ap.str.id);
       break;
     }
   }
@@ -376,7 +376,9 @@ int rr_on_timer(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
 
   printf("Timer reached for packet %d\n", pkt->id);
 
+  // !BLACKLIST LINK
   //rr->remote_links &= 0xffff ^ 1 << pkt->link_num;
+
   while (ring_lt(rr->recv_id, pkt->id)) {
     rr->recv_id++;
     rr_pkt_unroll (ctx, app_ctx);
diff --git a/src/algo_utils.c b/src/algo_utils.c
index 5e6542e..4f2b5d4 100644
--- a/src/algo_utils.c
+++ b/src/algo_utils.c
@@ -12,6 +12,15 @@ void free_naive(void* app_ctx) {
   free(ctx);
 }
 
+void iterate(int* fd, GQueue* q, int* waiting_count) {
+  fprintf(stderr, "Queue for fd=%d has length=%d\n", *fd, q->length);
+  waiting_count += q->length;
+}
+
+void iterate2(int* fd, struct buffer_packet *bp, gpointer user_data) {
+  fprintf(stderr, "fd=%d has a used_buffer entry\n", *fd);
+}
+
 /**
  * Returns a buffer if available, NULL otherwise
  */
@@ -25,6 +34,15 @@ struct buffer_packet* get_read_buffer(struct algo_ctx *app_ctx, struct evt_core_
   // 2. Get a new buffer otherwise
   bp = g_queue_pop_head(app_ctx->free_buffer);
   if (bp == NULL) {
+    fprintf(stderr, "No more free buffer for fd=%d.\n", fdinfo->fd);
+    int waiting_count = 0;
+    g_hash_table_foreach(app_ctx->write_waiting, (GHFunc)iterate, &waiting_count);
+    g_hash_table_foreach(app_ctx->used_buffer, (GHFunc)iterate2, NULL);
+    fprintf(stderr, "total_buffers=%d, used_buffers=%d. free_buffer=%d, waiting_buffer=%d.\n",
+            PACKET_BUFFER_SIZE,
+            g_hash_table_size(app_ctx->used_buffer),
+            app_ctx->free_buffer->length,
+            waiting_count);
     // 2.1 If no buffer is available, we subscribe to be notified later
     g_queue_push_tail (app_ctx->read_waiting, &(fdinfo->fd));
     return NULL;
@@ -86,10 +104,13 @@ void mv_buffer_wtor(struct algo_ctx* app_ctx, struct evt_core_fdinfo* fdinfo, st
 void notify_read(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx) {
   struct evt_core_fdinfo* next_fdinfo = NULL;
   while (next_fdinfo == NULL) {
-    int fd = GPOINTER_TO_INT(g_queue_pop_head(app_ctx->read_waiting));
-    if (fd == 0) break;
-    next_fdinfo = evt_core_get_from_fd (ctx, fd);
-    if (strcmp(next_fdinfo->cat->name, "tcp-read") == 0 || strcmp(next_fdinfo->cat->name, "udp-read") == 0) {
+    int* fd = g_queue_pop_head(app_ctx->read_waiting);
+    if (fd == NULL) break;
+    next_fdinfo = evt_core_get_from_fd (ctx, *fd);
+    if (next_fdinfo == NULL) {
+      fprintf(stderr, "Unable to find fdinfo for fd=%d\n", *fd);
+      exit(EXIT_FAILURE);
+    } else if (strcmp(next_fdinfo->cat->name, "tcp-read") == 0 || strcmp(next_fdinfo->cat->name, "udp-read") == 0) {
       next_fdinfo->cat->cb(ctx, next_fdinfo);
     } else {
       fprintf(stderr, "A fd from category %s can't be stored in read_waiting\n", next_fdinfo->cat->name);
diff --git a/src/algo_utils.h b/src/algo_utils.h
index 820dfef..f515190 100644
--- a/src/algo_utils.h
+++ b/src/algo_utils.h
@@ -3,7 +3,7 @@
 #include <glib-2.0/glib.h>
 #include <glib-2.0/gmodule.h>
 #include <glib-2.0/glib-object.h>
-#define PACKET_BUFFER_SIZE 100
+#define PACKET_BUFFER_SIZE 20
 
 typedef void (*algo_ctx_free_misc)(void*);
 

From 62169350841faff328ba3f778fe1c31e6d3ca36d Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Wed, 20 Mar 2019 15:13:16 +0100
Subject: [PATCH 25/26] Rewrite buffer management, not tested

---
 src/algo_naive.c | 13 ++++----
 src/algo_rr.c    | 48 ++++++++++++++++------------
 src/algo_utils.c | 83 +++++++++++++++++++++++++++++++++++++++++++-----
 src/algo_utils.h | 29 ++++++++++-------
 4 files changed, 127 insertions(+), 46 deletions(-)

diff --git a/src/algo_naive.c b/src/algo_naive.c
index cc8538a..2ae0af9 100644
--- a/src/algo_naive.c
+++ b/src/algo_naive.c
@@ -65,13 +65,13 @@ int on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   to_fdinfo = evt_core_get_from_url (ctx, url);
   if (to_fdinfo == NULL) {
     fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
-    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    mv_buffer_wtof (app_ctx, fdinfo);
     return 1;
   }
   //printf("Pass packet from %s to %s\n", fdinfo->url, url);
 
   // 4. We move the buffer and notify the target
-  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
+  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo);
   on_udp_write(ctx, to_fdinfo);
 
   return 0;
@@ -97,7 +97,7 @@ int on_tcp_write(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
 
   // 3. A whole packet has been written
   // Release the buffer and notify
-  mv_buffer_wtor(app_ctx, fdinfo, bp);
+  mv_buffer_wtof(app_ctx, fdinfo);
   notify_read(ctx, app_ctx);
 
   return 0;
@@ -127,13 +127,13 @@ int on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   to_fdinfo = evt_core_get_from_url (ctx, url);
   if (to_fdinfo == NULL) {
     fprintf(stderr, "No fd for URL %s in udp-read. Dropping packet :( \n", url);
-    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    mv_buffer_wtof (app_ctx, fdinfo);
     return 1;
   }
   //printf("Pass packet from %s to %s\n", fdinfo->url, url);
 
   // 4. We move the buffer and notify the target
-  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
+  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo);
   on_tcp_write(ctx, to_fdinfo);
 
   return 0;
@@ -158,7 +158,7 @@ int on_udp_write (struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
 
   // 3. A whole packet has been written
   // Release the buffer and notify
-  mv_buffer_wtor(app_ctx, fdinfo, bp);
+  mv_buffer_wtof(app_ctx, fdinfo);
   notify_read(ctx, app_ctx);
 
   return 0;
@@ -199,6 +199,7 @@ void algo_naive(struct evt_core_ctx* evt, struct algo_skel* as) {
   memset(ctx, 0, sizeof(struct algo_ctx));
   ctx->free_buffer = g_queue_new ();
   ctx->read_waiting = g_queue_new ();
+  ctx->application_waiting = g_hash_table_new (NULL, NULL);
   ctx->used_buffer = g_hash_table_new(g_int_hash, g_int_equal);
   ctx->write_waiting = g_hash_table_new_full (g_int_hash, g_int_equal, NULL, naive_free_simple);
   for (int i = 0; i < sizeof(ctx->bps) / sizeof(ctx->bps[0]); i++) {
diff --git a/src/algo_rr.c b/src/algo_rr.c
index ae6dc8e..65fc13f 100644
--- a/src/algo_rr.c
+++ b/src/algo_rr.c
@@ -12,7 +12,8 @@ struct waited_pkt {
 
 struct deferred_pkt {
   int link_fd;
-  struct buffer_packet* bp;
+  int idx;
+  uint8_t on;
 };
 
 struct rr_ctx {
@@ -125,7 +126,7 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   if (ring_gt(rr->recv_id, bp->ip.ap.str.id - 1)) {
     // Packet has already been delivered or dropped, we free the buffer
     fprintf(stderr, "Packet %d arrived too late (current: %d)\n", bp->ip.ap.str.id, rr->recv_id);
-    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    mv_buffer_wtof (app_ctx, fdinfo);
     return;
   }
 
@@ -144,9 +145,10 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
 
   // 4. We queue the packet
   int idx_real = bp->ip.ap.str.id % PACKET_BUFFER_SIZE;
-  rr->real[idx_real].bp = bp;
+  rr->real[idx_real].on = 1;
+  rr->real[idx_real].idx = idx_real;
   rr->real[idx_real].link_fd = fdinfo->fd;
-  g_hash_table_remove(app_ctx->used_buffer, &fdinfo->fd); // We remove the packet from the reading buffer
+  mv_buffer_rtoa(app_ctx, fdinfo, &rr->real[idx_real].idx);
 
   // 5. We make sure that the remote link is set to up
   char buffer[16];
@@ -155,18 +157,23 @@ void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo,
   rr->remote_links |= 1 << link_num; // Make sure that the link is marked as working
 }
 
-void rr_deliver(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+void rr_deliver(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct deferred_pkt* dp) {
   struct evt_core_fdinfo *to_fdinfo = NULL;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct rr_ctx* rr = app_ctx->misc;
   char url[255];
 
+  // 1. Marked the packet as handled
+  dp->on = 0;
+
+  // 2. Get the buffer
+  struct buffer_packet* bp = get_app_buffer (app_ctx, &dp->idx);
   //printf("Selected url %s for pkt %d to be delivered\n", fdinfo->url, bp->ip.ap.str.id);
 
-  // 0. We update our cursor
+  // 3. We update our cursor
   rr->recv_id = bp->ip.ap.str.id;
 
-  // 1. We check that we don't have a running timeout
+  // 4. We check that we don't have a running timeout
   int idx_real = bp->ip.ap.str.id % PACKET_BUFFER_SIZE;
   if (rr->wait[idx_real].on) {
     rr->wait[idx_real].on = 0;
@@ -174,22 +181,24 @@ void rr_deliver(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct
     printf("Removed timer for packet %d\n",bp->ip.ap.str.id);
   }
 
-  // 2. We free the buffer if it's a control packet and quit
+  // 5. We free the buffer if it's a control packet and quit
   if (bp->ip.ap.str.flags & PKT_CONTROL) {
-    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    mv_buffer_atof (app_ctx, &dp->idx);
     return;
   }
 
-  // 3. A whole packet has been read, we will find its target
+  // 6. Find its target
   sprintf(url, "udp:write:127.0.0.1:%d", bp->ip.ap.str.port);
   to_fdinfo = evt_core_get_from_url (ctx, url);
   if (to_fdinfo == NULL) {
     fprintf(stderr, "No fd for URL %s in udp:write for tcp-read. Dropping packet :( \n", url);
-    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    //mv_buffer_wtor (app_ctx, fdinfo, bp);
+    mv_buffer_atof (app_ctx, &dp->idx);
   }
 
   // 4. We move the buffer and notify the target
-  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
+  //mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
+  mv_buffer_atow (app_ctx, &dp->idx, to_fdinfo);
   rr_on_udp_write(ctx, to_fdinfo);
 }
 
@@ -200,9 +209,7 @@ void rr_pkt_unroll(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx) {
 
   while(1) {
     struct deferred_pkt* def = &rr->real[(rr->recv_id+1) % PACKET_BUFFER_SIZE];
-    if (def->bp == NULL) break;
-    bp = def->bp;
-    def->bp = NULL;
+    if (!def->on) break;
     fdinfo = evt_core_get_from_fd (ctx, def->link_fd);
     if (fdinfo == NULL) {
       fprintf(stderr, "An error occured as the link seems to be closed for the requested fd\n");
@@ -210,7 +217,7 @@ void rr_pkt_unroll(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx) {
       continue;
     }
 
-    rr_deliver(ctx, fdinfo, bp);
+    rr_deliver(ctx, fdinfo, def);
   }
 }
 
@@ -257,7 +264,7 @@ int rr_on_udp_write (struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
 
   // 3. A whole packet has been written
   // Release the buffer and notify
-  mv_buffer_wtor(app_ctx, fdinfo, bp);
+  mv_buffer_wtof(app_ctx, fdinfo);
   notify_read(ctx, app_ctx);
 
   return 0;
@@ -323,13 +330,13 @@ int rr_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   // 4. A whole packet has been read, we will find someone to write it
   if (to_fdinfo == NULL) {
     fprintf(stderr, "No fd for URL %s in udp-read. Dropping packet :( \n", fdinfo->url);
-    mv_buffer_wtor (app_ctx, fdinfo, bp);
+    mv_buffer_wtof (app_ctx, fdinfo);
     return 1;
   }
   //printf("Pass packet from %s to %s\n", fdinfo->url, url);
 
   // 5. We move the buffer and notify the target
-  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
+  mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo);
   rr_on_tcp_write(ctx, to_fdinfo);
 
   return 0;
@@ -356,7 +363,7 @@ int rr_on_tcp_write(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
 
   // 3. A whole packet has been written
   // Release the buffer and notify
-  mv_buffer_wtor(app_ctx, fdinfo, bp);
+  mv_buffer_wtof(app_ctx, fdinfo);
   notify_read(ctx, app_ctx);
 
   return 0;
@@ -419,6 +426,7 @@ void algo_rr(struct evt_core_ctx* evt, struct algo_skel* as) {
   memset(ctx, 0, sizeof(struct algo_ctx));
   ctx->free_buffer = g_queue_new ();
   ctx->read_waiting = g_queue_new ();
+  ctx->application_waiting = g_hash_table_new (NULL, NULL);
   ctx->used_buffer = g_hash_table_new(g_int_hash, g_int_equal);
   ctx->write_waiting = g_hash_table_new_full (g_int_hash, g_int_equal, NULL, naive_free_simple);
   struct rr_ctx* rr = malloc(sizeof(struct rr_ctx));
diff --git a/src/algo_utils.c b/src/algo_utils.c
index 4f2b5d4..a9e6c12 100644
--- a/src/algo_utils.c
+++ b/src/algo_utils.c
@@ -76,31 +76,98 @@ struct buffer_packet* get_write_buffer(struct algo_ctx *app_ctx, struct evt_core
   return bp;
 }
 
-void mv_buffer_rtow(struct algo_ctx* app_ctx,
-                    struct evt_core_fdinfo* from,
-                    struct evt_core_fdinfo* to,
-                    struct buffer_packet* bp) {
-
-  // 1. We get the target writing queue
+void mv_buffer_rtow(struct algo_ctx* app_ctx, struct evt_core_fdinfo* from, struct evt_core_fdinfo* to) {
   GQueue* q;
+  struct buffer_packet* bp;
+
+  // 1. We get the packet buffer
+  bp = g_hash_table_lookup (app_ctx->used_buffer, &from->fd);
+  if (bp == NULL) {
+    fprintf(stderr, "Unable to find a buffer for fd=%d url=%s", from->fd, from->url);
+    exit(EXIT_FAILURE);
+  }
+
+  // 2. We get the target writing queue
   q = g_hash_table_lookup(app_ctx->write_waiting, &(to->fd));
   if (q == NULL) {
     q = g_queue_new ();
     g_hash_table_insert(app_ctx->write_waiting, &(to->fd), q);
   }
 
-  // 2. We move the buffer to the target queue
+  // 3. We move the data
   g_hash_table_remove(app_ctx->used_buffer, &from->fd);
   g_queue_push_tail(q, bp);
 }
 
-void mv_buffer_wtor(struct algo_ctx* app_ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+void mv_buffer_wtof(struct algo_ctx* app_ctx, struct evt_core_fdinfo* fdinfo) {
+  struct buffer_packet* bp = g_hash_table_lookup (app_ctx->used_buffer, &(fdinfo->fd));
+  if (bp == NULL) {
+    fprintf(stderr, "Unable to find a buffer for fd=%d url=%s", fdinfo->fd, fdinfo->url);
+    exit(EXIT_FAILURE);
+  }
   bp->mode = BP_READING;
   bp->aread = 0;
   g_queue_push_tail (app_ctx->free_buffer, bp);
   g_hash_table_remove(app_ctx->used_buffer, &(fdinfo->fd));
 }
 
+void mv_buffer_rtoa(struct algo_ctx* app_ctx, struct evt_core_fdinfo* from, void* to) {
+  struct buffer_packet* bp;
+  bp = g_hash_table_lookup (app_ctx->used_buffer, &from->fd);
+  if (bp == NULL) {
+    fprintf(stderr, "Unable to find a buffer for fd=%d url=%s\n", from->fd, from->url);
+    exit(EXIT_FAILURE);
+  }
+  g_hash_table_remove(app_ctx->used_buffer, &from->fd);
+  if (g_hash_table_contains(app_ctx->application_waiting, to)) {
+    fprintf(stderr, "Data already exist for this entry\n");
+    exit(EXIT_FAILURE);
+  }
+  g_hash_table_insert(app_ctx->application_waiting, to, bp);
+}
+
+void mv_buffer_atow(struct algo_ctx* app_ctx, void* from, struct evt_core_fdinfo* to) {
+  GQueue* q;
+  struct buffer_packet* bp;
+
+  // 1. We get the buffer
+  bp = g_hash_table_lookup (app_ctx->application_waiting, from);
+  if (bp == NULL) {
+    fprintf(stderr, "Unable to find this application buffer\n");
+    exit(EXIT_FAILURE);
+  }
+
+  // 2. We get the target writing queue
+  q = g_hash_table_lookup(app_ctx->write_waiting, &(to->fd));
+  if (q == NULL) {
+    q = g_queue_new ();
+    g_hash_table_insert(app_ctx->write_waiting, &(to->fd), q);
+  }
+
+  // 3. We move the buffer
+  g_hash_table_remove (app_ctx->application_waiting, from);
+  g_queue_push_tail(q, bp);
+}
+
+void mv_buffer_atof(struct algo_ctx* app_ctx, void* from) {
+  struct buffer_packet* bp;
+
+  // 1. We get the buffer
+  bp = g_hash_table_lookup (app_ctx->application_waiting, from);
+  if (bp == NULL) {
+    fprintf(stderr, "Unable to find this application buffer\n");
+    exit(EXIT_FAILURE);
+  }
+
+  // 2. We move it
+  g_hash_table_remove (app_ctx->application_waiting, from);
+  g_queue_push_tail (app_ctx->free_buffer, bp);
+}
+
+struct buffer_packet* get_app_buffer(struct algo_ctx *app_ctx, void* idx) {
+  return g_hash_table_lookup (app_ctx->application_waiting, idx);
+}
+
 void notify_read(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx) {
   struct evt_core_fdinfo* next_fdinfo = NULL;
   while (next_fdinfo == NULL) {
diff --git a/src/algo_utils.h b/src/algo_utils.h
index f515190..815e2b8 100644
--- a/src/algo_utils.h
+++ b/src/algo_utils.h
@@ -10,22 +10,27 @@ typedef void (*algo_ctx_free_misc)(void*);
 struct algo_ctx {
   int ref_count;
   struct buffer_packet bps[PACKET_BUFFER_SIZE];
-  GQueue* free_buffer;          // Available buffers
-  GHashTable* used_buffer;      // Buffers used for reading or writing
-  GQueue* read_waiting;         // Who wait to be notified for a read
-  GHashTable* write_waiting;    // Structure to track packets waiting to be written
-  void* misc;                   // Additional structures
-  algo_ctx_free_misc free_misc; // Fx ptr to free misc
+  GQueue* free_buffer;              // Available buffers
+  GHashTable* used_buffer;          // Buffers used for reading or writing
+  GQueue* read_waiting;             // Who wait to be notified for a read
+  GHashTable* application_waiting;  // Structure that can be used by the algo for its internal logic
+  GHashTable* write_waiting;        // Structure to track packets waiting to be written
+  void* misc;                       // Additional structures
+  algo_ctx_free_misc free_misc;     // Fx ptr to free misc
 };
 
-void mv_buffer_rtow(struct algo_ctx* app_ctx,
-                    struct evt_core_fdinfo* from,
-                    struct evt_core_fdinfo* to,
-                    struct buffer_packet* bp);
-void mv_buffer_wtor(struct algo_ctx* app_ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp);
+void mv_buffer_rtow(struct algo_ctx* app_ctx, struct evt_core_fdinfo* from, struct evt_core_fdinfo* to);
+void mv_buffer_wtof(struct algo_ctx* app_ctx, struct evt_core_fdinfo* from);
+void mv_buffer_rtoa(struct algo_ctx* app_ctx, struct evt_core_fdinfo* from, void* to);
+void mv_buffer_atow(struct algo_ctx* app_ctx, void* from, struct evt_core_fdinfo* to);
+void mv_buffer_atof(struct algo_ctx* app_ctx, void* from);
+
 struct buffer_packet* get_write_buffer(struct algo_ctx *app_ctx, struct evt_core_fdinfo *fdinfo);
 struct buffer_packet* get_read_buffer(struct algo_ctx *app_ctx, struct evt_core_fdinfo *fdinfo);
+struct buffer_packet* get_app_buffer(struct algo_ctx *app_ctx, void* idx);
+
+void notify_read(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx);
+
 void free_naive(void* app_ctx);
 void free_nothing(void* app_ctx);
-void notify_read(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx);
 void naive_free_simple(void* v);

From 429e66cbc80a651ae457de28ff6dffba0dd4e442 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin.dufour@inria.fr>
Date: Wed, 20 Mar 2019 17:02:23 +0100
Subject: [PATCH 26/26] Don't crash on socks5

---
 src/algo_utils.c | 5 +++--
 src/socks5.c     | 5 +----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/algo_utils.c b/src/algo_utils.c
index a9e6c12..f8b7a04 100644
--- a/src/algo_utils.c
+++ b/src/algo_utils.c
@@ -38,10 +38,11 @@ struct buffer_packet* get_read_buffer(struct algo_ctx *app_ctx, struct evt_core_
     int waiting_count = 0;
     g_hash_table_foreach(app_ctx->write_waiting, (GHFunc)iterate, &waiting_count);
     g_hash_table_foreach(app_ctx->used_buffer, (GHFunc)iterate2, NULL);
-    fprintf(stderr, "total_buffers=%d, used_buffers=%d. free_buffer=%d, waiting_buffer=%d.\n",
+    fprintf(stderr, "total_buffers=%d, free_buffer=%d, used_buffers=%d, app_buffer=%d, write_buffer=%d.\n",
             PACKET_BUFFER_SIZE,
-            g_hash_table_size(app_ctx->used_buffer),
             app_ctx->free_buffer->length,
+            g_hash_table_size(app_ctx->used_buffer),
+            g_hash_table_size(app_ctx->application_waiting),
             waiting_count);
     // 2.1 If no buffer is available, we subscribe to be notified later
     g_queue_push_tail (app_ctx->read_waiting, &(fdinfo->fd));
diff --git a/src/socks5.c b/src/socks5.c
index a519965..1351cc4 100644
--- a/src/socks5.c
+++ b/src/socks5.c
@@ -42,10 +42,7 @@ int socks5_reply(int sock) {
     int res;
     struct server_reply sr = {0};
     res = read_entity(sock, &sr, sizeof(uint8_t) * 4);
-    if (res == -1) {
-      perror("read_entity");
-      exit(EXIT_FAILURE);
-    }
+    if (res == -1) goto read_error;
 
     switch(sr.atyp) {
         case ATYP_IPV4: