diff --git a/collection.c b/collection.c
index 8255631..928fce4 100644
--- a/collection.c
+++ b/collection.c
@@ -29,6 +29,12 @@
 struct cnx_collection {
     int num_cnx;  /* Number of connections in *cnx */
     struct connection *cnx; /* pointer to array of connections */
+
+    int num_fd; /* Number of file descriptors */
+    int* fd2cnx; /* Array indexed by file descriptor to index in cnx[] */
+    /* We don't try to keep the size of cnx and fd2cnx in sync at all,
+     * so that the implementation is independant of other uses for file
+     * descriptors, e.g. if sslh get integrated in another process */
 };
 
 /* cnx_num_alloc is the number of connection to allocate at once (at start-up,
@@ -40,6 +46,8 @@ struct cnx_collection {
  */
 static long cnx_num_alloc;
 
+static long fd_num_alloc; /* same, but for the file descriptor array */
+
 
 
 /* Allocates and initialises a new collection of connections. */
@@ -61,6 +69,17 @@ cnx_collection* collection_init(void)
     for (i = 0; i < collection->num_cnx; i++) {
         init_cnx(&collection->cnx[i]);
     }
+
+    fd_num_alloc = getpagesize() / sizeof(collection->fd2cnx[0]);
+
+    collection->num_fd = fd_num_alloc;
+    collection->fd2cnx = malloc(collection->num_fd * sizeof(collection->fd2cnx[0])); 
+    CHECK_ALLOC(collection->fd2cnx, "malloc(collection->fd2cnx)");
+
+    for (i = 0; i < collection->num_fd; i++) {
+        collection->fd2cnx[i] = -1;
+    }
+
     return collection;
 }
 
@@ -72,7 +91,7 @@ void collection_destroy(cnx_collection* collection)
 /* Increases the number of slots available in a collection of connections
  * After calling, collection->cnx might have moved
  * */
-static int collection_extend(struct cnx_collection* collection)
+static int extend_cnx(struct cnx_collection* collection)
 {
     struct connection* new;
     int i, new_length = collection->num_cnx + cnx_num_alloc;
@@ -91,6 +110,28 @@ static int collection_extend(struct cnx_collection* collection)
     return 0;
 }
 
+static int extend_fd2cnx(cnx_collection* collection)
+{
+    int* new_i;
+    int i, new_length;
+
+    new_length = collection->num_fd + fd_num_alloc;
+
+    new_i = realloc(collection->fd2cnx, new_length * sizeof(*new_i));
+    if (!new_i) {
+        return -1;
+    }
+
+    collection->fd2cnx = new_i;
+
+    for (i = collection->num_fd; i < new_length; i++) {
+        collection->fd2cnx[i] = -1;
+    }
+    collection->num_fd = new_length;
+
+    return 0;
+}
+
 
 int collection_add_fd(struct cnx_collection* collection, int fd)
 {
@@ -102,7 +143,7 @@ int collection_add_fd(struct cnx_collection* collection, int fd)
         /* nothing */
     }
     if (free >= collection->num_cnx)  {
-        res = collection_extend(collection);
+        res = extend_cnx(collection);
         if (res) {
             log_message(LOG_ERR, "unable to extend collection -- dropping connection\n");
             return -1;
@@ -112,11 +153,28 @@ int collection_add_fd(struct cnx_collection* collection, int fd)
     collection->cnx[free].state = ST_PROBING;
     collection->cnx[free].probe_timeout = time(NULL) + cfg.timeout;
 
+    if (fd > collection->num_fd) {
+        res = extend_fd2cnx(collection);
+        if (res) {
+            log_message(LOG_ERR, "unable to extend fd2cnx -- dropping connection\n");
+            return -1;
+        }
+    }
+    collection->fd2cnx[fd] = free;
+
     if (cfg.verbose) 
         fprintf(stderr, "accepted fd %d on slot %d\n", fd, free);
     return 0;
 }
 
+/* Remove a connection from the collection */
+int collection_remove_cnx(cnx_collection* collection, struct connection *cnx)
+{
+    collection->fd2cnx[cnx->q[0].fd] = -1;
+    collection->fd2cnx[cnx->q[1].fd] = -1;
+    init_cnx(cnx);
+    return 0;
+}
 
 /* Returns the indexed connection in the collection */
 struct connection* collection_get_cnx(struct cnx_collection* collection, int index)
diff --git a/sslh-select.c b/sslh-select.c
index aa1f86b..630cca8 100644
--- a/sslh-select.c
+++ b/sslh-select.c
@@ -73,7 +73,7 @@ static int tidy_connection(struct connection *cnx, struct select_info* fd_info)
                 free(cnx->q[i].deferred_data);
         }
     }
-    init_cnx(cnx);
+    collection_remove_cnx(fd_info->collection, cnx);
     return 0;
 }