Make sdcard daemon multi-threaded.

The essential idea here is that a handler thread only needs to
hold a lock on the global node table while it is manipulating
nodes.  The actual I/O operation is then performed without
holding any locks.

By default, we use 2 threads but this can be configured on the
command-line.  Work is sheduled somewhat arbitrarily by the
handler threads.  Whichever thread happens to read() the next
request first wins the right process it.  This policy is very
simple but potentially wastes threads when there isn't much
work to be done.  We can always improve this later if needed.

Change-Id: Id27a27c2c9b40d4f8e35a6bef9dd84f0dfacf337
diff --git a/sdcard/Android.mk b/sdcard/Android.mk
index c430ac8..fb04d6d 100644
--- a/sdcard/Android.mk
+++ b/sdcard/Android.mk
@@ -4,6 +4,7 @@
 
 LOCAL_SRC_FILES:= sdcard.c
 LOCAL_MODULE:= sdcard
+LOCAL_CFLAGS := -Wall -Wno-unused-parameter
 
 LOCAL_SHARED_LIBRARIES := libc
 
diff --git a/sdcard/sdcard.c b/sdcard/sdcard.c
index 7177395..316588c 100644
--- a/sdcard/sdcard.c
+++ b/sdcard/sdcard.c
@@ -27,6 +27,7 @@
 #include <dirent.h>
 #include <limits.h>
 #include <ctype.h>
+#include <pthread.h>
 
 #include <private/android_filesystem_config.h>
 
@@ -84,28 +85,31 @@
  * the largest possible data payload. */
 #define MAX_REQUEST_SIZE (sizeof(struct fuse_in_header) + sizeof(struct fuse_write_in) + MAX_WRITE)
 
+/* Default number of threads. */
+#define DEFAULT_NUM_THREADS 2
+
+/* Pseudo-error constant used to indicate that no fuse status is needed
+ * or that a reply has already been written. */
+#define NO_STATUS 1
+
 struct handle {
-    struct node *node;
     int fd;
 };
 
 struct dirhandle {
-    struct node *node;
     DIR *d;
 };
 
 struct node {
+    __u32 refcount;
     __u64 nid;
     __u64 gen;
 
     struct node *next;          /* per-dir sibling list */
     struct node *child;         /* first contained file by this dir */
-    struct node *all;           /* global node list */
     struct node *parent;        /* containing directory */
 
-    __u32 refcount;
-    __u32 namelen;
-
+    size_t namelen;
     char *name;
     /* If non-null, this is the real name of the file in the underlying storage.
      * This may differ from the field "name" only by case.
@@ -117,19 +121,19 @@
 
 /* Global data structure shared by all fuse handlers. */
 struct fuse {
+    pthread_mutex_t lock;
+
     __u64 next_generation;
-    __u64 next_node_id;
-
     int fd;
-
-    struct node *all;
-
     struct node root;
     char rootpath[PATH_MAX];
 };
 
 /* Private data used by a single fuse handler. */
 struct fuse_handler {
+    struct fuse* fuse;
+    int token;
+
     /* To save memory, we never use the contents of the request buffer and the read
      * buffer at the same time.  This allows us to share the underlying storage. */
     union {
@@ -138,81 +142,143 @@
     };
 };
 
-#define NO_CASE_SENSITIVE_MATCH 0
-#define CASE_SENSITIVE_MATCH 1
-
-/*
- * Get the real-life absolute path to a node.
- *   node: start at this node
- *   buf: storage for returned string
- *   name: append this string to path if set
- */
-char *do_node_get_path(struct node *node, char *buf, const char *name, int match_case_insensitive)
+static inline void *id_to_ptr(__u64 nid)
 {
-    struct node *in_node = node;
-    const char *in_name = name;
-    char *out = buf + PATH_MAX - 1;
-    int len;
-    out[0] = 0;
+    return (void *) (uintptr_t) nid;
+}
 
-    if (name) {
-        len = strlen(name);
-        goto start;
-    }
+static inline __u64 ptr_to_id(void *ptr)
+{
+    return (__u64) (uintptr_t) ptr;
+}
 
-    while (node) {
-        name = (node->actual_name ? node->actual_name : node->name);
-        len = node->namelen;
-        node = node->parent;
-    start:
-        if ((len + 1) > (out - buf))
-            return 0;
-        out -= len;
-        memcpy(out, name, len);
-        /* avoid double slash at beginning of path */
-        if (out[0] != '/') {
-            out --;
-            out[0] = '/';
+static void acquire_node_locked(struct node* node)
+{
+    node->refcount++;
+    TRACE("ACQUIRE %p (%s) rc=%d\n", node, node->name, node->refcount);
+}
+
+static void remove_node_from_parent_locked(struct node* node);
+
+static void release_node_locked(struct node* node)
+{
+    TRACE("RELEASE %p (%s) rc=%d\n", node, node->name, node->refcount);
+    if (node->refcount > 0) {
+        node->refcount--;
+        if (!node->refcount) {
+            TRACE("DESTROY %p (%s)\n", node, node->name);
+            remove_node_from_parent_locked(node);
+
+                /* TODO: remove debugging - poison memory */
+            memset(node->name, 0xef, node->namelen);
+            free(node->name);
+            free(node->actual_name);
+            memset(node, 0xfc, sizeof(*node));
+            free(node);
         }
+    } else {
+        ERROR("Zero refcnt %p\n", node);
+    }
+}
+
+static void add_node_to_parent_locked(struct node *node, struct node *parent) {
+    node->parent = parent;
+    node->next = parent->child;
+    parent->child = node;
+    acquire_node_locked(parent);
+}
+
+static void remove_node_from_parent_locked(struct node* node)
+{
+    if (node->parent) {
+        if (node->parent->child == node) {
+            node->parent->child = node->parent->child->next;
+        } else {
+            struct node *node2;
+            node2 = node->parent->child;
+            while (node2->next != node)
+                node2 = node2->next;
+            node2->next = node->next;
+        }
+        release_node_locked(node->parent);
+        node->parent = NULL;
+        node->next = NULL;
+    }
+}
+
+/* Gets the absolute path to a node into the provided buffer.
+ *
+ * Populates 'buf' with the path and returns the length of the path on success,
+ * or returns -1 if the path is too long for the provided buffer.
+ */
+static ssize_t get_node_path_locked(struct node* node, char* buf, size_t bufsize)
+{
+    size_t namelen = node->namelen;
+    if (bufsize < namelen + 1) {
+        return -1;
     }
 
-    /* If we are searching for a file within node (rather than computing node's path)
-     * and fail, then we need to look for a case insensitive match.
-     */
-    if (in_name && match_case_insensitive && access(out, F_OK) != 0) {
-        char *path, buffer[PATH_MAX];
-        DIR* dir;
+    ssize_t pathlen = 0;
+    if (node->parent) {
+        pathlen = get_node_path_locked(node->parent, buf, bufsize - namelen - 2);
+        if (pathlen < 0) {
+            return -1;
+        }
+        buf[pathlen++] = '/';
+    }
+
+    const char* name = node->actual_name ? node->actual_name : node->name;
+    memcpy(buf + pathlen, name, namelen + 1); /* include trailing \0 */
+    return pathlen + namelen;
+}
+
+/* Finds the absolute path of a file within a given directory.
+ * Performs a case-insensitive search for the file and sets the buffer to the path
+ * of the first matching file.  If 'search' is zero or if no match is found, sets
+ * the buffer to the path that the file would have, assuming the name were case-sensitive.
+ *
+ * Populates 'buf' with the path and returns the actual name (within 'buf') on success,
+ * or returns NULL if the path is too long for the provided buffer.
+ */
+static char* find_file_within(const char* path, const char* name,
+        char* buf, size_t bufsize, int search)
+{
+    size_t pathlen = strlen(path);
+    size_t namelen = strlen(name);
+    size_t childlen = pathlen + namelen + 1;
+    char* actual;
+
+    if (bufsize <= childlen) {
+        return NULL;
+    }
+
+    memcpy(buf, path, pathlen);
+    buf[pathlen] = '/';
+    actual = buf + pathlen + 1;
+    memcpy(actual, name, namelen + 1);
+
+    if (search && access(buf, F_OK)) {
         struct dirent* entry;
-        path = do_node_get_path(in_node, buffer, NULL, NO_CASE_SENSITIVE_MATCH);
-        dir = opendir(path);
+        DIR* dir = opendir(path);
         if (!dir) {
             ERROR("opendir %s failed: %s", path, strerror(errno));
-            return out;
+            return actual;
         }
-
         while ((entry = readdir(dir))) {
-            if (!strcasecmp(entry->d_name, in_name)) {
-                /* we have a match - replace the name */
-                len = strlen(in_name);
-                memcpy(buf + PATH_MAX - len - 1, entry->d_name, len);
+            if (!strcasecmp(entry->d_name, name)) {
+                /* we have a match - replace the name, don't need to copy the null again */
+                memcpy(actual, entry->d_name, namelen);
                 break;
             }
         }
         closedir(dir);
     }
-
-   return out;
+    return actual;
 }
 
-char *node_get_path(struct node *node, char *buf, const char *name)
+static void attr_from_stat(struct fuse_attr *attr, const struct stat *s, __u64 nid)
 {
-    /* We look for case insensitive matches by default */
-    return do_node_get_path(node, buf, name, CASE_SENSITIVE_MATCH);
-}
-
-void attr_from_stat(struct fuse_attr *attr, struct stat *s)
-{
-    attr->ino = s->st_ino;
+    attr->ino = nid;
     attr->size = s->st_size;
     attr->blocks = s->st_blocks;
     attr->atime = s->st_atime;
@@ -239,142 +305,106 @@
     attr->gid = AID_SDCARD_RW;
 }
 
-int node_get_attr(struct node *node, struct fuse_attr *attr)
-{
-    int res;
-    struct stat s;
-    char *path, buffer[PATH_MAX];
-
-    path = node_get_path(node, buffer, 0);
-    res = lstat(path, &s);
-    if (res < 0) {
-        ERROR("lstat('%s') errno %d\n", path, errno);
-        return -1;
-    }
-
-    attr_from_stat(attr, &s);    
-    attr->ino = node->nid;
-
-    return 0;
-}
-
-static void add_node_to_parent(struct node *node, struct node *parent) {
-    node->parent = parent;
-    node->next = parent->child;
-    parent->child = node;
-    parent->refcount++;
-}
-
-/* Check to see if our parent directory already has a file with a name
- * that differs only by case.  If we find one, store it in the actual_name
- * field so node_get_path will map it to this file in the underlying storage.
- */
-static void node_find_actual_name(struct node *node)
-{
-    char *path, buffer[PATH_MAX];
-    const char *node_name = node->name;
-    DIR* dir;
-    struct dirent* entry;
-
-    if (!node->parent) return;
-
-    path = node_get_path(node->parent, buffer, 0);
-    dir = opendir(path);
-    if (!dir) {
-        ERROR("opendir %s failed: %s", path, strerror(errno));
-        return;
-    }
-
-    while ((entry = readdir(dir))) {
-        const char *test_name = entry->d_name;
-        if (strcmp(test_name, node_name) && !strcasecmp(test_name, node_name)) {
-            /* we have a match - differs but only by case */
-            node->actual_name = strdup(test_name);
-            if (!node->actual_name) {
-                ERROR("strdup failed - out of memory\n");
-                exit(1);
-            }
-            break;
-        }
-    }
-    closedir(dir);
-}
-
-struct node *node_create(struct node *parent, const char *name, __u64 nid, __u64 gen)
+struct node *create_node_locked(struct fuse* fuse,
+        struct node *parent, const char *name, const char* actual_name)
 {
     struct node *node;
-    int namelen = strlen(name);
+    size_t namelen = strlen(name);
 
     node = calloc(1, sizeof(struct node));
-    if (node == 0) {
-        return 0;
+    if (!node) {
+        return NULL;
     }
     node->name = malloc(namelen + 1);
-    if (node->name == 0) {
+    if (!node->name) {
         free(node);
-        return 0;
+        return NULL;
     }
-
-    node->nid = nid;
-    node->gen = gen;
-    add_node_to_parent(node, parent);
     memcpy(node->name, name, namelen + 1);
+    if (strcmp(name, actual_name)) {
+        node->actual_name = malloc(namelen + 1);
+        if (!node->actual_name) {
+            free(node->name);
+            free(node);
+            return NULL;
+        }
+        memcpy(node->actual_name, actual_name, namelen + 1);
+    }
     node->namelen = namelen;
-    node_find_actual_name(node);
+    node->nid = ptr_to_id(node);
+    node->gen = fuse->next_generation++;
+    acquire_node_locked(node);
+    add_node_to_parent_locked(node, parent);
     return node;
 }
 
-static char *rename_node(struct node *node, const char *name)
+static int rename_node_locked(struct node *node, const char *name,
+        const char* actual_name)
 {
-    node->namelen = strlen(name);
-    char *newname = realloc(node->name, node->namelen + 1);
-    if (newname == 0)
-        return 0;
-    node->name = newname;
-    memcpy(node->name, name, node->namelen + 1);
-    node_find_actual_name(node);
-    return node->name;
+    size_t namelen = strlen(name);
+    int need_actual_name = strcmp(name, actual_name);
+
+    /* make the storage bigger without actually changing the name
+     * in case an error occurs part way */
+    if (namelen > node->namelen) {
+        char* new_name = realloc(node->name, namelen + 1);
+        if (!new_name) {
+            return -ENOMEM;
+        }
+        node->name = new_name;
+        if (need_actual_name && node->actual_name) {
+            char* new_actual_name = realloc(node->actual_name, namelen + 1);
+            if (!new_actual_name) {
+                return -ENOMEM;
+            }
+            node->actual_name = new_actual_name;
+        }
+    }
+
+    /* update the name, taking care to allocate storage before overwriting the old name */
+    if (need_actual_name) {
+        if (!node->actual_name) {
+            node->actual_name = malloc(namelen + 1);
+            if (!node->actual_name) {
+                return -ENOMEM;
+            }
+        }
+        memcpy(node->actual_name, actual_name, namelen + 1);
+    } else {
+        free(node->actual_name);
+        node->actual_name = NULL;
+    }
+    memcpy(node->name, name, namelen + 1);
+    node->namelen = namelen;
+    return 0;
 }
 
-void fuse_init(struct fuse *fuse, int fd, const char *path)
+static struct node *lookup_node_by_id_locked(struct fuse *fuse, __u64 nid)
 {
-    fuse->fd = fd;
-    fuse->next_node_id = 2;
-    fuse->next_generation = 0;
-
-    fuse->all = &fuse->root;
-
-    memset(&fuse->root, 0, sizeof(fuse->root));
-    fuse->root.nid = FUSE_ROOT_ID; /* 1 */
-    fuse->root.refcount = 2;
-    rename_node(&fuse->root, path);
-}
-
-static inline void *id_to_ptr(__u64 nid)
-{
-    return (void *) (uintptr_t) nid;
-}
-
-static inline __u64 ptr_to_id(void *ptr)
-{
-    return (__u64) (uintptr_t) ptr;
-}
-
-
-struct node *lookup_by_inode(struct fuse *fuse, __u64 nid)
-{
-    if (!nid) {
-        return NULL;
-    } else if (nid == FUSE_ROOT_ID) {
+    if (nid == FUSE_ROOT_ID) {
         return &fuse->root;
     } else {
         return id_to_ptr(nid);
     }
 }
 
-struct node *lookup_child_by_name(struct node *node, const char *name)
+static struct node* lookup_node_and_path_by_id_locked(struct fuse* fuse, __u64 nid,
+        char* buf, size_t bufsize)
+{
+    struct node* node = lookup_node_by_id_locked(fuse, nid);
+    if (node && get_node_path_locked(node, buf, bufsize) < 0) {
+        node = NULL;
+    }
+    return node;
+}
+
+static struct node *lookup_child_by_name_locked(struct node *node, const char *name)
 {
     for (node = node->child; node; node = node->next) {
+        /* use exact string comparison, nodes that differ by case
+         * must be considered distinct even if they refer to the same
+         * underlying file as otherwise operations such as "mv x x"
+         * will not work because the source and target nodes are the same. */
         if (!strcmp(name, node->name)) {
             return node;
         }
@@ -382,123 +412,43 @@
     return 0;
 }
 
-struct node *lookup_child_by_inode(struct node *node, __u64 nid)
+static struct node* acquire_or_create_child_locked(
+        struct fuse* fuse, struct node* parent,
+        const char* name, const char* actual_name)
 {
-    for (node = node->child; node; node = node->next) {
-        if (node->nid == nid) {
-            return node;
-        }
-    }
-    return 0;
-}
-
-static void dec_refcount(struct node *node) {
-    if (node->refcount > 0) {
-        node->refcount--;
-        TRACE("dec_refcount %p(%s) -> %d\n", node, node->name, node->refcount);
+    struct node* child = lookup_child_by_name_locked(parent, name);
+    if (child) {
+        acquire_node_locked(child);
     } else {
-        ERROR("Zero refcnt %p\n", node);
+        child = create_node_locked(fuse, parent, name, actual_name);
     }
- }
-
-static struct node *remove_child(struct node *parent, __u64 nid)
-{
-    struct node *prev = 0;
-    struct node *node;
-
-    for (node = parent->child; node; node = node->next) {
-        if (node->nid == nid) {
-            if (prev) {
-                prev->next = node->next;
-            } else {
-                parent->child = node->next;
-            }
-            node->next = 0;
-            node->parent = 0;
-            dec_refcount(parent);
-            return node;
-        }
-        prev = node;
-    }
-    return 0;
+    return child;
 }
 
-struct node *node_lookup(struct fuse *fuse, struct node *parent, const char *name,
-                         struct fuse_attr *attr)
+static void fuse_init(struct fuse *fuse, int fd, const char *path)
 {
-    int res;
-    struct stat s;
-    char *path, buffer[PATH_MAX];
-    struct node *node;
+    pthread_mutex_init(&fuse->lock, NULL);
 
-    path = node_get_path(parent, buffer, name);
-        /* XXX error? */
+    fuse->fd = fd;
+    fuse->next_generation = 0;
 
-    res = lstat(path, &s);
-    if (res < 0)
-        return 0;
-    
-    node = lookup_child_by_name(parent, name);
-    if (!node) {
-        node = node_create(parent, name, fuse->next_node_id++, fuse->next_generation++);
-        if (!node)
-            return 0;
-        node->nid = ptr_to_id(node);
-        node->all = fuse->all;
-        fuse->all = node;
-    }
-
-    attr_from_stat(attr, &s);
-    attr->ino = node->nid;
-
-    return node;
+    memset(&fuse->root, 0, sizeof(fuse->root));
+    fuse->root.nid = FUSE_ROOT_ID; /* 1 */
+    fuse->root.refcount = 2;
+    fuse->root.namelen = strlen(path);
+    fuse->root.name = strdup(path);
 }
 
-void node_release(struct node *node)
-{
-    TRACE("RELEASE %p (%s) rc=%d\n", node, node->name, node->refcount);
-    dec_refcount(node);
-    if (node->refcount == 0) {
-        if (node->parent->child == node) {
-            node->parent->child = node->parent->child->next;
-        } else {
-            struct node *node2;
-
-            node2 = node->parent->child;
-            while (node2->next != node)
-                node2 = node2->next;
-            node2->next = node->next;            
-        }
-
-        TRACE("DESTROY %p (%s)\n", node, node->name);
-
-        node_release(node->parent);
-
-        node->parent = 0;
-        node->next = 0;
-
-            /* TODO: remove debugging - poison memory */
-        memset(node->name, 0xef, node->namelen);
-        free(node->name);
-        free(node->actual_name);
-        memset(node, 0xfc, sizeof(*node));
-        free(node);
-    }
-}
-
-void fuse_status(struct fuse *fuse, __u64 unique, int err)
+static void fuse_status(struct fuse *fuse, __u64 unique, int err)
 {
     struct fuse_out_header hdr;
     hdr.len = sizeof(hdr);
     hdr.error = err;
     hdr.unique = unique;
-    if (err) {
-//        ERROR("*** %d ***\n", err);
-    }
     write(fuse->fd, &hdr, sizeof(hdr));
 }
 
-void fuse_reply(struct fuse *fuse, __u64 unique, void *data, int len)
+static void fuse_reply(struct fuse *fuse, __u64 unique, void *data, int len)
 {
     struct fuse_out_header hdr;
     struct iovec vec[2];
@@ -519,106 +469,132 @@
     }
 }
 
-static void lookup_entry(struct fuse* fuse, struct fuse_handler* handler,
-        __u64 unique, struct node* parent_node, const char* name)
+static int fuse_reply_entry(struct fuse* fuse, __u64 unique,
+        struct node* parent, const char* name, const char* actual_name,
+        const char* path)
 {
-    struct fuse_entry_out out;
     struct node* node;
+    struct fuse_entry_out out;
+    struct stat s;
 
-    memset(&out, 0, sizeof(out));
-    node = node_lookup(fuse, parent_node, name, &out.attr);
-    if (!node) {
-        fuse_status(fuse, unique, -ENOENT);
-        return;
+    if (lstat(path, &s) < 0) {
+        return -errno;
     }
 
-    node->refcount++;
-    TRACE("ACQUIRE %p (%s) rc=%d\n", node, node->name, node->refcount);
+    pthread_mutex_lock(&fuse->lock);
+    node = acquire_or_create_child_locked(fuse, parent, name, actual_name);
+    if (!node) {
+        pthread_mutex_unlock(&fuse->lock);
+        return -ENOMEM;
+    }
+    memset(&out, 0, sizeof(out));
+    attr_from_stat(&out.attr, &s, node->nid);
+    out.attr_valid = 10;
+    out.entry_valid = 10;
     out.nodeid = node->nid;
     out.generation = node->gen;
-    out.entry_valid = 10;
-    out.attr_valid = 10;
-
+    pthread_mutex_unlock(&fuse->lock);
     fuse_reply(fuse, unique, &out, sizeof(out));
+    return NO_STATUS;
 }
 
-static void handle_lookup(struct fuse* fuse, struct fuse_handler* handler,
+static int fuse_reply_attr(struct fuse* fuse, __u64 unique, __u64 nid,
+        const char* path)
+{
+    struct fuse_attr_out out;
+    struct stat s;
+
+    if (lstat(path, &s) < 0) {
+        return -errno;
+    }
+    memset(&out, 0, sizeof(out));
+    attr_from_stat(&out.attr, &s, nid);
+    out.attr_valid = 10;
+    fuse_reply(fuse, unique, &out, sizeof(out));
+    return NO_STATUS;
+}
+
+static int handle_lookup(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header *hdr, const char* name)
 {
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
+    struct node* parent_node;
+    char parent_path[PATH_MAX];
+    char child_path[PATH_MAX];
+    const char* actual_name;
 
-    TRACE("LOOKUP %s @ %llx (%s)\n", name, hdr->nodeid, node ? node->name : "?");
-    if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+    pthread_mutex_lock(&fuse->lock);
+    parent_node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid,
+            parent_path, sizeof(parent_path));
+    TRACE("[%d] LOOKUP %s @ %llx (%s)\n", handler->token, name, hdr->nodeid,
+        parent_node ? parent_node->name : "?");
+    pthread_mutex_unlock(&fuse->lock);
+
+    if (!parent_node || !(actual_name = find_file_within(parent_path, name,
+            child_path, sizeof(child_path), 1))) {
+        return -ENOENT;
     }
-
-    lookup_entry(fuse, handler, hdr->unique, node, name);
+    return fuse_reply_entry(fuse, hdr->unique, parent_node, name, actual_name, child_path);
 }
 
-static void handle_forget(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_forget(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header *hdr, const struct fuse_forget_in *req)
 {
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
-    __u64 n = req->nlookup;
+    struct node* node;
 
-    TRACE("FORGET #%lld @ %llx (%s)\n", n, hdr->nodeid, node ? node->name : "?");
-    if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+    pthread_mutex_lock(&fuse->lock);
+    node = lookup_node_by_id_locked(fuse, hdr->nodeid);
+    TRACE("[%d] FORGET #%lld @ %llx (%s)\n", handler->token, req->nlookup,
+            hdr->nodeid, node ? node->name : "?");
+    if (node) {
+        __u64 n = req->nlookup;
+        while (n--) {
+            release_node_locked(node);
+        }
     }
-
-    while (n--) {
-        node_release(node);
-    }
-    /* no reply */
+    pthread_mutex_unlock(&fuse->lock);
+    return NO_STATUS; /* no reply */
 }
 
-static void handle_getaddr(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_getattr(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header *hdr, const struct fuse_getattr_in *req)
 {
-    struct fuse_attr_out out;
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
+    struct node* node;
+    char path[PATH_MAX];
 
-    TRACE("GETATTR flags=%x fh=%llx @ %llx (%s)\n", req->getattr_flags, req->fh,
-            hdr->nodeid, node ? node->name : "?");
+    pthread_mutex_lock(&fuse->lock);
+    node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid, path, sizeof(path));
+    TRACE("[%d] GETATTR flags=%x fh=%llx @ %llx (%s)\n", handler->token,
+            req->getattr_flags, req->fh, hdr->nodeid, node ? node->name : "?");
+    pthread_mutex_unlock(&fuse->lock);
+
     if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+        return -ENOENT;
     }
-
-    memset(&out, 0, sizeof(out));
-    node_get_attr(node, &out.attr);
-    out.attr_valid = 10;
-
-    fuse_reply(fuse, hdr->unique, &out, sizeof(out));
+    return fuse_reply_attr(fuse, hdr->unique, hdr->nodeid, path);
 }
 
-static void handle_setattr(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_setattr(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header *hdr, const struct fuse_setattr_in *req)
 {
-    struct fuse_attr_out out;
-    char *path, buffer[PATH_MAX];
-    int res = 0;
+    struct node* node;
+    char path[PATH_MAX];
     struct timespec times[2];
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
 
-    TRACE("SETATTR fh=%llx valid=%x @ %llx (%s)\n", req->fh, req->valid,
-            hdr->nodeid, node ? node->name : "?");
+    pthread_mutex_lock(&fuse->lock);
+    node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid, path, sizeof(path));
+    TRACE("[%d] SETATTR fh=%llx valid=%x @ %llx (%s)\n", handler->token,
+            req->fh, req->valid, hdr->nodeid, node ? node->name : "?");
+    pthread_mutex_unlock(&fuse->lock);
+
     if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+        return -ENOENT;
     }
 
-    /* XXX: incomplete implementation on purpose.   chmod/chown
-     * should NEVER be implemented.*/
+    /* XXX: incomplete implementation on purpose.
+     * chmod/chown should NEVER be implemented.*/
 
-    path = node_get_path(node, buffer, 0);
-    if (req->valid & FATTR_SIZE)
-        res = truncate(path, req->size);
-    if (res < 0) {
-        fuse_status(fuse, hdr->unique, -errno);
-        return;
+    if ((req->valid & FATTR_SIZE) && truncate(path, req->size) < 0) {
+        return -errno;
     }
 
     /* Handle changing atime and mtime.  If FATTR_ATIME_and FATTR_ATIME_NOW
@@ -646,253 +622,284 @@
               times[1].tv_nsec = req->mtimensec;
             }
         }
-        TRACE("Calling utimensat on %s with atime %ld, mtime=%ld\n", path, times[0].tv_sec, times[1].tv_sec);
-        res = utimensat(-1, path, times, 0);
-        if (res < 0) {
-            fuse_status(fuse, hdr->unique, -errno);
-            return;
+        TRACE("[%d] Calling utimensat on %s with atime %ld, mtime=%ld\n",
+                handler->token, path, times[0].tv_sec, times[1].tv_sec);
+        if (utimensat(-1, path, times, 0) < 0) {
+            return -errno;
         }
     }
-
-    memset(&out, 0, sizeof(out));
-    node_get_attr(node, &out.attr);
-    out.attr_valid = 10;
-    fuse_reply(fuse, hdr->unique, &out, sizeof(out));
+    return fuse_reply_attr(fuse, hdr->unique, hdr->nodeid, path);
 }
 
-static void handle_mknod(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_mknod(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_mknod_in* req, const char* name)
 {
-    char *path, buffer[PATH_MAX];
-    int res;
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
+    struct node* parent_node;
+    char parent_path[PATH_MAX];
+    char child_path[PATH_MAX];
+    const char* actual_name;
 
-    TRACE("MKNOD %s @ %llx (%s)\n", name, hdr->nodeid, node ? node->name : "?");
-    if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+    pthread_mutex_lock(&fuse->lock);
+    parent_node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid,
+            parent_path, sizeof(parent_path));
+    TRACE("[%d] MKNOD %s 0%o @ %llx (%s)\n", handler->token,
+            name, req->mode, hdr->nodeid, parent_node ? parent_node->name : "?");
+    pthread_mutex_unlock(&fuse->lock);
+
+    if (!parent_node || !(actual_name = find_file_within(parent_path, name,
+            child_path, sizeof(child_path), 1))) {
+        return -ENOENT;
     }
-
-    path = node_get_path(node, buffer, name);
-
     __u32 mode = (req->mode & (~0777)) | 0664;
-    res = mknod(path, mode, req->rdev); /* XXX perm?*/
-    if (res < 0) {
-        fuse_status(fuse, hdr->unique, -errno);
-        return;
+    if (mknod(child_path, mode, req->rdev) < 0) {
+        return -errno;
     }
-
-    lookup_entry(fuse, handler, hdr->unique, node, name);
+    return fuse_reply_entry(fuse, hdr->unique, parent_node, name, actual_name, child_path);
 }
 
-static void handle_mkdir(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_mkdir(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_mkdir_in* req, const char* name)
 {
-    struct fuse_entry_out out;
-    char *path, buffer[PATH_MAX];
-    int res;
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
+    struct node* parent_node;
+    char parent_path[PATH_MAX];
+    char child_path[PATH_MAX];
+    const char* actual_name;
 
-    TRACE("MKDIR %s 0%o @ %llx (%s)\n", name, req->mode, hdr->nodeid, node ? node->name : "?");
-    if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+    pthread_mutex_lock(&fuse->lock);
+    parent_node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid,
+            parent_path, sizeof(parent_path));
+    TRACE("[%d] MKDIR %s 0%o @ %llx (%s)\n", handler->token,
+            name, req->mode, hdr->nodeid, parent_node ? parent_node->name : "?");
+    pthread_mutex_unlock(&fuse->lock);
+
+    if (!parent_node || !(actual_name = find_file_within(parent_path, name,
+            child_path, sizeof(child_path), 1))) {
+        return -ENOENT;
     }
-
-    path = node_get_path(node, buffer, name);
-
     __u32 mode = (req->mode & (~0777)) | 0775;
-    res = mkdir(path, mode);
-    if (res < 0) {
-        fuse_status(fuse, hdr->unique, -errno);
-        return;
+    if (mkdir(child_path, mode) < 0) {
+        return -errno;
     }
-
-    lookup_entry(fuse, handler, hdr->unique, node, name);
+    return fuse_reply_entry(fuse, hdr->unique, parent_node, name, actual_name, child_path);
 }
 
-static void handle_unlink(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_unlink(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const char* name)
 {
-    char *path, buffer[PATH_MAX];
-    int res;
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
+    struct node* parent_node;
+    char parent_path[PATH_MAX];
+    char child_path[PATH_MAX];
 
-    TRACE("UNLINK %s @ %llx (%s)\n", name, hdr->nodeid, node ? node->name : "?");
-    if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+    pthread_mutex_lock(&fuse->lock);
+    parent_node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid,
+            parent_path, sizeof(parent_path));
+    TRACE("[%d] UNLINK %s @ %llx (%s)\n", handler->token,
+            name, hdr->nodeid, parent_node ? parent_node->name : "?");
+    pthread_mutex_unlock(&fuse->lock);
+
+    if (!parent_node || !find_file_within(parent_path, name,
+            child_path, sizeof(child_path), 1)) {
+        return -ENOENT;
     }
-
-    path = node_get_path(node, buffer, name);
-    res = unlink(path);
-    fuse_status(fuse, hdr->unique, res ? -errno : 0);
+    if (unlink(child_path) < 0) {
+        return -errno;
+    }
+    return 0;
 }
 
-static void handle_rmdir(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_rmdir(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const char* name)
 {
-    char *path, buffer[PATH_MAX];
-    int res;
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
+    struct node* parent_node;
+    char parent_path[PATH_MAX];
+    char child_path[PATH_MAX];
 
-    TRACE("RMDIR %s @ %llx (%s)\n", name, hdr->nodeid, node ? node->name : "?");
-    if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+    pthread_mutex_lock(&fuse->lock);
+    parent_node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid,
+            parent_path, sizeof(parent_path));
+    TRACE("[%d] RMDIR %s @ %llx (%s)\n", handler->token,
+            name, hdr->nodeid, parent_node ? parent_node->name : "?");
+    pthread_mutex_unlock(&fuse->lock);
+
+    if (!parent_node || !find_file_within(parent_path, name,
+            child_path, sizeof(child_path), 1)) {
+        return -ENOENT;
     }
-
-    path = node_get_path(node, buffer, name);
-    res = rmdir(path);
-    fuse_status(fuse, hdr->unique, res ? -errno : 0);
+    if (rmdir(child_path) < 0) {
+        return -errno;
+    }
+    return 0;
 }
 
-static void handle_rename(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_rename(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_rename_in* req,
-        const char* oldname, const char* newname)
+        const char* old_name, const char* new_name)
 {
-    char *oldpath, oldbuffer[PATH_MAX];
-    char *newpath, newbuffer[PATH_MAX];
-    struct node *target;
-    struct node *newparent;
+    struct node* old_parent_node;
+    struct node* new_parent_node;
+    struct node* child_node;
+    char old_parent_path[PATH_MAX];
+    char new_parent_path[PATH_MAX];
+    char old_child_path[PATH_MAX];
+    char new_child_path[PATH_MAX];
+    const char* new_actual_name;
     int res;
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
 
-    TRACE("RENAME %s->%s @ %llx (%s)\n", oldname, newname, hdr->nodeid, node ? node->name : "?");
-    if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+    pthread_mutex_lock(&fuse->lock);
+    old_parent_node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid,
+            old_parent_path, sizeof(old_parent_path));
+    new_parent_node = lookup_node_and_path_by_id_locked(fuse, req->newdir,
+            new_parent_path, sizeof(new_parent_path));
+    TRACE("[%d] RENAME %s->%s @ %llx (%s) -> %llx (%s)\n", handler->token,
+            old_name, new_name,
+            hdr->nodeid, old_parent_node ? old_parent_node->name : "?",
+            req->newdir, new_parent_node ? new_parent_node->name : "?");
+    if (!old_parent_node || !new_parent_node) {
+        res = -ENOENT;
+        goto lookup_error;
+    }
+    child_node = lookup_child_by_name_locked(old_parent_node, old_name);
+    if (!child_node || get_node_path_locked(child_node,
+            old_child_path, sizeof(old_child_path)) < 0) {
+        res = -ENOENT;
+        goto lookup_error;
+    }
+    acquire_node_locked(child_node);
+    pthread_mutex_unlock(&fuse->lock);
+
+    /* Special case for renaming a file where destination is same path
+     * differing only by case.  In this case we don't want to look for a case
+     * insensitive match.  This allows commands like "mv foo FOO" to work as expected.
+     */
+    int search = old_parent_node != new_parent_node
+            || strcasecmp(old_name, new_name);
+    if (!(new_actual_name = find_file_within(new_parent_path, new_name,
+            new_child_path, sizeof(new_child_path), search))) {
+        res = -ENOENT;
+        goto io_error;
     }
 
-    target = lookup_child_by_name(node, oldname);
-    if (!target) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
-    }
-    oldpath = node_get_path(node, oldbuffer, oldname);
-
-    newparent = lookup_by_inode(fuse, req->newdir);
-    if (!newparent) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
-    }
-    if (newparent == node) {
-        /* Special case for renaming a file where destination
-         * is same path differing only by case.
-         * In this case we don't want to look for a case insensitive match.
-         * This allows commands like "mv foo FOO" to work as expected.
-         */
-        newpath = do_node_get_path(newparent, newbuffer, newname, NO_CASE_SENSITIVE_MATCH);
-    } else {
-        newpath = node_get_path(newparent, newbuffer, newname);
+    TRACE("[%d] RENAME %s->%s\n", handler->token, old_child_path, new_child_path);
+    res = rename(old_child_path, new_child_path);
+    if (res < 0) {
+        res = -errno;
+        goto io_error;
     }
 
-    if (!remove_child(node, target->nid)) {
-        ERROR("RENAME remove_child not found");
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+    pthread_mutex_lock(&fuse->lock);
+    res = rename_node_locked(child_node, new_name, new_actual_name);
+    if (!res) {
+        remove_node_from_parent_locked(child_node);
+        add_node_to_parent_locked(child_node, new_parent_node);
     }
-    if (!rename_node(target, newname)) {
-        fuse_status(fuse, hdr->unique, -ENOMEM);
-        return;
-    }
-    add_node_to_parent(target, newparent);
+    goto done;
 
-    res = rename(oldpath, newpath);
-    TRACE("RENAME result %d\n", res);
-
-    fuse_status(fuse, hdr->unique, res ? -errno : 0);
+io_error:
+    pthread_mutex_lock(&fuse->lock);
+done:
+    release_node_locked(child_node);
+lookup_error:
+    pthread_mutex_unlock(&fuse->lock);
+    return res;
 }
 
-static void handle_open(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_open(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_open_in* req)
 {
+    struct node* node;
+    char path[PATH_MAX];
     struct fuse_open_out out;
-    char *path, buffer[PATH_MAX];
     struct handle *h;
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
 
+    pthread_mutex_lock(&fuse->lock);
+    node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid, path, sizeof(path));
+    TRACE("[%d] OPEN 0%o @ %llx (%s)\n", handler->token,
+            req->flags, hdr->nodeid, node ? node->name : "?");
+    pthread_mutex_unlock(&fuse->lock);
+
+    if (!node) {
+        return -ENOENT;
+    }
     h = malloc(sizeof(*h));
     if (!h) {
-        fuse_status(fuse, hdr->unique, -ENOMEM);
-        return;
+        return -ENOMEM;
     }
-
-    TRACE("OPEN 0%o fh=%p @ %llx (%s)\n", req->flags, h, hdr->nodeid, node ? node->name : "?");
-    if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
-    }
-
-    path = node_get_path(node, buffer, 0);
+    TRACE("[%d] OPEN %s\n", handler->token, path);
     h->fd = open(path, req->flags);
     if (h->fd < 0) {
-        ERROR("ERROR\n");
-        fuse_status(fuse, hdr->unique, -errno);
         free(h);
-        return;
+        return -errno;
     }
     out.fh = ptr_to_id(h);
     out.open_flags = 0;
     out.padding = 0;
     fuse_reply(fuse, hdr->unique, &out, sizeof(out));
+    return NO_STATUS;
 }
 
-static void handle_read(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_read(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_read_in* req)
 {
     struct handle *h = id_to_ptr(req->fh);
     __u64 unique = hdr->unique;
     __u32 size = req->size;
     __u64 offset = req->offset;
+    int res;
+
     /* Don't access any other fields of hdr or req beyond this point, the read buffer
      * overlaps the request buffer and will clobber data in the request.  This
      * saves us 128KB per request handler thread at the cost of this scary comment. */
-    int res;
-    TRACE("READ %p(%d) %u@%llu\n", h, h->fd, size, offset);
+
+    TRACE("[%d] READ %p(%d) %u@%llu\n", handler->token,
+            h, h->fd, size, offset);
     if (size > sizeof(handler->read_buffer)) {
-        fuse_status(fuse, unique, -EINVAL);
-        return;
+        return -EINVAL;
     }
     res = pread64(h->fd, handler->read_buffer, size, offset);
     if (res < 0) {
-        fuse_status(fuse, unique, -errno);
-        return;
+        return -errno;
     }
     fuse_reply(fuse, unique, handler->read_buffer, res);
+    return NO_STATUS;
 }
 
-static void handle_write(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_write(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_write_in* req,
         const void* buffer)
 {
     struct fuse_write_out out;
     struct handle *h = id_to_ptr(req->fh);
     int res;
-    TRACE("WRITE %p(%d) %u@%llu\n", h, h->fd, req->size, req->offset);
+
+    TRACE("[%d] WRITE %p(%d) %u@%llu\n", handler->token,
+            h, h->fd, req->size, req->offset);
     res = pwrite64(h->fd, buffer, req->size, req->offset);
     if (res < 0) {
-        fuse_status(fuse, hdr->unique, -errno);
-        return;
+        return -errno;
     }
     out.size = res;
     fuse_reply(fuse, hdr->unique, &out, sizeof(out));
+    return NO_STATUS;
 }
 
-static void handle_statfs(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_statfs(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr)
 {
+    char path[PATH_MAX];
     struct statfs stat;
     struct fuse_statfs_out out;
     int res;
 
-    TRACE("STATFS\n");
-
-    if (statfs(fuse->root.name, &stat)) {
-        fuse_status(fuse, hdr->unique, -errno);
-        return;
+    pthread_mutex_lock(&fuse->lock);
+    TRACE("[%d] STATFS\n", handler->token);
+    res = get_node_path_locked(&fuse->root, path, sizeof(path));
+    pthread_mutex_unlock(&fuse->lock);
+    if (res < 0) {
+        return -ENOENT;
     }
-
+    if (statfs(fuse->root.name, &stat) < 0) {
+        return -errno;
+    }
     memset(&out, 0, sizeof(out));
     out.st.blocks = stat.f_blocks;
     out.st.bfree = stat.f_bfree;
@@ -903,88 +910,92 @@
     out.st.namelen = stat.f_namelen;
     out.st.frsize = stat.f_frsize;
     fuse_reply(fuse, hdr->unique, &out, sizeof(out));
+    return NO_STATUS;
 }
 
-static void handle_release(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_release(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_release_in* req)
 {
     struct handle *h = id_to_ptr(req->fh);
-    TRACE("RELEASE %p(%d)\n", h, h->fd);
+
+    TRACE("[%d] RELEASE %p(%d)\n", handler->token, h, h->fd);
     close(h->fd);
     free(h);
-    fuse_status(fuse, hdr->unique, 0);
+    return 0;
 }
 
-static void handle_fsync(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_fsync(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_fsync_in* req)
 {
     int is_data_sync = req->fsync_flags & 1;
     struct handle *h = id_to_ptr(req->fh);
     int res;
-    TRACE("FSYNC %p(%d) is_data_sync=%d\n", h, h->fd, is_data_sync);
+
+    TRACE("[%d] FSYNC %p(%d) is_data_sync=%d\n", handler->token,
+            h, h->fd, is_data_sync);
     res = is_data_sync ? fdatasync(h->fd) : fsync(h->fd);
     if (res < 0) {
-        fuse_status(fuse, hdr->unique, -errno);
-        return;
+        return -errno;
     }
-    fuse_status(fuse, hdr->unique, 0);
+    return 0;
 }
 
-static void handle_flush(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_flush(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr)
 {
-    fuse_status(fuse, hdr->unique, 0);
+    TRACE("[%d] FLUSH\n", handler->token);
+    return 0;
 }
 
-static void handle_opendir(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_opendir(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_open_in* req)
 {
+    struct node* node;
+    char path[PATH_MAX];
     struct fuse_open_out out;
-    char *path, buffer[PATH_MAX];
     struct dirhandle *h;
-    struct node* node = lookup_by_inode(fuse, hdr->nodeid);
 
-    TRACE("OPENDIR @ %llx (%s)\n", hdr->nodeid, node ? node->name : "?");
+    pthread_mutex_lock(&fuse->lock);
+    node = lookup_node_and_path_by_id_locked(fuse, hdr->nodeid, path, sizeof(path));
+    TRACE("[%d] OPENDIR @ %llx (%s)\n", handler->token,
+            hdr->nodeid, node ? node->name : "?");
+    pthread_mutex_unlock(&fuse->lock);
+
     if (!node) {
-        fuse_status(fuse, hdr->unique, -ENOENT);
-        return;
+        return -ENOENT;
     }
-
     h = malloc(sizeof(*h));
     if (!h) {
-        fuse_status(fuse, hdr->unique, -ENOMEM);
-        return;
+        return -ENOMEM;
     }
-
-    path = node_get_path(node, buffer, 0);
+    TRACE("[%d] OPENDIR %s\n", handler->token, path);
     h->d = opendir(path);
-    if (h->d == 0) {
-        ERROR("ERROR\n");
-        fuse_status(fuse, hdr->unique, -errno);
+    if (!h->d) {
         free(h);
-        return;
+        return -errno;
     }
     out.fh = ptr_to_id(h);
     fuse_reply(fuse, hdr->unique, &out, sizeof(out));
+    return NO_STATUS;
 }
 
-static void handle_readdir(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_readdir(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_read_in* req)
 {
     char buffer[8192];
     struct fuse_dirent *fde = (struct fuse_dirent*) buffer;
     struct dirent *de;
     struct dirhandle *h = id_to_ptr(req->fh);
-    TRACE("READDIR %p\n", h);
+
+    TRACE("[%d] READDIR %p\n", handler->token, h);
     if (req->offset == 0) {
         /* rewinddir() might have been called above us, so rewind here too */
-        TRACE("calling rewinddir()\n");
+        TRACE("[%d] calling rewinddir()\n", handler->token);
         rewinddir(h->d);
     }
     de = readdir(h->d);
     if (!de) {
-        fuse_status(fuse, hdr->unique, 0);
-        return;
+        return 0;
     }
     fde->ino = FUSE_UNKNOWN_INO;
     /* increment the offset so we can detect when rewinddir() seeks back to the beginning */
@@ -993,27 +1004,28 @@
     fde->namelen = strlen(de->d_name);
     memcpy(fde->name, de->d_name, fde->namelen + 1);
     fuse_reply(fuse, hdr->unique, fde,
-               FUSE_DIRENT_ALIGN(sizeof(struct fuse_dirent) + fde->namelen));
+            FUSE_DIRENT_ALIGN(sizeof(struct fuse_dirent) + fde->namelen));
+    return NO_STATUS;
 }
 
-static void handle_releasedir(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_releasedir(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_release_in* req)
 {
     struct dirhandle *h = id_to_ptr(req->fh);
-    TRACE("RELEASEDIR %p\n",h);
+
+    TRACE("[%d] RELEASEDIR %p\n", handler->token, h);
     closedir(h->d);
     free(h);
-    fuse_status(fuse, hdr->unique, 0);
+    return 0;
 }
 
-static void handle_init(struct fuse* fuse, struct fuse_handler* handler,
+static int handle_init(struct fuse* fuse, struct fuse_handler* handler,
         const struct fuse_in_header* hdr, const struct fuse_init_in* req)
 {
     struct fuse_init_out out;
 
-    TRACE("INIT ver=%d.%d maxread=%d flags=%x\n",
-            req->major, req->minor, req->max_readahead, req->flags);
-
+    TRACE("[%d] INIT ver=%d.%d maxread=%d flags=%x\n",
+            handler->token, req->major, req->minor, req->max_readahead, req->flags);
     out.major = FUSE_KERNEL_VERSION;
     out.minor = FUSE_KERNEL_MINOR_VERSION;
     out.max_readahead = req->max_readahead;
@@ -1021,36 +1033,32 @@
     out.max_background = 32;
     out.congestion_threshold = 32;
     out.max_write = MAX_WRITE;
-
     fuse_reply(fuse, hdr->unique, &out, sizeof(out));
+    return NO_STATUS;
 }
 
-static void handle_fuse_request(struct fuse *fuse, struct fuse_handler* handler,
+static int handle_fuse_request(struct fuse *fuse, struct fuse_handler* handler,
         const struct fuse_in_header *hdr, const void *data, size_t data_len)
 {
     switch (hdr->opcode) {
     case FUSE_LOOKUP: { /* bytez[] -> entry_out */
         const char* name = data;
-        handle_lookup(fuse, handler, hdr, name);
-        return;
+        return handle_lookup(fuse, handler, hdr, name);
     }
 
     case FUSE_FORGET: {
         const struct fuse_forget_in *req = data;
-        handle_forget(fuse, handler, hdr, req);
-        return;
+        return handle_forget(fuse, handler, hdr, req);
     }
 
     case FUSE_GETATTR: { /* getattr_in -> attr_out */
         const struct fuse_getattr_in *req = data;
-        handle_getaddr(fuse, handler, hdr, req);
-        return;
+        return handle_getattr(fuse, handler, hdr, req);
     }
 
     case FUSE_SETATTR: { /* setattr_in -> attr_out */
         const struct fuse_setattr_in *req = data;
-        handle_setattr(fuse, handler, hdr, req);
-        return;
+        return handle_setattr(fuse, handler, hdr, req);
     }
 
 //    case FUSE_READLINK:
@@ -1058,72 +1066,61 @@
     case FUSE_MKNOD: { /* mknod_in, bytez[] -> entry_out */
         const struct fuse_mknod_in *req = data;
         const char *name = ((const char*) data) + sizeof(*req);
-        handle_mknod(fuse, handler, hdr, req, name);
-        return;
+        return handle_mknod(fuse, handler, hdr, req, name);
     }
 
     case FUSE_MKDIR: { /* mkdir_in, bytez[] -> entry_out */
         const struct fuse_mkdir_in *req = data;
         const char *name = ((const char*) data) + sizeof(*req);
-        handle_mkdir(fuse, handler, hdr, req, name);
-        return;
+        return handle_mkdir(fuse, handler, hdr, req, name);
     }
 
     case FUSE_UNLINK: { /* bytez[] -> */
         const char* name = data;
-        handle_unlink(fuse, handler, hdr, name);
-        return;
+        return handle_unlink(fuse, handler, hdr, name);
     }
 
     case FUSE_RMDIR: { /* bytez[] -> */
         const char* name = data;
-        handle_rmdir(fuse, handler, hdr, name);
-        return;
+        return handle_rmdir(fuse, handler, hdr, name);
     }
 
     case FUSE_RENAME: { /* rename_in, oldname, newname ->  */
         const struct fuse_rename_in *req = data;
-        const char *oldname = ((const char*) data) + sizeof(*req);
-        const char *newname = oldname + strlen(oldname) + 1;
-        handle_rename(fuse, handler, hdr, req, oldname, newname);
-        return;
+        const char *old_name = ((const char*) data) + sizeof(*req);
+        const char *new_name = old_name + strlen(old_name) + 1;
+        return handle_rename(fuse, handler, hdr, req, old_name, new_name);
     }
 
 //    case FUSE_LINK:
     case FUSE_OPEN: { /* open_in -> open_out */
         const struct fuse_open_in *req = data;
-        handle_open(fuse, handler, hdr, req);
-        return;
+        return handle_open(fuse, handler, hdr, req);
     }
 
     case FUSE_READ: { /* read_in -> byte[] */
         const struct fuse_read_in *req = data;
-        handle_read(fuse, handler, hdr, req);
-        return;
+        return handle_read(fuse, handler, hdr, req);
     }
 
     case FUSE_WRITE: { /* write_in, byte[write_in.size] -> write_out */
         const struct fuse_write_in *req = data;
         const void* buffer = (const __u8*)data + sizeof(*req);
-        handle_write(fuse, handler, hdr, req, buffer);
-        return;
+        return handle_write(fuse, handler, hdr, req, buffer);
     }
 
     case FUSE_STATFS: { /* getattr_in -> attr_out */
-        handle_statfs(fuse, handler, hdr);
-        return;
+        return handle_statfs(fuse, handler, hdr);
     }
 
     case FUSE_RELEASE: { /* release_in -> */
         const struct fuse_release_in *req = data;
-        handle_release(fuse, handler, hdr, req);
-        return;
+        return handle_release(fuse, handler, hdr, req);
     }
 
     case FUSE_FSYNC: {
         const struct fuse_fsync_in *req = data;
-        handle_fsync(fuse, handler, hdr, req);
-        return;
+        return handle_fsync(fuse, handler, hdr, req);
     }
 
 //    case FUSE_SETXATTR:
@@ -1131,90 +1128,129 @@
 //    case FUSE_LISTXATTR:
 //    case FUSE_REMOVEXATTR:
     case FUSE_FLUSH: {
-        handle_flush(fuse, handler, hdr);
-        return;
+        return handle_flush(fuse, handler, hdr);
     }
 
     case FUSE_OPENDIR: { /* open_in -> open_out */
         const struct fuse_open_in *req = data;
-        handle_opendir(fuse, handler, hdr, req);
-        return;
+        return handle_opendir(fuse, handler, hdr, req);
     }
 
     case FUSE_READDIR: {
         const struct fuse_read_in *req = data;
-        handle_readdir(fuse, handler, hdr, req);
-        return;
+        return handle_readdir(fuse, handler, hdr, req);
     }
 
     case FUSE_RELEASEDIR: { /* release_in -> */
         const struct fuse_release_in *req = data;
-        handle_releasedir(fuse, handler, hdr, req);
-        return;
+        return handle_releasedir(fuse, handler, hdr, req);
     }
 
 //    case FUSE_FSYNCDIR:
     case FUSE_INIT: { /* init_in -> init_out */
         const struct fuse_init_in *req = data;
-        handle_init(fuse, handler, hdr, req);
-        return;
+        return handle_init(fuse, handler, hdr, req);
     }
 
     default: {
-        ERROR("NOTIMPL op=%d uniq=%llx nid=%llx\n",
-                hdr->opcode, hdr->unique, hdr->nodeid);
-        fuse_status(fuse, hdr->unique, -ENOSYS);
-        break;
+        TRACE("[%d] NOTIMPL op=%d uniq=%llx nid=%llx\n",
+                handler->token, hdr->opcode, hdr->unique, hdr->nodeid);
+        return -ENOSYS;
     }
     }
 }
 
-static void handle_fuse_requests(struct fuse *fuse, struct fuse_handler* handler)
+static void handle_fuse_requests(struct fuse_handler* handler)
 {
+    struct fuse* fuse = handler->fuse;
     for (;;) {
-        ssize_t len = read(fuse->fd, handler->request_buffer, sizeof(handler->request_buffer));
+        ssize_t len = read(fuse->fd,
+                handler->request_buffer, sizeof(handler->request_buffer));
         if (len < 0) {
-            if (errno == EINTR)
-                continue;
-            ERROR("handle_fuse_requests: errno=%d\n", errno);
-            return;
+            if (errno != EINTR) {
+                ERROR("[%d] handle_fuse_requests: errno=%d\n", handler->token, errno);
+            }
+            continue;
         }
 
         if ((size_t)len < sizeof(struct fuse_in_header)) {
-            ERROR("request too short: len=%zu\n", (size_t)len);
-            return;
+            ERROR("[%d] request too short: len=%zu\n", handler->token, (size_t)len);
+            continue;
         }
 
         const struct fuse_in_header *hdr = (void*)handler->request_buffer;
         if (hdr->len != (size_t)len) {
-            ERROR("malformed header: len=%zu, hdr->len=%u\n", (size_t)len, hdr->len);
-            return;
+            ERROR("[%d] malformed header: len=%zu, hdr->len=%u\n",
+                    handler->token, (size_t)len, hdr->len);
+            continue;
         }
 
         const void *data = handler->request_buffer + sizeof(struct fuse_in_header);
         size_t data_len = len - sizeof(struct fuse_in_header);
-        handle_fuse_request(fuse, handler, hdr, data, data_len);
+        __u64 unique = hdr->unique;
+        int res = handle_fuse_request(fuse, handler, hdr, data, data_len);
 
         /* We do not access the request again after this point because the underlying
          * buffer storage may have been reused while processing the request. */
+
+        if (res != NO_STATUS) {
+            if (res) {
+                TRACE("[%d] ERROR %d\n", handler->token, res);
+            }
+            fuse_status(fuse, unique, res);
+        }
     }
 }
 
-static int ignite_fuse(struct fuse* fuse)
+static void* start_handler(void* data)
 {
-    /* use only one handler thread for now */
-    struct fuse_handler handler;
-    handle_fuse_requests(fuse, &handler);
-    return 0;
+    struct fuse_handler* handler = data;
+    handle_fuse_requests(handler);
+    return NULL;
+}
+
+static int ignite_fuse(struct fuse* fuse, int num_threads)
+{
+    struct fuse_handler* handlers;
+    int i;
+
+    handlers = malloc(num_threads * sizeof(struct fuse_handler));
+    if (!handlers) {
+        ERROR("cannot allocate storage for threads");
+        return -ENOMEM;
+    }
+
+    for (i = 0; i < num_threads; i++) {
+        handlers[i].fuse = fuse;
+        handlers[i].token = i;
+    }
+
+    for (i = 1; i < num_threads; i++) {
+        pthread_t thread;
+        int res = pthread_create(&thread, NULL, start_handler, &handlers[i]);
+        if (res) {
+            ERROR("failed to start thread #%d, error=%d", i, res);
+            goto quit;
+        }
+    }
+    handle_fuse_requests(&handlers[0]);
+    ERROR("terminated prematurely");
+
+    /* don't bother killing all of the other threads or freeing anything,
+     * should never get here anyhow */
+quit:
+    exit(1);
 }
 
 static int usage()
 {
-    ERROR("usage: sdcard <path> <uid> <gid>\n\n");
+    ERROR("usage: sdcard [-t<threads>] <path> <uid> <gid>\n"
+            "    -t<threads>: specify number of threads to use, default -t%d\n"
+            "\n", DEFAULT_NUM_THREADS);
     return 1;
 }
 
-static int run(const char* path, uid_t uid, gid_t gid)
+static int run(const char* path, uid_t uid, gid_t gid, int num_threads)
 {
     int fd;
     char opts[256];
@@ -1255,7 +1291,7 @@
     fuse_init(&fuse, fd, path);
 
     umask(0);
-    res = ignite_fuse(&fuse);
+    res = ignite_fuse(&fuse, num_threads);
 
     /* we do not attempt to umount the file system here because we are no longer
      * running as the root user */
@@ -1267,16 +1303,18 @@
 
 int main(int argc, char **argv)
 {
-    int fd;
     int res;
     const char *path = NULL;
     uid_t uid = 0;
     gid_t gid = 0;
+    int num_threads = DEFAULT_NUM_THREADS;
     int i;
 
     for (i = 1; i < argc; i++) {
         char* arg = argv[i];
-        if (!path)
+        if (!strncmp(arg, "-t", 2))
+            num_threads = strtoul(arg + 2, 0, 10);
+        else if (!path)
             path = arg;
         else if (!uid)
             uid = strtoul(arg, 0, 10);
@@ -1296,7 +1334,11 @@
         ERROR("uid and gid must be nonzero\n");
         return usage();
     }
+    if (num_threads < 1) {
+        ERROR("number of threads must be at least 1\n");
+        return usage();
+    }
 
-    res = run(path, uid, gid);
+    res = run(path, uid, gid, num_threads);
     return res < 0 ? 1 : 0;
 }