Merge "adb: don't use <error.h>."
diff --git a/adb/Android.bp b/adb/Android.bp
index 9f16c40..00e98fe 100644
--- a/adb/Android.bp
+++ b/adb/Android.bp
@@ -25,6 +25,7 @@
         "-Wvla",
     ],
     rtti: true,
+    cpp_std: "gnu++17",
 
     use_version_lib: true,
 
@@ -314,6 +315,8 @@
         "daemon/auth.cpp",
         "daemon/jdwp_service.cpp",
         "daemon/usb.cpp",
+        "daemon/usb_ffs.cpp",
+        "daemon/usb_legacy.cpp",
     ],
 
     local_include_dirs: [
diff --git a/adb/daemon/include/adbd/usb.h b/adb/daemon/include/adbd/usb.h
index 78e4cd5..3213f69 100644
--- a/adb/daemon/include/adbd/usb.h
+++ b/adb/daemon/include/adbd/usb.h
@@ -62,3 +62,5 @@
 };
 
 usb_handle *create_usb_handle(unsigned num_bufs, unsigned io_size);
+bool open_functionfs(android::base::unique_fd* control, android::base::unique_fd* bulk_out,
+                     android::base::unique_fd* bulk_in);
diff --git a/adb/daemon/usb.cpp b/adb/daemon/usb.cpp
index 9d495b0..f603d13 100644
--- a/adb/daemon/usb.cpp
+++ b/adb/daemon/usb.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007 The Android Open Source Project
+ * Copyright (C) 2018 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,528 +18,568 @@
 
 #include "sysdeps.h"
 
-#include <dirent.h>
 #include <errno.h>
-#include <linux/usb/ch9.h>
-#include <linux/usb/functionfs.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/ioctl.h>
-#include <sys/mman.h>
 #include <sys/types.h>
 #include <unistd.h>
 
-#include <algorithm>
-#include <atomic>
-#include <chrono>
-#include <condition_variable>
+#include <linux/usb/functionfs.h>
+#include <sys/eventfd.h>
+
+#include <array>
+#include <future>
+#include <memory>
 #include <mutex>
-#include <thread>
+#include <optional>
+#include <vector>
+
+#include <asyncio/AsyncIO.h>
 
 #include <android-base/logging.h>
+#include <android-base/macros.h>
 #include <android-base/properties.h>
+#include <android-base/thread_annotations.h>
 
-#include "adb.h"
-#include "adbd/usb.h"
+#include <adbd/usb.h>
+
+#include "adb_unique_fd.h"
+#include "adb_utils.h"
+#include "sysdeps/chrono.h"
 #include "transport.h"
+#include "types.h"
 
-using namespace std::chrono_literals;
+using android::base::StringPrintf;
 
-#define MAX_PACKET_SIZE_FS 64
-#define MAX_PACKET_SIZE_HS 512
-#define MAX_PACKET_SIZE_SS 1024
+static constexpr size_t kUsbReadQueueDepth = 16;
+static constexpr size_t kUsbReadSize = 16384;
 
-#define USB_FFS_BULK_SIZE 16384
+static constexpr size_t kUsbWriteQueueDepth = 16;
 
-// Number of buffers needed to fit MAX_PAYLOAD, with an extra for ZLPs.
-#define USB_FFS_NUM_BUFS ((4 * MAX_PAYLOAD / USB_FFS_BULK_SIZE) + 1)
-
-#define cpu_to_le16(x) htole16(x)
-#define cpu_to_le32(x) htole32(x)
-
-static unique_fd& dummy_fd = *new unique_fd();
-
-struct func_desc {
-    struct usb_interface_descriptor intf;
-    struct usb_endpoint_descriptor_no_audio source;
-    struct usb_endpoint_descriptor_no_audio sink;
-} __attribute__((packed));
-
-struct ss_func_desc {
-    struct usb_interface_descriptor intf;
-    struct usb_endpoint_descriptor_no_audio source;
-    struct usb_ss_ep_comp_descriptor source_comp;
-    struct usb_endpoint_descriptor_no_audio sink;
-    struct usb_ss_ep_comp_descriptor sink_comp;
-} __attribute__((packed));
-
-struct desc_v1 {
-    struct usb_functionfs_descs_head_v1 {
-        __le32 magic;
-        __le32 length;
-        __le32 fs_count;
-        __le32 hs_count;
-    } __attribute__((packed)) header;
-    struct func_desc fs_descs, hs_descs;
-} __attribute__((packed));
-
-struct desc_v2 {
-    struct usb_functionfs_descs_head_v2 header;
-    // The rest of the structure depends on the flags in the header.
-    __le32 fs_count;
-    __le32 hs_count;
-    __le32 ss_count;
-    __le32 os_count;
-    struct func_desc fs_descs, hs_descs;
-    struct ss_func_desc ss_descs;
-    struct usb_os_desc_header os_header;
-    struct usb_ext_compat_desc os_desc;
-} __attribute__((packed));
-
-static struct func_desc fs_descriptors = {
-    .intf = {
-        .bLength = sizeof(fs_descriptors.intf),
-        .bDescriptorType = USB_DT_INTERFACE,
-        .bInterfaceNumber = 0,
-        .bNumEndpoints = 2,
-        .bInterfaceClass = ADB_CLASS,
-        .bInterfaceSubClass = ADB_SUBCLASS,
-        .bInterfaceProtocol = ADB_PROTOCOL,
-        .iInterface = 1, /* first string from the provided table */
-    },
-    .source = {
-        .bLength = sizeof(fs_descriptors.source),
-        .bDescriptorType = USB_DT_ENDPOINT,
-        .bEndpointAddress = 1 | USB_DIR_OUT,
-        .bmAttributes = USB_ENDPOINT_XFER_BULK,
-        .wMaxPacketSize = MAX_PACKET_SIZE_FS,
-    },
-    .sink = {
-        .bLength = sizeof(fs_descriptors.sink),
-        .bDescriptorType = USB_DT_ENDPOINT,
-        .bEndpointAddress = 2 | USB_DIR_IN,
-        .bmAttributes = USB_ENDPOINT_XFER_BULK,
-        .wMaxPacketSize = MAX_PACKET_SIZE_FS,
-    },
-};
-
-static struct func_desc hs_descriptors = {
-    .intf = {
-        .bLength = sizeof(hs_descriptors.intf),
-        .bDescriptorType = USB_DT_INTERFACE,
-        .bInterfaceNumber = 0,
-        .bNumEndpoints = 2,
-        .bInterfaceClass = ADB_CLASS,
-        .bInterfaceSubClass = ADB_SUBCLASS,
-        .bInterfaceProtocol = ADB_PROTOCOL,
-        .iInterface = 1, /* first string from the provided table */
-    },
-    .source = {
-        .bLength = sizeof(hs_descriptors.source),
-        .bDescriptorType = USB_DT_ENDPOINT,
-        .bEndpointAddress = 1 | USB_DIR_OUT,
-        .bmAttributes = USB_ENDPOINT_XFER_BULK,
-        .wMaxPacketSize = MAX_PACKET_SIZE_HS,
-    },
-    .sink = {
-        .bLength = sizeof(hs_descriptors.sink),
-        .bDescriptorType = USB_DT_ENDPOINT,
-        .bEndpointAddress = 2 | USB_DIR_IN,
-        .bmAttributes = USB_ENDPOINT_XFER_BULK,
-        .wMaxPacketSize = MAX_PACKET_SIZE_HS,
-    },
-};
-
-static struct ss_func_desc ss_descriptors = {
-    .intf = {
-        .bLength = sizeof(ss_descriptors.intf),
-        .bDescriptorType = USB_DT_INTERFACE,
-        .bInterfaceNumber = 0,
-        .bNumEndpoints = 2,
-        .bInterfaceClass = ADB_CLASS,
-        .bInterfaceSubClass = ADB_SUBCLASS,
-        .bInterfaceProtocol = ADB_PROTOCOL,
-        .iInterface = 1, /* first string from the provided table */
-    },
-    .source = {
-        .bLength = sizeof(ss_descriptors.source),
-        .bDescriptorType = USB_DT_ENDPOINT,
-        .bEndpointAddress = 1 | USB_DIR_OUT,
-        .bmAttributes = USB_ENDPOINT_XFER_BULK,
-        .wMaxPacketSize = MAX_PACKET_SIZE_SS,
-    },
-    .source_comp = {
-        .bLength = sizeof(ss_descriptors.source_comp),
-        .bDescriptorType = USB_DT_SS_ENDPOINT_COMP,
-        .bMaxBurst = 4,
-    },
-    .sink = {
-        .bLength = sizeof(ss_descriptors.sink),
-        .bDescriptorType = USB_DT_ENDPOINT,
-        .bEndpointAddress = 2 | USB_DIR_IN,
-        .bmAttributes = USB_ENDPOINT_XFER_BULK,
-        .wMaxPacketSize = MAX_PACKET_SIZE_SS,
-    },
-    .sink_comp = {
-        .bLength = sizeof(ss_descriptors.sink_comp),
-        .bDescriptorType = USB_DT_SS_ENDPOINT_COMP,
-        .bMaxBurst = 4,
-    },
-};
-
-struct usb_ext_compat_desc os_desc_compat = {
-    .bFirstInterfaceNumber = 0,
-    .Reserved1 = cpu_to_le32(1),
-    .CompatibleID = {0},
-    .SubCompatibleID = {0},
-    .Reserved2 = {0},
-};
-
-static struct usb_os_desc_header os_desc_header = {
-    .interface = cpu_to_le32(1),
-    .dwLength = cpu_to_le32(sizeof(os_desc_header) + sizeof(os_desc_compat)),
-    .bcdVersion = cpu_to_le32(1),
-    .wIndex = cpu_to_le32(4),
-    .bCount = cpu_to_le32(1),
-    .Reserved = cpu_to_le32(0),
-};
-
-#define STR_INTERFACE_ "ADB Interface"
-
-static const struct {
-    struct usb_functionfs_strings_head header;
-    struct {
-        __le16 code;
-        const char str1[sizeof(STR_INTERFACE_)];
-    } __attribute__((packed)) lang0;
-} __attribute__((packed)) strings = {
-    .header = {
-        .magic = cpu_to_le32(FUNCTIONFS_STRINGS_MAGIC),
-        .length = cpu_to_le32(sizeof(strings)),
-        .str_count = cpu_to_le32(1),
-        .lang_count = cpu_to_le32(1),
-    },
-    .lang0 = {
-        cpu_to_le16(0x0409), /* en-us */
-        STR_INTERFACE_,
-    },
-};
-
-static void aio_block_init(aio_block* aiob, unsigned num_bufs) {
-    aiob->iocb.resize(num_bufs);
-    aiob->iocbs.resize(num_bufs);
-    aiob->events.resize(num_bufs);
-    aiob->num_submitted = 0;
-    for (unsigned i = 0; i < num_bufs; i++) {
-        aiob->iocbs[i] = &aiob->iocb[i];
-    }
-    memset(&aiob->ctx, 0, sizeof(aiob->ctx));
-    if (io_setup(num_bufs, &aiob->ctx)) {
-        D("[ aio: got error on io_setup (%d) ]", errno);
+static const char* to_string(enum usb_functionfs_event_type type) {
+    switch (type) {
+        case FUNCTIONFS_BIND:
+            return "FUNCTIONFS_BIND";
+        case FUNCTIONFS_UNBIND:
+            return "FUNCTIONFS_UNBIND";
+        case FUNCTIONFS_ENABLE:
+            return "FUNCTIONFS_ENABLE";
+        case FUNCTIONFS_DISABLE:
+            return "FUNCTIONFS_DISABLE";
+        case FUNCTIONFS_SETUP:
+            return "FUNCTIONFS_SETUP";
+        case FUNCTIONFS_SUSPEND:
+            return "FUNCTIONFS_SUSPEND";
+        case FUNCTIONFS_RESUME:
+            return "FUNCTIONFS_RESUME";
     }
 }
 
-static int getMaxPacketSize(int ffs_fd) {
-    usb_endpoint_descriptor desc;
-    if (ioctl(ffs_fd, FUNCTIONFS_ENDPOINT_DESC, reinterpret_cast<unsigned long>(&desc))) {
-        D("[ could not get endpoint descriptor! (%d) ]", errno);
-        return MAX_PACKET_SIZE_HS;
-    } else {
-        return desc.wMaxPacketSize;
+enum class TransferDirection : uint64_t {
+    READ = 0,
+    WRITE = 1,
+};
+
+struct TransferId {
+    TransferDirection direction : 1;
+    uint64_t id : 63;
+
+    TransferId() : TransferId(TransferDirection::READ, 0) {}
+
+  private:
+    TransferId(TransferDirection direction, uint64_t id) : direction(direction), id(id) {}
+
+  public:
+    explicit operator uint64_t() const {
+        uint64_t result;
+        static_assert(sizeof(*this) == sizeof(result));
+        memcpy(&result, this, sizeof(*this));
+        return result;
     }
-}
 
-static bool init_functionfs(struct usb_handle* h) {
-    LOG(INFO) << "initializing functionfs";
+    static TransferId read(uint64_t id) { return TransferId(TransferDirection::READ, id); }
+    static TransferId write(uint64_t id) { return TransferId(TransferDirection::WRITE, id); }
 
-    ssize_t ret;
-    struct desc_v1 v1_descriptor;
-    struct desc_v2 v2_descriptor;
+    static TransferId from_value(uint64_t value) {
+        TransferId result;
+        memcpy(&result, &value, sizeof(value));
+        return result;
+    }
+};
 
-    v2_descriptor.header.magic = cpu_to_le32(FUNCTIONFS_DESCRIPTORS_MAGIC_V2);
-    v2_descriptor.header.length = cpu_to_le32(sizeof(v2_descriptor));
-    v2_descriptor.header.flags = FUNCTIONFS_HAS_FS_DESC | FUNCTIONFS_HAS_HS_DESC |
-                                 FUNCTIONFS_HAS_SS_DESC | FUNCTIONFS_HAS_MS_OS_DESC;
-    v2_descriptor.fs_count = 3;
-    v2_descriptor.hs_count = 3;
-    v2_descriptor.ss_count = 5;
-    v2_descriptor.os_count = 1;
-    v2_descriptor.fs_descs = fs_descriptors;
-    v2_descriptor.hs_descs = hs_descriptors;
-    v2_descriptor.ss_descs = ss_descriptors;
-    v2_descriptor.os_header = os_desc_header;
-    v2_descriptor.os_desc = os_desc_compat;
+struct IoBlock {
+    bool pending;
+    struct iocb control;
+    Block payload;
 
-    if (h->control < 0) { // might have already done this before
-        LOG(INFO) << "opening control endpoint " << USB_FFS_ADB_EP0;
-        h->control.reset(adb_open(USB_FFS_ADB_EP0, O_WRONLY));
-        if (h->control < 0) {
-            PLOG(ERROR) << "cannot open control endpoint " << USB_FFS_ADB_EP0;
-            goto err;
+    TransferId id() const { return TransferId::from_value(control.aio_data); }
+};
+
+struct ScopedAioContext {
+    ScopedAioContext() = default;
+    ~ScopedAioContext() { reset(); }
+
+    ScopedAioContext(ScopedAioContext&& move) { reset(move.release()); }
+    ScopedAioContext(const ScopedAioContext& copy) = delete;
+
+    ScopedAioContext& operator=(ScopedAioContext&& move) {
+        reset(move.release());
+        return *this;
+    }
+    ScopedAioContext& operator=(const ScopedAioContext& copy) = delete;
+
+    static ScopedAioContext Create(size_t max_events) {
+        aio_context_t ctx = 0;
+        if (io_setup(max_events, &ctx) != 0) {
+            PLOG(FATAL) << "failed to create aio_context_t";
+        }
+        ScopedAioContext result;
+        result.reset(ctx);
+        return result;
+    }
+
+    aio_context_t release() {
+        aio_context_t result = context_;
+        context_ = 0;
+        return result;
+    }
+
+    void reset(aio_context_t new_context = 0) {
+        if (context_ != 0) {
+            io_destroy(context_);
         }
 
-        ret = adb_write(h->control.get(), &v2_descriptor, sizeof(v2_descriptor));
-        if (ret < 0) {
-            v1_descriptor.header.magic = cpu_to_le32(FUNCTIONFS_DESCRIPTORS_MAGIC);
-            v1_descriptor.header.length = cpu_to_le32(sizeof(v1_descriptor));
-            v1_descriptor.header.fs_count = 3;
-            v1_descriptor.header.hs_count = 3;
-            v1_descriptor.fs_descs = fs_descriptors;
-            v1_descriptor.hs_descs = hs_descriptors;
-            D("[ %s: Switching to V1_descriptor format errno=%d ]", USB_FFS_ADB_EP0, errno);
-            ret = adb_write(h->control.get(), &v1_descriptor, sizeof(v1_descriptor));
-            if (ret < 0) {
-                D("[ %s: write descriptors failed: errno=%d ]", USB_FFS_ADB_EP0, errno);
-                goto err;
+        context_ = new_context;
+    }
+
+    aio_context_t get() { return context_; }
+
+  private:
+    aio_context_t context_ = 0;
+};
+
+struct UsbFfsConnection : public Connection {
+    UsbFfsConnection(unique_fd control, unique_fd read, unique_fd write,
+                     std::promise<void> destruction_notifier)
+        : stopped_(false),
+          destruction_notifier_(std::move(destruction_notifier)),
+          control_fd_(std::move(control)),
+          read_fd_(std::move(read)),
+          write_fd_(std::move(write)) {
+        LOG(INFO) << "UsbFfsConnection constructed";
+        event_fd_.reset(eventfd(0, EFD_CLOEXEC));
+        if (event_fd_ == -1) {
+            PLOG(FATAL) << "failed to create eventfd";
+        }
+
+        aio_context_ = ScopedAioContext::Create(kUsbReadQueueDepth + kUsbWriteQueueDepth);
+    }
+
+    ~UsbFfsConnection() {
+        LOG(INFO) << "UsbFfsConnection being destroyed";
+        Stop();
+        monitor_thread_.join();
+        destruction_notifier_.set_value();
+    }
+
+    virtual bool Write(std::unique_ptr<apacket> packet) override final {
+        LOG(DEBUG) << "USB write: " << dump_header(&packet->msg);
+        Block header(sizeof(packet->msg));
+        memcpy(header.data(), &packet->msg, sizeof(packet->msg));
+
+        std::lock_guard<std::mutex> lock(write_mutex_);
+        write_requests_.push_back(CreateWriteBlock(std::move(header), next_write_id_++));
+        if (!packet->payload.empty()) {
+            write_requests_.push_back(
+                    CreateWriteBlock(std::move(packet->payload), next_write_id_++));
+        }
+        SubmitWrites();
+        return true;
+    }
+
+    virtual void Start() override final { StartMonitor(); }
+
+    virtual void Stop() override final {
+        if (stopped_.exchange(true)) {
+            return;
+        }
+        stopped_ = true;
+        uint64_t notify = 1;
+        ssize_t rc = adb_write(event_fd_.get(), &notify, sizeof(notify));
+        if (rc < 0) {
+            PLOG(FATAL) << "failed to notify eventfd to stop UsbFfsConnection";
+        }
+        CHECK_EQ(static_cast<size_t>(rc), sizeof(notify));
+    }
+
+  private:
+    void StartMonitor() {
+        // This is a bit of a mess.
+        // It's possible for io_submit to end up blocking, if we call it as the endpoint
+        // becomes disabled. Work around this by having a monitor thread to listen for functionfs
+        // lifecycle events. If we notice an error condition (either we've become disabled, or we
+        // were never enabled in the first place), we send interruption signals to the worker thread
+        // until it dies, and then report failure to the transport via HandleError, which will
+        // eventually result in the transport being destroyed, which will result in UsbFfsConnection
+        // being destroyed, which unblocks the open thread and restarts this entire process.
+        static constexpr int kInterruptionSignal = SIGUSR1;
+        static std::once_flag handler_once;
+        std::call_once(handler_once, []() { signal(kInterruptionSignal, [](int) {}); });
+
+        monitor_thread_ = std::thread([this]() {
+            adb_thread_setname("UsbFfs-monitor");
+
+            bool bound = false;
+            bool started = false;
+            bool running = true;
+            while (running) {
+                if (!bound || !started) {
+                    adb_pollfd pfd = {.fd = control_fd_.get(), .events = POLLIN, .revents = 0};
+                    int rc = TEMP_FAILURE_RETRY(adb_poll(&pfd, 1, 5000 /*ms*/));
+                    if (rc == -1) {
+                        PLOG(FATAL) << "poll on USB control fd failed";
+                    } else if (rc == 0) {
+                        // Something in the kernel presumably went wrong.
+                        // Close our endpoints, wait for a bit, and then try again.
+                        aio_context_.reset();
+                        read_fd_.reset();
+                        write_fd_.reset();
+                        control_fd_.reset();
+                        std::this_thread::sleep_for(5s);
+                        HandleError("didn't receive FUNCTIONFS_ENABLE, retrying");
+                        return;
+                    }
+                }
+
+                struct usb_functionfs_event event;
+                if (TEMP_FAILURE_RETRY(adb_read(control_fd_.get(), &event, sizeof(event))) !=
+                    sizeof(event)) {
+                    PLOG(FATAL) << "failed to read functionfs event";
+                }
+
+                LOG(INFO) << "USB event: "
+                          << to_string(static_cast<usb_functionfs_event_type>(event.type));
+
+                switch (event.type) {
+                    case FUNCTIONFS_BIND:
+                        CHECK(!started) << "received FUNCTIONFS_ENABLE while already bound?";
+                        bound = true;
+                        break;
+
+                    case FUNCTIONFS_ENABLE:
+                        CHECK(!started) << "received FUNCTIONFS_ENABLE while already running?";
+                        started = true;
+                        StartWorker();
+                        break;
+
+                    case FUNCTIONFS_DISABLE:
+                        running = false;
+                        break;
+                }
+            }
+
+            pthread_t worker_thread_handle = worker_thread_.native_handle();
+            while (true) {
+                int rc = pthread_kill(worker_thread_handle, kInterruptionSignal);
+                if (rc != 0) {
+                    LOG(ERROR) << "failed to send interruption signal to worker: " << strerror(rc);
+                    break;
+                }
+
+                std::this_thread::sleep_for(100ms);
+
+                rc = pthread_kill(worker_thread_handle, 0);
+                if (rc == 0) {
+                    continue;
+                } else if (rc == ESRCH) {
+                    break;
+                } else {
+                    LOG(ERROR) << "failed to send interruption signal to worker: " << strerror(rc);
+                }
+            }
+
+            worker_thread_.join();
+
+            aio_context_.reset();
+            read_fd_.reset();
+            write_fd_.reset();
+        });
+    }
+
+    void StartWorker() {
+        worker_thread_ = std::thread([this]() {
+            adb_thread_setname("UsbFfs-worker");
+            for (size_t i = 0; i < kUsbReadQueueDepth; ++i) {
+                read_requests_[i] = CreateReadBlock(next_read_id_++);
+                SubmitRead(&read_requests_[i]);
+            }
+
+            while (!stopped_) {
+                uint64_t dummy;
+                ssize_t rc = adb_read(event_fd_.get(), &dummy, sizeof(dummy));
+                if (rc == -1) {
+                    PLOG(FATAL) << "failed to read from eventfd";
+                } else if (rc == 0) {
+                    LOG(FATAL) << "hit EOF on eventfd";
+                }
+
+                WaitForEvents();
+            }
+        });
+    }
+
+    void PrepareReadBlock(IoBlock* block, uint64_t id) {
+        block->pending = false;
+        block->payload.resize(kUsbReadSize);
+        block->control.aio_data = static_cast<uint64_t>(TransferId::read(id));
+        block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload.data());
+        block->control.aio_nbytes = block->payload.size();
+    }
+
+    IoBlock CreateReadBlock(uint64_t id) {
+        IoBlock block;
+        PrepareReadBlock(&block, id);
+        block.control.aio_rw_flags = 0;
+        block.control.aio_lio_opcode = IOCB_CMD_PREAD;
+        block.control.aio_reqprio = 0;
+        block.control.aio_fildes = read_fd_.get();
+        block.control.aio_offset = 0;
+        block.control.aio_flags = IOCB_FLAG_RESFD;
+        block.control.aio_resfd = event_fd_.get();
+        return block;
+    }
+
+    void WaitForEvents() {
+        static constexpr size_t kMaxEvents = kUsbReadQueueDepth + kUsbWriteQueueDepth;
+        struct io_event events[kMaxEvents];
+        struct timespec timeout = {.tv_sec = 0, .tv_nsec = 0};
+        int rc = io_getevents(aio_context_.get(), 0, kMaxEvents, events, &timeout);
+        if (rc == -1) {
+            HandleError(StringPrintf("io_getevents failed while reading: %s", strerror(errno)));
+            return;
+        }
+
+        for (int event_idx = 0; event_idx < rc; ++event_idx) {
+            auto& event = events[event_idx];
+            TransferId id = TransferId::from_value(event.data);
+
+            if (event.res < 0) {
+                std::string error =
+                        StringPrintf("%s %" PRIu64 " failed with error %s",
+                                     id.direction == TransferDirection::READ ? "read" : "write",
+                                     id.id, strerror(-event.res));
+                HandleError(error);
+                return;
+            }
+
+            if (id.direction == TransferDirection::READ) {
+                HandleRead(id, event.res);
+            } else {
+                HandleWrite(id);
+            }
+        }
+    }
+
+    void HandleRead(TransferId id, int64_t size) {
+        uint64_t read_idx = id.id % kUsbReadQueueDepth;
+        IoBlock* block = &read_requests_[read_idx];
+        block->pending = false;
+        block->payload.resize(size);
+
+        // Notification for completed reads can be received out of order.
+        if (block->id().id != needed_read_id_) {
+            LOG(VERBOSE) << "read " << block->id().id << " completed while waiting for "
+                         << needed_read_id_;
+            return;
+        }
+
+        for (uint64_t id = needed_read_id_;; ++id) {
+            size_t read_idx = id % kUsbReadQueueDepth;
+            IoBlock* current_block = &read_requests_[read_idx];
+            if (current_block->pending) {
+                break;
+            }
+            ProcessRead(current_block);
+            ++needed_read_id_;
+        }
+    }
+
+    void ProcessRead(IoBlock* block) {
+        if (!block->payload.empty()) {
+            if (!incoming_header_.has_value()) {
+                CHECK_EQ(sizeof(amessage), block->payload.size());
+                amessage msg;
+                memcpy(&msg, block->payload.data(), sizeof(amessage));
+                LOG(DEBUG) << "USB read:" << dump_header(&msg);
+                incoming_header_ = msg;
+            } else {
+                size_t bytes_left = incoming_header_->data_length - incoming_payload_.size();
+                Block payload = std::move(block->payload);
+                CHECK_LE(payload.size(), bytes_left);
+                incoming_payload_.append(std::make_unique<Block>(std::move(payload)));
+            }
+
+            if (incoming_header_->data_length == incoming_payload_.size()) {
+                auto packet = std::make_unique<apacket>();
+                packet->msg = *incoming_header_;
+
+                // TODO: Make apacket contain an IOVector so we don't have to coalesce.
+                packet->payload = incoming_payload_.coalesce();
+                read_callback_(this, std::move(packet));
+
+                incoming_header_.reset();
+                incoming_payload_.clear();
             }
         }
 
-        ret = adb_write(h->control.get(), &strings, sizeof(strings));
-        if (ret < 0) {
-            D("[ %s: writing strings failed: errno=%d]", USB_FFS_ADB_EP0, errno);
-            goto err;
+        PrepareReadBlock(block, block->id().id + kUsbReadQueueDepth);
+        SubmitRead(block);
+    }
+
+    void SubmitRead(IoBlock* block) {
+        block->pending = true;
+        struct iocb* iocb = &block->control;
+        if (io_submit(aio_context_.get(), 1, &iocb) != 1) {
+            HandleError(StringPrintf("failed to submit read: %s", strerror(errno)));
+            return;
         }
-        //Signal only when writing the descriptors to ffs
-        android::base::SetProperty("sys.usb.ffs.ready", "1");
     }
 
-    h->bulk_out.reset(adb_open(USB_FFS_ADB_OUT, O_RDONLY));
-    if (h->bulk_out < 0) {
-        PLOG(ERROR) << "cannot open bulk-out endpoint " << USB_FFS_ADB_OUT;
-        goto err;
+    void HandleWrite(TransferId id) {
+        std::lock_guard<std::mutex> lock(write_mutex_);
+        auto it =
+                std::find_if(write_requests_.begin(), write_requests_.end(), [id](const auto& req) {
+                    return static_cast<uint64_t>(req->id()) == static_cast<uint64_t>(id);
+                });
+        CHECK(it != write_requests_.end());
+
+        write_requests_.erase(it);
+        size_t outstanding_writes = --writes_submitted_;
+        LOG(DEBUG) << "USB write: reaped, down to " << outstanding_writes;
+
+        SubmitWrites();
     }
 
-    h->bulk_in.reset(adb_open(USB_FFS_ADB_IN, O_WRONLY));
-    if (h->bulk_in < 0) {
-        PLOG(ERROR) << "cannot open bulk-in endpoint " << USB_FFS_ADB_IN;
-        goto err;
+    std::unique_ptr<IoBlock> CreateWriteBlock(Block payload, uint64_t id) {
+        auto block = std::make_unique<IoBlock>();
+        block->payload = std::move(payload);
+        block->control.aio_data = static_cast<uint64_t>(TransferId::write(id));
+        block->control.aio_rw_flags = 0;
+        block->control.aio_lio_opcode = IOCB_CMD_PWRITE;
+        block->control.aio_reqprio = 0;
+        block->control.aio_fildes = write_fd_.get();
+        block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload.data());
+        block->control.aio_nbytes = block->payload.size();
+        block->control.aio_offset = 0;
+        block->control.aio_flags = IOCB_FLAG_RESFD;
+        block->control.aio_resfd = event_fd_.get();
+        return block;
     }
 
-    h->read_aiob.fd = h->bulk_out;
-    h->write_aiob.fd = h->bulk_in;
-    h->reads_zero_packets = true;
-    return true;
+    void SubmitWrites() REQUIRES(write_mutex_) {
+        if (writes_submitted_ == kUsbWriteQueueDepth) {
+            return;
+        }
 
-err:
-    h->bulk_in.reset();
-    h->bulk_out.reset();
-    h->control.reset();
-    return false;
-}
+        ssize_t writes_to_submit = std::min(kUsbWriteQueueDepth - writes_submitted_,
+                                            write_requests_.size() - writes_submitted_);
+        CHECK_GE(writes_to_submit, 0);
+        if (writes_to_submit == 0) {
+            return;
+        }
 
-static void usb_ffs_open_thread(usb_handle *usb) {
+        struct iocb* iocbs[kUsbWriteQueueDepth];
+        for (int i = 0; i < writes_to_submit; ++i) {
+            CHECK(!write_requests_[writes_submitted_ + i]->pending);
+            write_requests_[writes_submitted_ + i]->pending = true;
+            iocbs[i] = &write_requests_[writes_submitted_ + i]->control;
+            LOG(VERBOSE) << "submitting write_request " << static_cast<void*>(iocbs[i]);
+        }
+
+        int rc = io_submit(aio_context_.get(), writes_to_submit, iocbs);
+        if (rc == -1) {
+            HandleError(StringPrintf("failed to submit write requests: %s", strerror(errno)));
+            return;
+        } else if (rc != writes_to_submit) {
+            LOG(FATAL) << "failed to submit all writes: wanted to submit " << writes_to_submit
+                       << ", actually submitted " << rc;
+        }
+
+        writes_submitted_ += rc;
+    }
+
+    void HandleError(const std::string& error) {
+        std::call_once(error_flag_, [&]() {
+            error_callback_(this, error);
+            if (!stopped_) {
+                Stop();
+            }
+        });
+    }
+
+    std::thread monitor_thread_;
+    std::thread worker_thread_;
+
+    std::atomic<bool> stopped_;
+    std::promise<void> destruction_notifier_;
+    std::once_flag error_flag_;
+
+    unique_fd event_fd_;
+
+    ScopedAioContext aio_context_;
+    unique_fd control_fd_;
+    unique_fd read_fd_;
+    unique_fd write_fd_;
+
+    std::optional<amessage> incoming_header_;
+    IOVector incoming_payload_;
+
+    std::array<IoBlock, kUsbReadQueueDepth> read_requests_;
+    IOVector read_data_;
+
+    // ID of the next request that we're going to send out.
+    size_t next_read_id_ = 0;
+
+    // ID of the next packet we're waiting for.
+    size_t needed_read_id_ = 0;
+
+    std::mutex write_mutex_;
+    std::deque<std::unique_ptr<IoBlock>> write_requests_ GUARDED_BY(write_mutex_);
+    size_t next_write_id_ GUARDED_BY(write_mutex_) = 0;
+    size_t writes_submitted_ GUARDED_BY(write_mutex_) = 0;
+};
+
+static void usb_ffs_open_thread() {
     adb_thread_setname("usb ffs open");
 
     while (true) {
-        // wait until the USB device needs opening
-        std::unique_lock<std::mutex> lock(usb->lock);
-        while (!usb->open_new_connection) {
-            usb->notify.wait(lock);
-        }
-        usb->open_new_connection = false;
-        lock.unlock();
-
-        while (true) {
-            if (init_functionfs(usb)) {
-                LOG(INFO) << "functionfs successfully initialized";
-                break;
-            }
+        unique_fd control;
+        unique_fd bulk_out;
+        unique_fd bulk_in;
+        if (!open_functionfs(&control, &bulk_out, &bulk_in)) {
             std::this_thread::sleep_for(1s);
-        }
-
-        LOG(INFO) << "registering usb transport";
-        register_usb_transport(usb, nullptr, nullptr, 1);
-    }
-
-    // never gets here
-    abort();
-}
-
-static int usb_ffs_write(usb_handle* h, const void* data, int len) {
-    D("about to write (fd=%d, len=%d)", h->bulk_in.get(), len);
-
-    const char* buf = static_cast<const char*>(data);
-    int orig_len = len;
-    while (len > 0) {
-        int write_len = std::min(USB_FFS_BULK_SIZE, len);
-        int n = adb_write(h->bulk_in, buf, write_len);
-        if (n < 0) {
-            D("ERROR: fd = %d, n = %d: %s", h->bulk_in.get(), n, strerror(errno));
-            return -1;
-        }
-        buf += n;
-        len -= n;
-    }
-
-    D("[ done fd=%d ]", h->bulk_in.get());
-    return orig_len;
-}
-
-static int usb_ffs_read(usb_handle* h, void* data, int len) {
-    D("about to read (fd=%d, len=%d)", h->bulk_out.get(), len);
-
-    char* buf = static_cast<char*>(data);
-    int orig_len = len;
-    while (len > 0) {
-        int read_len = std::min(USB_FFS_BULK_SIZE, len);
-        int n = adb_read(h->bulk_out, buf, read_len);
-        if (n < 0) {
-            D("ERROR: fd = %d, n = %d: %s", h->bulk_out.get(), n, strerror(errno));
-            return -1;
-        }
-        buf += n;
-        len -= n;
-    }
-
-    D("[ done fd=%d ]", h->bulk_out.get());
-    return orig_len;
-}
-
-static int usb_ffs_do_aio(usb_handle* h, const void* data, int len, bool read) {
-    aio_block* aiob = read ? &h->read_aiob : &h->write_aiob;
-    bool zero_packet = false;
-
-    int num_bufs = len / h->io_size + (len % h->io_size == 0 ? 0 : 1);
-    const char* cur_data = reinterpret_cast<const char*>(data);
-    int packet_size = getMaxPacketSize(aiob->fd);
-
-    if (posix_madvise(const_cast<void*>(data), len, POSIX_MADV_SEQUENTIAL | POSIX_MADV_WILLNEED) <
-        0) {
-        D("[ Failed to madvise: %d ]", errno);
-    }
-
-    for (int i = 0; i < num_bufs; i++) {
-        int buf_len = std::min(len, static_cast<int>(h->io_size));
-        io_prep(&aiob->iocb[i], aiob->fd, cur_data, buf_len, 0, read);
-
-        len -= buf_len;
-        cur_data += buf_len;
-
-        if (len == 0 && buf_len % packet_size == 0 && read) {
-            // adb does not expect the device to send a zero packet after data transfer,
-            // but the host *does* send a zero packet for the device to read.
-            zero_packet = h->reads_zero_packets;
-        }
-    }
-    if (zero_packet) {
-        io_prep(&aiob->iocb[num_bufs], aiob->fd, reinterpret_cast<const void*>(cur_data),
-                packet_size, 0, read);
-        num_bufs += 1;
-    }
-
-    while (true) {
-        if (TEMP_FAILURE_RETRY(io_submit(aiob->ctx, num_bufs, aiob->iocbs.data())) < num_bufs) {
-            PLOG(ERROR) << "aio: got error submitting " << (read ? "read" : "write");
-            return -1;
-        }
-        if (TEMP_FAILURE_RETRY(io_getevents(aiob->ctx, num_bufs, num_bufs, aiob->events.data(),
-                                            nullptr)) < num_bufs) {
-            PLOG(ERROR) << "aio: got error waiting " << (read ? "read" : "write");
-            return -1;
-        }
-        if (num_bufs == 1 && aiob->events[0].res == -EINTR) {
             continue;
         }
-        int ret = 0;
-        for (int i = 0; i < num_bufs; i++) {
-            if (aiob->events[i].res < 0) {
-                errno = -aiob->events[i].res;
-                PLOG(ERROR) << "aio: got error event on " << (read ? "read" : "write")
-                            << " total bufs " << num_bufs;
-                return -1;
-            }
-            ret += aiob->events[i].res;
-        }
-        return ret;
+
+        atransport* transport = new atransport();
+        transport->serial = "UsbFfs";
+        std::promise<void> destruction_notifier;
+        std::future<void> future = destruction_notifier.get_future();
+        transport->SetConnection(std::make_unique<UsbFfsConnection>(
+                std::move(control), std::move(bulk_out), std::move(bulk_in),
+                std::move(destruction_notifier)));
+        register_transport(transport);
+        future.wait();
     }
 }
 
-static int usb_ffs_aio_read(usb_handle* h, void* data, int len) {
-    return usb_ffs_do_aio(h, data, len, true);
-}
-
-static int usb_ffs_aio_write(usb_handle* h, const void* data, int len) {
-    return usb_ffs_do_aio(h, data, len, false);
-}
-
-static void usb_ffs_kick(usb_handle* h) {
-    int err;
-
-    err = ioctl(h->bulk_in.get(), FUNCTIONFS_CLEAR_HALT);
-    if (err < 0) {
-        D("[ kick: source (fd=%d) clear halt failed (%d) ]", h->bulk_in.get(), errno);
-    }
-
-    err = ioctl(h->bulk_out.get(), FUNCTIONFS_CLEAR_HALT);
-    if (err < 0) {
-        D("[ kick: sink (fd=%d) clear halt failed (%d) ]", h->bulk_out.get(), errno);
-    }
-
-    // don't close ep0 here, since we may not need to reinitialize it with
-    // the same descriptors again. if however ep1/ep2 fail to re-open in
-    // init_functionfs, only then would we close and open ep0 again.
-    // Ditto the comment in usb_adb_kick.
-    h->kicked = true;
-    TEMP_FAILURE_RETRY(dup2(dummy_fd.get(), h->bulk_out.get()));
-    TEMP_FAILURE_RETRY(dup2(dummy_fd.get(), h->bulk_in.get()));
-}
-
-static void usb_ffs_close(usb_handle* h) {
-    LOG(INFO) << "closing functionfs transport";
-
-    h->kicked = false;
-    h->bulk_out.reset();
-    h->bulk_in.reset();
-
-    // Notify usb_adb_open_thread to open a new connection.
-    h->lock.lock();
-    h->open_new_connection = true;
-    h->lock.unlock();
-    h->notify.notify_one();
-}
-
-usb_handle *create_usb_handle(unsigned num_bufs, unsigned io_size) {
-    usb_handle* h = new usb_handle();
-
-    if (android::base::GetBoolProperty("sys.usb.ffs.aio_compat", false)) {
-        // Devices on older kernels (< 3.18) will not have aio support for ffs
-        // unless backported. Fall back on the non-aio functions instead.
-        h->write = usb_ffs_write;
-        h->read = usb_ffs_read;
-    } else {
-        h->write = usb_ffs_aio_write;
-        h->read = usb_ffs_aio_read;
-        aio_block_init(&h->read_aiob, num_bufs);
-        aio_block_init(&h->write_aiob, num_bufs);
-    }
-    h->io_size = io_size;
-    h->kick = usb_ffs_kick;
-    h->close = usb_ffs_close;
-    return h;
-}
-
+void usb_init_legacy();
 void usb_init() {
-    D("[ usb_init - using FunctionFS ]");
-    dummy_fd.reset(adb_open("/dev/null", O_WRONLY | O_CLOEXEC));
-    CHECK_NE(-1, dummy_fd.get());
-
-    std::thread(usb_ffs_open_thread, create_usb_handle(USB_FFS_NUM_BUFS, USB_FFS_BULK_SIZE)).detach();
-}
-
-int usb_write(usb_handle* h, const void* data, int len) {
-    return h->write(h, data, len);
-}
-
-int usb_read(usb_handle* h, void* data, int len) {
-    return h->read(h, data, len);
-}
-
-int usb_close(usb_handle* h) {
-    h->close(h);
-    return 0;
-}
-
-void usb_kick(usb_handle* h) {
-    h->kick(h);
+    if (!android::base::GetBoolProperty("persist.adb.nonblocking_ffs", false)) {
+        usb_init_legacy();
+    } else {
+        std::thread(usb_ffs_open_thread).detach();
+    }
 }
diff --git a/adb/daemon/usb_ffs.cpp b/adb/daemon/usb_ffs.cpp
new file mode 100644
index 0000000..07b4ba8
--- /dev/null
+++ b/adb/daemon/usb_ffs.cpp
@@ -0,0 +1,278 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define TRACE_TAG USB
+
+#include "sysdeps.h"
+
+#include <linux/usb/ch9.h>
+#include <linux/usb/functionfs.h>
+
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+#include <android-base/unique_fd.h>
+
+#include "adb.h"
+#include "adbd/usb.h"
+
+#define MAX_PACKET_SIZE_FS 64
+#define MAX_PACKET_SIZE_HS 512
+#define MAX_PACKET_SIZE_SS 1024
+
+#define USB_FFS_BULK_SIZE 16384
+
+// Number of buffers needed to fit MAX_PAYLOAD, with an extra for ZLPs.
+#define USB_FFS_NUM_BUFS ((4 * MAX_PAYLOAD / USB_FFS_BULK_SIZE) + 1)
+
+#define cpu_to_le16(x) htole16(x)
+#define cpu_to_le32(x) htole32(x)
+
+struct func_desc {
+    struct usb_interface_descriptor intf;
+    struct usb_endpoint_descriptor_no_audio source;
+    struct usb_endpoint_descriptor_no_audio sink;
+} __attribute__((packed));
+
+struct ss_func_desc {
+    struct usb_interface_descriptor intf;
+    struct usb_endpoint_descriptor_no_audio source;
+    struct usb_ss_ep_comp_descriptor source_comp;
+    struct usb_endpoint_descriptor_no_audio sink;
+    struct usb_ss_ep_comp_descriptor sink_comp;
+} __attribute__((packed));
+
+struct desc_v1 {
+    struct usb_functionfs_descs_head_v1 {
+        __le32 magic;
+        __le32 length;
+        __le32 fs_count;
+        __le32 hs_count;
+    } __attribute__((packed)) header;
+    struct func_desc fs_descs, hs_descs;
+} __attribute__((packed));
+
+struct desc_v2 {
+    struct usb_functionfs_descs_head_v2 header;
+    // The rest of the structure depends on the flags in the header.
+    __le32 fs_count;
+    __le32 hs_count;
+    __le32 ss_count;
+    __le32 os_count;
+    struct func_desc fs_descs, hs_descs;
+    struct ss_func_desc ss_descs;
+    struct usb_os_desc_header os_header;
+    struct usb_ext_compat_desc os_desc;
+} __attribute__((packed));
+
+// clang-format off
+static struct func_desc fs_descriptors = {
+    .intf = {
+        .bLength = sizeof(fs_descriptors.intf),
+        .bDescriptorType = USB_DT_INTERFACE,
+        .bInterfaceNumber = 0,
+        .bNumEndpoints = 2,
+        .bInterfaceClass = ADB_CLASS,
+        .bInterfaceSubClass = ADB_SUBCLASS,
+        .bInterfaceProtocol = ADB_PROTOCOL,
+        .iInterface = 1, /* first string from the provided table */
+    },
+    .source = {
+        .bLength = sizeof(fs_descriptors.source),
+        .bDescriptorType = USB_DT_ENDPOINT,
+        .bEndpointAddress = 1 | USB_DIR_OUT,
+        .bmAttributes = USB_ENDPOINT_XFER_BULK,
+        .wMaxPacketSize = MAX_PACKET_SIZE_FS,
+    },
+    .sink = {
+        .bLength = sizeof(fs_descriptors.sink),
+        .bDescriptorType = USB_DT_ENDPOINT,
+        .bEndpointAddress = 2 | USB_DIR_IN,
+        .bmAttributes = USB_ENDPOINT_XFER_BULK,
+        .wMaxPacketSize = MAX_PACKET_SIZE_FS,
+    },
+};
+
+static struct func_desc hs_descriptors = {
+    .intf = {
+        .bLength = sizeof(hs_descriptors.intf),
+        .bDescriptorType = USB_DT_INTERFACE,
+        .bInterfaceNumber = 0,
+        .bNumEndpoints = 2,
+        .bInterfaceClass = ADB_CLASS,
+        .bInterfaceSubClass = ADB_SUBCLASS,
+        .bInterfaceProtocol = ADB_PROTOCOL,
+        .iInterface = 1, /* first string from the provided table */
+    },
+    .source = {
+        .bLength = sizeof(hs_descriptors.source),
+        .bDescriptorType = USB_DT_ENDPOINT,
+        .bEndpointAddress = 1 | USB_DIR_OUT,
+        .bmAttributes = USB_ENDPOINT_XFER_BULK,
+        .wMaxPacketSize = MAX_PACKET_SIZE_HS,
+    },
+    .sink = {
+        .bLength = sizeof(hs_descriptors.sink),
+        .bDescriptorType = USB_DT_ENDPOINT,
+        .bEndpointAddress = 2 | USB_DIR_IN,
+        .bmAttributes = USB_ENDPOINT_XFER_BULK,
+        .wMaxPacketSize = MAX_PACKET_SIZE_HS,
+    },
+};
+
+static struct ss_func_desc ss_descriptors = {
+    .intf = {
+        .bLength = sizeof(ss_descriptors.intf),
+        .bDescriptorType = USB_DT_INTERFACE,
+        .bInterfaceNumber = 0,
+        .bNumEndpoints = 2,
+        .bInterfaceClass = ADB_CLASS,
+        .bInterfaceSubClass = ADB_SUBCLASS,
+        .bInterfaceProtocol = ADB_PROTOCOL,
+        .iInterface = 1, /* first string from the provided table */
+    },
+    .source = {
+        .bLength = sizeof(ss_descriptors.source),
+        .bDescriptorType = USB_DT_ENDPOINT,
+        .bEndpointAddress = 1 | USB_DIR_OUT,
+        .bmAttributes = USB_ENDPOINT_XFER_BULK,
+        .wMaxPacketSize = MAX_PACKET_SIZE_SS,
+    },
+    .source_comp = {
+        .bLength = sizeof(ss_descriptors.source_comp),
+        .bDescriptorType = USB_DT_SS_ENDPOINT_COMP,
+        .bMaxBurst = 4,
+    },
+    .sink = {
+        .bLength = sizeof(ss_descriptors.sink),
+        .bDescriptorType = USB_DT_ENDPOINT,
+        .bEndpointAddress = 2 | USB_DIR_IN,
+        .bmAttributes = USB_ENDPOINT_XFER_BULK,
+        .wMaxPacketSize = MAX_PACKET_SIZE_SS,
+    },
+    .sink_comp = {
+        .bLength = sizeof(ss_descriptors.sink_comp),
+        .bDescriptorType = USB_DT_SS_ENDPOINT_COMP,
+        .bMaxBurst = 4,
+    },
+};
+
+struct usb_ext_compat_desc os_desc_compat = {
+    .bFirstInterfaceNumber = 0,
+    .Reserved1 = cpu_to_le32(1),
+    .CompatibleID = {0},
+    .SubCompatibleID = {0},
+    .Reserved2 = {0},
+};
+
+static struct usb_os_desc_header os_desc_header = {
+    .interface = cpu_to_le32(1),
+    .dwLength = cpu_to_le32(sizeof(os_desc_header) + sizeof(os_desc_compat)),
+    .bcdVersion = cpu_to_le32(1),
+    .wIndex = cpu_to_le32(4),
+    .bCount = cpu_to_le32(1),
+    .Reserved = cpu_to_le32(0),
+};
+
+#define STR_INTERFACE_ "ADB Interface"
+
+static const struct {
+    struct usb_functionfs_strings_head header;
+    struct {
+        __le16 code;
+        const char str1[sizeof(STR_INTERFACE_)];
+    } __attribute__((packed)) lang0;
+} __attribute__((packed)) strings = {
+    .header = {
+        .magic = cpu_to_le32(FUNCTIONFS_STRINGS_MAGIC),
+        .length = cpu_to_le32(sizeof(strings)),
+        .str_count = cpu_to_le32(1),
+        .lang_count = cpu_to_le32(1),
+    },
+    .lang0 = {
+        cpu_to_le16(0x0409), /* en-us */
+        STR_INTERFACE_,
+    },
+};
+// clang-format on
+
+bool open_functionfs(android::base::unique_fd* out_control, android::base::unique_fd* out_bulk_out,
+                     android::base::unique_fd* out_bulk_in) {
+    unique_fd control, bulk_out, bulk_in;
+    struct desc_v1 v1_descriptor = {};
+    struct desc_v2 v2_descriptor = {};
+
+    v2_descriptor.header.magic = cpu_to_le32(FUNCTIONFS_DESCRIPTORS_MAGIC_V2);
+    v2_descriptor.header.length = cpu_to_le32(sizeof(v2_descriptor));
+    v2_descriptor.header.flags = FUNCTIONFS_HAS_FS_DESC | FUNCTIONFS_HAS_HS_DESC |
+                                 FUNCTIONFS_HAS_SS_DESC | FUNCTIONFS_HAS_MS_OS_DESC;
+    v2_descriptor.fs_count = 3;
+    v2_descriptor.hs_count = 3;
+    v2_descriptor.ss_count = 5;
+    v2_descriptor.os_count = 1;
+    v2_descriptor.fs_descs = fs_descriptors;
+    v2_descriptor.hs_descs = hs_descriptors;
+    v2_descriptor.ss_descs = ss_descriptors;
+    v2_descriptor.os_header = os_desc_header;
+    v2_descriptor.os_desc = os_desc_compat;
+
+    if (out_control->get() < 0) {  // might have already done this before
+        LOG(INFO) << "opening control endpoint " << USB_FFS_ADB_EP0;
+        control.reset(adb_open(USB_FFS_ADB_EP0, O_RDWR));
+        if (control < 0) {
+            PLOG(ERROR) << "cannot open control endpoint " << USB_FFS_ADB_EP0;
+            return false;
+        }
+
+        if (adb_write(control.get(), &v2_descriptor, sizeof(v2_descriptor)) < 0) {
+            D("[ %s: Switching to V1_descriptor format errno=%s ]", USB_FFS_ADB_EP0,
+              strerror(errno));
+            v1_descriptor.header.magic = cpu_to_le32(FUNCTIONFS_DESCRIPTORS_MAGIC);
+            v1_descriptor.header.length = cpu_to_le32(sizeof(v1_descriptor));
+            v1_descriptor.header.fs_count = 3;
+            v1_descriptor.header.hs_count = 3;
+            v1_descriptor.fs_descs = fs_descriptors;
+            v1_descriptor.hs_descs = hs_descriptors;
+            if (adb_write(control.get(), &v1_descriptor, sizeof(v1_descriptor)) < 0) {
+                PLOG(ERROR) << "failed to write USB descriptors";
+                return false;
+            }
+        }
+
+        if (adb_write(control.get(), &strings, sizeof(strings)) < 0) {
+            PLOG(ERROR) << "failed to write USB strings";
+            return false;
+        }
+        // Signal only when writing the descriptors to ffs
+        android::base::SetProperty("sys.usb.ffs.ready", "1");
+    }
+
+    bulk_out.reset(adb_open(USB_FFS_ADB_OUT, O_RDONLY));
+    if (bulk_out < 0) {
+        PLOG(ERROR) << "cannot open bulk-out endpoint " << USB_FFS_ADB_OUT;
+        return false;
+    }
+
+    bulk_in.reset(adb_open(USB_FFS_ADB_IN, O_WRONLY));
+    if (bulk_in < 0) {
+        PLOG(ERROR) << "cannot open bulk-in endpoint " << USB_FFS_ADB_IN;
+        return false;
+    }
+
+    *out_control = std::move(control);
+    *out_bulk_in = std::move(bulk_in);
+    *out_bulk_out = std::move(bulk_out);
+    return true;
+}
diff --git a/adb/daemon/usb_legacy.cpp b/adb/daemon/usb_legacy.cpp
new file mode 100644
index 0000000..7ace59d
--- /dev/null
+++ b/adb/daemon/usb_legacy.cpp
@@ -0,0 +1,312 @@
+/*
+ * Copyright (C) 2007 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define TRACE_TAG USB
+
+#include "sysdeps.h"
+
+#include <dirent.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <linux/usb/ch9.h>
+#include <linux/usb/functionfs.h>
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <mutex>
+#include <thread>
+
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+
+#include "adb.h"
+#include "adbd/usb.h"
+#include "transport.h"
+
+using namespace std::chrono_literals;
+
+#define MAX_PACKET_SIZE_FS 64
+#define MAX_PACKET_SIZE_HS 512
+#define MAX_PACKET_SIZE_SS 1024
+
+#define USB_FFS_BULK_SIZE 16384
+
+// Number of buffers needed to fit MAX_PAYLOAD, with an extra for ZLPs.
+#define USB_FFS_NUM_BUFS ((4 * MAX_PAYLOAD / USB_FFS_BULK_SIZE) + 1)
+
+static unique_fd& dummy_fd = *new unique_fd();
+
+static void aio_block_init(aio_block* aiob, unsigned num_bufs) {
+    aiob->iocb.resize(num_bufs);
+    aiob->iocbs.resize(num_bufs);
+    aiob->events.resize(num_bufs);
+    aiob->num_submitted = 0;
+    for (unsigned i = 0; i < num_bufs; i++) {
+        aiob->iocbs[i] = &aiob->iocb[i];
+    }
+    memset(&aiob->ctx, 0, sizeof(aiob->ctx));
+    if (io_setup(num_bufs, &aiob->ctx)) {
+        D("[ aio: got error on io_setup (%d) ]", errno);
+    }
+}
+
+static int getMaxPacketSize(int ffs_fd) {
+    usb_endpoint_descriptor desc;
+    if (ioctl(ffs_fd, FUNCTIONFS_ENDPOINT_DESC, reinterpret_cast<unsigned long>(&desc))) {
+        D("[ could not get endpoint descriptor! (%d) ]", errno);
+        return MAX_PACKET_SIZE_HS;
+    } else {
+        return desc.wMaxPacketSize;
+    }
+}
+
+static bool init_functionfs(struct usb_handle* h) {
+    LOG(INFO) << "initializing functionfs";
+    if (!open_functionfs(&h->control, &h->bulk_out, &h->bulk_in)) {
+        return false;
+    }
+
+    h->read_aiob.fd = h->bulk_out.get();
+    h->write_aiob.fd = h->bulk_in.get();
+    h->reads_zero_packets = true;
+    return true;
+}
+
+static void usb_legacy_ffs_open_thread(usb_handle* usb) {
+    adb_thread_setname("usb legacy ffs open");
+
+    while (true) {
+        // wait until the USB device needs opening
+        std::unique_lock<std::mutex> lock(usb->lock);
+        while (!usb->open_new_connection) {
+            usb->notify.wait(lock);
+        }
+        usb->open_new_connection = false;
+        lock.unlock();
+
+        while (true) {
+            if (init_functionfs(usb)) {
+                LOG(INFO) << "functionfs successfully initialized";
+                break;
+            }
+            std::this_thread::sleep_for(1s);
+        }
+
+        LOG(INFO) << "registering usb transport";
+        register_usb_transport(usb, nullptr, nullptr, 1);
+    }
+
+    // never gets here
+    abort();
+}
+
+static int usb_ffs_write(usb_handle* h, const void* data, int len) {
+    D("about to write (fd=%d, len=%d)", h->bulk_in.get(), len);
+
+    const char* buf = static_cast<const char*>(data);
+    int orig_len = len;
+    while (len > 0) {
+        int write_len = std::min(USB_FFS_BULK_SIZE, len);
+        int n = adb_write(h->bulk_in, buf, write_len);
+        if (n < 0) {
+            D("ERROR: fd = %d, n = %d: %s", h->bulk_in.get(), n, strerror(errno));
+            return -1;
+        }
+        buf += n;
+        len -= n;
+    }
+
+    D("[ done fd=%d ]", h->bulk_in.get());
+    return orig_len;
+}
+
+static int usb_ffs_read(usb_handle* h, void* data, int len) {
+    D("about to read (fd=%d, len=%d)", h->bulk_out.get(), len);
+
+    char* buf = static_cast<char*>(data);
+    int orig_len = len;
+    while (len > 0) {
+        int read_len = std::min(USB_FFS_BULK_SIZE, len);
+        int n = adb_read(h->bulk_out, buf, read_len);
+        if (n < 0) {
+            D("ERROR: fd = %d, n = %d: %s", h->bulk_out.get(), n, strerror(errno));
+            return -1;
+        }
+        buf += n;
+        len -= n;
+    }
+
+    D("[ done fd=%d ]", h->bulk_out.get());
+    return orig_len;
+}
+
+static int usb_ffs_do_aio(usb_handle* h, const void* data, int len, bool read) {
+    aio_block* aiob = read ? &h->read_aiob : &h->write_aiob;
+    bool zero_packet = false;
+
+    int num_bufs = len / h->io_size + (len % h->io_size == 0 ? 0 : 1);
+    const char* cur_data = reinterpret_cast<const char*>(data);
+    int packet_size = getMaxPacketSize(aiob->fd);
+
+    if (posix_madvise(const_cast<void*>(data), len, POSIX_MADV_SEQUENTIAL | POSIX_MADV_WILLNEED) <
+        0) {
+        D("[ Failed to madvise: %d ]", errno);
+    }
+
+    for (int i = 0; i < num_bufs; i++) {
+        int buf_len = std::min(len, static_cast<int>(h->io_size));
+        io_prep(&aiob->iocb[i], aiob->fd, cur_data, buf_len, 0, read);
+
+        len -= buf_len;
+        cur_data += buf_len;
+
+        if (len == 0 && buf_len % packet_size == 0 && read) {
+            // adb does not expect the device to send a zero packet after data transfer,
+            // but the host *does* send a zero packet for the device to read.
+            zero_packet = h->reads_zero_packets;
+        }
+    }
+    if (zero_packet) {
+        io_prep(&aiob->iocb[num_bufs], aiob->fd, reinterpret_cast<const void*>(cur_data),
+                packet_size, 0, read);
+        num_bufs += 1;
+    }
+
+    while (true) {
+        if (TEMP_FAILURE_RETRY(io_submit(aiob->ctx, num_bufs, aiob->iocbs.data())) < num_bufs) {
+            PLOG(ERROR) << "aio: got error submitting " << (read ? "read" : "write");
+            return -1;
+        }
+        if (TEMP_FAILURE_RETRY(io_getevents(aiob->ctx, num_bufs, num_bufs, aiob->events.data(),
+                                            nullptr)) < num_bufs) {
+            PLOG(ERROR) << "aio: got error waiting " << (read ? "read" : "write");
+            return -1;
+        }
+        if (num_bufs == 1 && aiob->events[0].res == -EINTR) {
+            continue;
+        }
+        int ret = 0;
+        for (int i = 0; i < num_bufs; i++) {
+            if (aiob->events[i].res < 0) {
+                errno = -aiob->events[i].res;
+                PLOG(ERROR) << "aio: got error event on " << (read ? "read" : "write")
+                            << " total bufs " << num_bufs;
+                return -1;
+            }
+            ret += aiob->events[i].res;
+        }
+        return ret;
+    }
+}
+
+static int usb_ffs_aio_read(usb_handle* h, void* data, int len) {
+    return usb_ffs_do_aio(h, data, len, true);
+}
+
+static int usb_ffs_aio_write(usb_handle* h, const void* data, int len) {
+    return usb_ffs_do_aio(h, data, len, false);
+}
+
+static void usb_ffs_kick(usb_handle* h) {
+    int err;
+
+    err = ioctl(h->bulk_in.get(), FUNCTIONFS_CLEAR_HALT);
+    if (err < 0) {
+        D("[ kick: source (fd=%d) clear halt failed (%d) ]", h->bulk_in.get(), errno);
+    }
+
+    err = ioctl(h->bulk_out.get(), FUNCTIONFS_CLEAR_HALT);
+    if (err < 0) {
+        D("[ kick: sink (fd=%d) clear halt failed (%d) ]", h->bulk_out.get(), errno);
+    }
+
+    // don't close ep0 here, since we may not need to reinitialize it with
+    // the same descriptors again. if however ep1/ep2 fail to re-open in
+    // init_functionfs, only then would we close and open ep0 again.
+    // Ditto the comment in usb_adb_kick.
+    h->kicked = true;
+    TEMP_FAILURE_RETRY(dup2(dummy_fd.get(), h->bulk_out.get()));
+    TEMP_FAILURE_RETRY(dup2(dummy_fd.get(), h->bulk_in.get()));
+}
+
+static void usb_ffs_close(usb_handle* h) {
+    LOG(INFO) << "closing functionfs transport";
+
+    h->kicked = false;
+    h->bulk_out.reset();
+    h->bulk_in.reset();
+
+    // Notify usb_adb_open_thread to open a new connection.
+    h->lock.lock();
+    h->open_new_connection = true;
+    h->lock.unlock();
+    h->notify.notify_one();
+}
+
+usb_handle* create_usb_handle(unsigned num_bufs, unsigned io_size) {
+    usb_handle* h = new usb_handle();
+
+    if (android::base::GetBoolProperty("sys.usb.ffs.aio_compat", false)) {
+        // Devices on older kernels (< 3.18) will not have aio support for ffs
+        // unless backported. Fall back on the non-aio functions instead.
+        h->write = usb_ffs_write;
+        h->read = usb_ffs_read;
+    } else {
+        h->write = usb_ffs_aio_write;
+        h->read = usb_ffs_aio_read;
+        aio_block_init(&h->read_aiob, num_bufs);
+        aio_block_init(&h->write_aiob, num_bufs);
+    }
+    h->io_size = io_size;
+    h->kick = usb_ffs_kick;
+    h->close = usb_ffs_close;
+    return h;
+}
+
+void usb_init_legacy() {
+    D("[ usb_init - using legacy FunctionFS ]");
+    dummy_fd.reset(adb_open("/dev/null", O_WRONLY | O_CLOEXEC));
+    CHECK_NE(-1, dummy_fd.get());
+
+    std::thread(usb_legacy_ffs_open_thread, create_usb_handle(USB_FFS_NUM_BUFS, USB_FFS_BULK_SIZE))
+            .detach();
+}
+
+int usb_write(usb_handle* h, const void* data, int len) {
+    return h->write(h, data, len);
+}
+
+int usb_read(usb_handle* h, void* data, int len) {
+    return h->read(h, data, len);
+}
+
+int usb_close(usb_handle* h) {
+    h->close(h);
+    return 0;
+}
+
+void usb_kick(usb_handle* h) {
+    h->kick(h);
+}
diff --git a/adb/fdevent_test.h b/adb/fdevent_test.h
index 5a417e0..8d853c3 100644
--- a/adb/fdevent_test.h
+++ b/adb/fdevent_test.h
@@ -20,6 +20,7 @@
 #include <mutex>
 #include <thread>
 
+#include "adb_io.h"
 #include "socket.h"
 #include "sysdeps.h"
 #include "sysdeps/chrono.h"
diff --git a/adb/transport.cpp b/adb/transport.cpp
index ac4ae98..0fbeec6 100644
--- a/adb/transport.cpp
+++ b/adb/transport.cpp
@@ -52,7 +52,6 @@
 #include "fdevent.h"
 #include "sysdeps/chrono.h"
 
-static void register_transport(atransport* transport);
 static void remove_transport(atransport* transport);
 static void transport_unref(atransport* transport);
 
@@ -671,7 +670,7 @@
             return true;
         });
         t->connection()->SetErrorCallback([t](Connection*, const std::string& error) {
-            D("%s: connection terminated: %s", t->serial.c_str(), error.c_str());
+            LOG(INFO) << t->serial_name() << ": connection terminated: " << error;
             fdevent_run_on_main_thread([t]() {
                 handle_offline(t);
                 transport_unref(t);
@@ -730,7 +729,7 @@
 }
 
 /* the fdevent select pump is single threaded */
-static void register_transport(atransport* transport) {
+void register_transport(atransport* transport) {
     tmsg m;
     m.transport = transport;
     m.action = 1;
@@ -758,6 +757,7 @@
     CHECK_GT(t->ref_count, 0u);
     t->ref_count--;
     if (t->ref_count == 0) {
+        LOG(INFO) << "destroying transport " << t->serial_name();
         t->connection()->Stop();
 #if ADB_HOST
         if (t->IsTcpDevice() && !t->kicked()) {
@@ -1293,6 +1293,7 @@
     register_transport(t);
 }
 
+#if ADB_HOST
 // This should only be used for transports with connection_state == kCsNoPerm.
 void unregister_usb_transport(usb_handle* usb) {
     std::lock_guard<std::recursive_mutex> lock(transport_lock);
@@ -1304,6 +1305,7 @@
         return false;
     });
 }
+#endif
 
 bool check_header(apacket* p, atransport* t) {
     if (p->msg.magic != (p->msg.command ^ 0xffffffff)) {
diff --git a/adb/transport.h b/adb/transport.h
index f854ce5..1350e63 100644
--- a/adb/transport.h
+++ b/adb/transport.h
@@ -362,6 +362,7 @@
 void kick_all_tcp_devices();
 void kick_all_transports();
 
+void register_transport(atransport* transport);
 void register_usb_transport(usb_handle* h, const char* serial,
                             const char* devpath, unsigned writeable);
 
diff --git a/adb/transport_test.cpp b/adb/transport_test.cpp
index 8c628d8..b66f8fa 100644
--- a/adb/transport_test.cpp
+++ b/adb/transport_test.cpp
@@ -19,13 +19,16 @@
 #include <gtest/gtest.h>
 
 #include "adb.h"
+#include "fdevent_test.h"
+
+struct TransportTest : public FdeventTest {};
 
 static void DisconnectFunc(void* arg, atransport*) {
     int* count = reinterpret_cast<int*>(arg);
     ++*count;
 }
 
-TEST(transport, RunDisconnects) {
+TEST_F(TransportTest, RunDisconnects) {
     atransport t;
     // RunDisconnects() can be called with an empty atransport.
     t.RunDisconnects();
@@ -49,7 +52,7 @@
     ASSERT_EQ(0, count);
 }
 
-TEST(transport, SetFeatures) {
+TEST_F(TransportTest, SetFeatures) {
     atransport t;
     ASSERT_EQ(0U, t.features().size());
 
@@ -77,8 +80,7 @@
     ASSERT_EQ(0U, t.features().size());
 }
 
-TEST(transport, parse_banner_no_features) {
-    set_main_thread();
+TEST_F(TransportTest, parse_banner_no_features) {
     atransport t;
 
     parse_banner("host::", &t);
@@ -91,7 +93,7 @@
     ASSERT_EQ(std::string(), t.device);
 }
 
-TEST(transport, parse_banner_product_features) {
+TEST_F(TransportTest, parse_banner_product_features) {
     atransport t;
 
     const char banner[] =
@@ -107,9 +109,8 @@
     ASSERT_EQ(std::string("baz"), t.device);
 }
 
-TEST(transport, parse_banner_features) {
+TEST_F(TransportTest, parse_banner_features) {
     atransport t;
-
     const char banner[] =
         "host::ro.product.name=foo;ro.product.model=bar;ro.product.device=baz;"
         "features=woodly,doodly";
@@ -126,7 +127,7 @@
     ASSERT_EQ(std::string("baz"), t.device);
 }
 
-TEST(transport, test_matches_target) {
+TEST_F(TransportTest, test_matches_target) {
     std::string serial = "foo";
     std::string devpath = "/path/to/bar";
     std::string product = "test_product";
@@ -157,7 +158,7 @@
     }
 }
 
-TEST(transport, test_matches_target_local) {
+TEST_F(TransportTest, test_matches_target_local) {
     std::string serial = "100.100.100.100:5555";
 
     atransport t;
diff --git a/fs_mgr/liblp/Android.bp b/fs_mgr/liblp/Android.bp
index bbdec5b..4953655 100644
--- a/fs_mgr/liblp/Android.bp
+++ b/fs_mgr/liblp/Android.bp
@@ -47,6 +47,9 @@
     cppflags: [
         "-Wno-unused-parameter",
     ],
+    static_libs: [
+        "libgmock",
+    ],
     shared_libs: [
         "liblp",
         "libbase",
diff --git a/fs_mgr/liblp/builder.cpp b/fs_mgr/liblp/builder.cpp
index 80257fe..2c57a35 100644
--- a/fs_mgr/liblp/builder.cpp
+++ b/fs_mgr/liblp/builder.cpp
@@ -339,7 +339,7 @@
     return nullptr;
 }
 
-PartitionGroup* MetadataBuilder::FindGroup(const std::string& group_name) const {
+PartitionGroup* MetadataBuilder::FindGroup(const std::string& group_name) {
     for (const auto& group : groups_) {
         if (group->name() == group_name) {
             return group.get();
@@ -368,6 +368,57 @@
     }
 }
 
+void MetadataBuilder::ExtentsToFreeList(const std::vector<Interval>& extents,
+                                        std::vector<Interval>* free_regions) const {
+    // Convert the extent list into a list of gaps between the extents; i.e.,
+    // the list of ranges that are free on the disk.
+    for (size_t i = 1; i < extents.size(); i++) {
+        const Interval& previous = extents[i - 1];
+        const Interval& current = extents[i];
+
+        uint64_t aligned = AlignSector(previous.end);
+        if (aligned >= current.start) {
+            // There is no gap between these two extents, try the next one.
+            // Note that we check with >= instead of >, since alignment may
+            // bump the ending sector past the beginning of the next extent.
+            continue;
+        }
+
+        // The new interval represents the free space starting at the end of
+        // the previous interval, and ending at the start of the next interval.
+        free_regions->emplace_back(aligned, current.start);
+    }
+}
+
+auto MetadataBuilder::GetFreeRegions() const -> std::vector<Interval> {
+    std::vector<Interval> free_regions;
+
+    // Collect all extents in the partition table, then sort them by starting
+    // sector.
+    std::vector<Interval> extents;
+    for (const auto& partition : partitions_) {
+        for (const auto& extent : partition->extents()) {
+            LinearExtent* linear = extent->AsLinearExtent();
+            if (!linear) {
+                continue;
+            }
+            extents.emplace_back(linear->physical_sector(),
+                                 linear->physical_sector() + extent->num_sectors());
+        }
+    }
+
+    // Add 0-length intervals for the first and last sectors. This will cause
+    // ExtentsToFreeList() to treat the space in between as available.
+    uint64_t last_sector = geometry_.block_device_size / LP_SECTOR_SIZE;
+    extents.emplace_back(geometry_.first_logical_sector, geometry_.first_logical_sector);
+    extents.emplace_back(last_sector, last_sector);
+
+    std::sort(extents.begin(), extents.end());
+
+    ExtentsToFreeList(extents, &free_regions);
+    return free_regions;
+}
+
 bool MetadataBuilder::GrowPartition(Partition* partition, uint64_t aligned_size) {
     PartitionGroup* group = FindGroup(partition->group_name());
     CHECK(group);
@@ -389,59 +440,7 @@
     uint64_t sectors_needed = space_needed / LP_SECTOR_SIZE;
     DCHECK(sectors_needed * LP_SECTOR_SIZE == space_needed);
 
-    struct Interval {
-        uint64_t start;
-        uint64_t end;
-
-        Interval(uint64_t start, uint64_t end) : start(start), end(end) {}
-        uint64_t length() const { return end - start; }
-        bool operator<(const Interval& other) const { return start < other.start; }
-    };
-
-    // Collect all extents in the partition table, then sort them by starting
-    // sector.
-    std::vector<Interval> extents;
-    for (const auto& partition : partitions_) {
-        for (const auto& extent : partition->extents()) {
-            LinearExtent* linear = extent->AsLinearExtent();
-            if (!linear) {
-                continue;
-            }
-            extents.emplace_back(linear->physical_sector(),
-                                 linear->physical_sector() + extent->num_sectors());
-        }
-    }
-    std::sort(extents.begin(), extents.end());
-
-    // Convert the extent list into a list of gaps between the extents; i.e.,
-    // the list of ranges that are free on the disk.
-    std::vector<Interval> free_regions;
-    for (size_t i = 1; i < extents.size(); i++) {
-        const Interval& previous = extents[i - 1];
-        const Interval& current = extents[i];
-
-        uint64_t aligned = AlignSector(previous.end);
-        if (aligned >= current.start) {
-            // There is no gap between these two extents, try the next one.
-            // Note that we check with >= instead of >, since alignment may
-            // bump the ending sector past the beginning of the next extent.
-            continue;
-        }
-
-        // The new interval represents the free space starting at the end of
-        // the previous interval, and ending at the start of the next interval.
-        free_regions.emplace_back(aligned, current.start);
-    }
-
-    // Add a final interval representing the remainder of the free space.
-    uint64_t last_free_extent_start =
-            extents.empty() ? geometry_.first_logical_sector : extents.back().end;
-    last_free_extent_start = AlignSector(last_free_extent_start);
-
-    uint64_t last_sector = geometry_.block_device_size / LP_SECTOR_SIZE;
-    if (last_free_extent_start < last_sector) {
-        free_regions.emplace_back(last_free_extent_start, last_sector);
-    }
+    std::vector<Interval> free_regions = GetFreeRegions();
 
     const uint64_t sectors_per_block = geometry_.logical_block_size / LP_SECTOR_SIZE;
     CHECK_NE(sectors_per_block, 0);
@@ -566,7 +565,7 @@
     return size;
 }
 
-uint64_t MetadataBuilder::AlignSector(uint64_t sector) {
+uint64_t MetadataBuilder::AlignSector(uint64_t sector) const {
     // Note: when reading alignment info from the Kernel, we don't assume it
     // is aligned to the sector size, so we round up to the nearest sector.
     uint64_t lba = sector * LP_SECTOR_SIZE;
@@ -625,5 +624,36 @@
     return true;
 }
 
+std::vector<std::string> MetadataBuilder::ListGroups() const {
+    std::vector<std::string> names;
+    for (const auto& group : groups_) {
+        names.emplace_back(group->name());
+    }
+    return names;
+}
+
+void MetadataBuilder::RemoveGroupAndPartitions(const std::string& group_name) {
+    if (group_name == "default") {
+        // Cannot remove the default group.
+        return;
+    }
+    std::vector<std::string> partition_names;
+    for (const auto& partition : partitions_) {
+        if (partition->group_name() == group_name) {
+            partition_names.emplace_back(partition->name());
+        }
+    }
+
+    for (const auto& partition_name : partition_names) {
+        RemovePartition(partition_name);
+    }
+    for (auto iter = groups_.begin(); iter != groups_.end(); iter++) {
+        if ((*iter)->name() == group_name) {
+            groups_.erase(iter);
+            break;
+        }
+    }
+}
+
 }  // namespace fs_mgr
 }  // namespace android
diff --git a/fs_mgr/liblp/builder_test.cpp b/fs_mgr/liblp/builder_test.cpp
index f5d39a8..27ad250 100644
--- a/fs_mgr/liblp/builder_test.cpp
+++ b/fs_mgr/liblp/builder_test.cpp
@@ -14,13 +14,16 @@
  * limitations under the License.
  */
 
+#include <fs_mgr.h>
+#include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include <liblp/builder.h>
-#include "fs_mgr.h"
+
 #include "utility.h"
 
 using namespace std;
 using namespace android::fs_mgr;
+using ::testing::ElementsAre;
 
 TEST(liblp, BuildBasic) {
     unique_ptr<MetadataBuilder> builder = MetadataBuilder::New(1024 * 1024, 1024, 2);
@@ -515,3 +518,57 @@
     EXPECT_FALSE(builder->ResizePartition(partition, 32768));
     EXPECT_EQ(partition->size(), 16384);
 }
+
+constexpr unsigned long long operator"" _GiB(unsigned long long x) {  // NOLINT
+    return x << 30;
+}
+
+TEST(liblp, RemoveAndAddFirstPartition) {
+    auto builder = MetadataBuilder::New(10_GiB, 65536, 2);
+    ASSERT_NE(nullptr, builder);
+    ASSERT_TRUE(builder->AddGroup("foo_a", 5_GiB));
+    ASSERT_TRUE(builder->AddGroup("foo_b", 5_GiB));
+    android::fs_mgr::Partition* p;
+    p = builder->AddPartition("system_a", "foo_a", 0);
+    ASSERT_TRUE(p && builder->ResizePartition(p, 2_GiB));
+    p = builder->AddPartition("vendor_a", "foo_a", 0);
+    ASSERT_TRUE(p && builder->ResizePartition(p, 1_GiB));
+    p = builder->AddPartition("system_b", "foo_b", 0);
+    ASSERT_TRUE(p && builder->ResizePartition(p, 2_GiB));
+    p = builder->AddPartition("vendor_b", "foo_b", 0);
+    ASSERT_TRUE(p && builder->ResizePartition(p, 1_GiB));
+
+    builder->RemovePartition("system_a");
+    builder->RemovePartition("vendor_a");
+    p = builder->AddPartition("system_a", "foo_a", 0);
+    ASSERT_TRUE(p && builder->ResizePartition(p, 3_GiB));
+    p = builder->AddPartition("vendor_a", "foo_a", 0);
+    ASSERT_TRUE(p && builder->ResizePartition(p, 1_GiB));
+}
+
+TEST(liblp, ListGroups) {
+    BlockDeviceInfo device_info(1024 * 1024, 0, 0, 4096);
+    unique_ptr<MetadataBuilder> builder = MetadataBuilder::New(device_info, 1024, 1);
+    ASSERT_NE(builder, nullptr);
+    ASSERT_TRUE(builder->AddGroup("example", 0));
+
+    std::vector<std::string> groups = builder->ListGroups();
+    ASSERT_THAT(groups, ElementsAre("default", "example"));
+}
+
+TEST(liblp, RemoveGroupAndPartitions) {
+    BlockDeviceInfo device_info(1024 * 1024, 0, 0, 4096);
+    unique_ptr<MetadataBuilder> builder = MetadataBuilder::New(device_info, 1024, 1);
+    ASSERT_NE(builder, nullptr);
+    ASSERT_TRUE(builder->AddGroup("example", 0));
+    ASSERT_NE(builder->AddPartition("system", "default", 0), nullptr);
+    ASSERT_NE(builder->AddPartition("vendor", "example", 0), nullptr);
+
+    builder->RemoveGroupAndPartitions("example");
+    ASSERT_NE(builder->FindPartition("system"), nullptr);
+    ASSERT_EQ(builder->FindPartition("vendor"), nullptr);
+    ASSERT_THAT(builder->ListGroups(), ElementsAre("default"));
+
+    builder->RemoveGroupAndPartitions("default");
+    ASSERT_NE(builder->FindPartition("system"), nullptr);
+}
diff --git a/fs_mgr/liblp/include/liblp/builder.h b/fs_mgr/liblp/include/liblp/builder.h
index 8dbba84..7e07df4 100644
--- a/fs_mgr/liblp/include/liblp/builder.h
+++ b/fs_mgr/liblp/include/liblp/builder.h
@@ -199,6 +199,9 @@
     // Find a partition by name. If no partition is found, nullptr is returned.
     Partition* FindPartition(const std::string& name);
 
+    // Find a group by name. If no group is found, nullptr is returned.
+    PartitionGroup* FindGroup(const std::string& name);
+
     // Grow or shrink a partition to the requested size. This size will be
     // rounded UP to the nearest block (512 bytes).
     //
@@ -215,6 +218,12 @@
     uint64_t AllocatableSpace() const;
     uint64_t UsedSpace() const;
 
+    // Return a list of all group names.
+    std::vector<std::string> ListGroups() const;
+
+    // Remove all partitions belonging to a group, then remove the group.
+    void RemoveGroupAndPartitions(const std::string& group_name);
+
     bool GetBlockDeviceInfo(BlockDeviceInfo* info) const;
     bool UpdateBlockDeviceInfo(const BlockDeviceInfo& info);
 
@@ -228,10 +237,23 @@
     bool Init(const LpMetadata& metadata);
     bool GrowPartition(Partition* partition, uint64_t aligned_size);
     void ShrinkPartition(Partition* partition, uint64_t aligned_size);
-    uint64_t AlignSector(uint64_t sector);
-    PartitionGroup* FindGroup(const std::string& group_name) const;
+    uint64_t AlignSector(uint64_t sector) const;
     uint64_t TotalSizeOfGroup(PartitionGroup* group) const;
 
+    struct Interval {
+        uint64_t start;
+        uint64_t end;
+
+        Interval(uint64_t start, uint64_t end) : start(start), end(end) {}
+        uint64_t length() const { return end - start; }
+        bool operator<(const Interval& other) const {
+            return (start == other.start) ? end < other.end : start < other.start;
+        }
+    };
+    std::vector<Interval> GetFreeRegions() const;
+    void ExtentsToFreeList(const std::vector<Interval>& extents,
+                           std::vector<Interval>* free_regions) const;
+
     LpMetadataGeometry geometry_;
     LpMetadataHeader header_;
     std::vector<std::unique_ptr<Partition>> partitions_;
diff --git a/fs_mgr/tests/adb-remount-test.sh b/fs_mgr/tests/adb-remount-test.sh
new file mode 100755
index 0000000..d4ca574
--- /dev/null
+++ b/fs_mgr/tests/adb-remount-test.sh
@@ -0,0 +1,264 @@
+#! /bin/bash
+#
+# adb remount tests (overlayfs focus)
+#
+# Conditions:
+#  - Must be a userdebug build.
+#  - Must be in adb mode.
+#  - Kernel must have overlayfs enabled and patched to support override_creds.
+#  - Must have either squashfs, ext4-dedupe or right-sized partitions.
+#  - Minimum expectation system and vender are overlayfs covered partitions.
+#
+
+# Helper Variables
+
+SPACE=" "
+# A _real_ embedded tab character
+TAB="`echo | tr '\n' '\t'`"
+# A _real_ embedded escape character
+ESCAPE="`echo | tr '\n' '\033'`"
+GREEN="${ESCAPE}[38;5;40m"
+RED="${ESCAPE}[38;5;196m"
+ORANGE="${ESCAPE}[38;5;255:165:0m"
+NORMAL="${ESCAPE}[0m"
+
+# Helper functions
+
+[ "USAGE: inFastboot
+
+Returns: true if device is in fastboot mode" ]
+inFastboot() {
+  fastboot devices | grep "^${ANDROID_SERIAL}[${SPACE}${TAB}]" > /dev/null
+}
+
+[ "USAGE: inAdb
+
+Returns: true if device is in adb mode" ]
+inAdb() {
+  adb devices | grep -v 'List of devices attached' | grep "^${ANDROID_SERIAL}[${SPACE}${TAB}]" > /dev/null
+}
+
+[ "USAGE: adb_sh <commands>
+
+Returns: true if the command succeeded" ]
+adb_sh() {
+  adb shell "${@}"
+}
+
+[ "USAGE: get_property <prop>
+
+Returns the property value" ]
+get_property() {
+  adb_sh getprop ${1} 2>&1 </dev/null
+}
+
+[ "USAGE: isDebuggable
+
+Returns: true if device is (likely) a debug build" ]
+isDebuggable() {
+  if inAdb && [ 1 -ne `get_property ro.debuggable` ]; then
+    false
+  fi
+}
+
+[ "USAGE: adb_su <commands>
+
+Returns: true if the command running as root succeeded" ]
+adb_su() {
+  adb_sh su root "${@}"
+}
+
+[ "USAGE: adb_cat <file> >stdout
+
+Returns: content of file to stdout with carriage returns skipped,
+         true of the file exists" ]
+adb_cat() {
+    OUTPUT="`adb_sh cat ${1} </dev/null 2>&1`"
+    retval=${?}
+    echo "${OUTPUT}" | tr -d '\r'
+    return ${retval}
+}
+
+[ "USAGE: adb_reboot
+
+Returns: true if the reboot command succeeded" ]
+adb_reboot() {
+  adb reboot remount-test
+}
+
+[ "USAGE: adb_wait
+
+Returns: waits until the device has returned" ]
+adb_wait() {
+  adb wait-for-device
+}
+
+[ "USAGE: adb_root
+
+Returns: true if device in root state" ]
+adb_root() {
+  adb root >/dev/null </dev/null 2>&1 &&
+  sleep 1 &&
+  adb_wait &&
+  sleep 1
+}
+
+die() {
+  echo "${RED}[  FAILED  ]${NORMAL} ${@}" >&2
+  exit 1
+}
+
+[ "USAGE: EXPECT_EQ <lval> <rval> [message]
+
+Returns true if (regex) lval matches rval" ]
+EXPECT_EQ() {
+  lval="${1}"
+  rval="${2}"
+  shift 2
+  if ! ( echo X"${rval}" | grep '^X'"${lval}"'$' >/dev/null 2>/dev/null ); then
+    if [ `echo ${lval}${rval}${*} | wc -c` -gt 50 -o "${rval}" != "${rval%
+*}" ]; then
+      echo "ERROR: expected \"${lval}\"" >&2
+      echo "       got \"${rval}\"" |
+        sed ': again
+             N
+             s/\(\n\)\([^ ]\)/\1             \2/
+             t again' >&2
+      if [ -n "${*}" ] ; then
+        echo "       ${*}" >&2
+      fi
+    else
+      echo "ERROR: expected \"${lval}\" got \"${rval}\" ${*}" >&2
+    fi
+    return 1
+  fi
+  if [ -n "${*}" ] ; then
+    if [ X"${lval}" != X"${rval}" ]; then
+      if [ `echo ${lval}${rval}${*} | wc -c` -gt 60 -o "${rval}" != "${rval% *}" ]; then
+        echo "INFO: ok \"${lval}\"" >&2
+        echo "       = \"${rval}\"" |
+          sed ': again
+               N
+               s/\(\n\)\([^ ]\)/\1          \2/
+               t again' >&2
+        if [ -n "${*}" ] ; then
+          echo "      ${*}" >&2
+        fi
+      else
+        echo "INFO: ok \"${lval}\" = \"${rval}\" ${*}" >&2
+      fi
+    else
+      echo "INFO: ok \"${lval}\" ${*}" >&2
+    fi
+  fi
+  return 0
+}
+
+[ "USAGE: check_eq <lval> <rval> [message]
+
+Exits if (regex) lval mismatches rval" ]
+check_eq() {
+  left="${1}"
+  right="${2}"
+  shift 2
+  EXPECT_EQ "${left}" "${right}" ||
+    die "${@}"
+}
+
+[ "USAGE: skip_administrative_mounts
+
+Filters out all administrative (eg: sysfs) mounts" ]
+skip_administrative_mounts() {
+  grep -v -e "^\(overlay\|tmpfs\|none\|sysfs\|proc\|selinuxfs\|debugfs\|bpf\|cg2_bpf\|pstore\|tracefs\|adb\|mtp\|ptp\|devpts\|/data/media\) " -e " /\(cache\|mnt/scratch\|mnt/vendor/persist\|metadata\|data\) "
+}
+
+if [ X"-s" = X"${1}" -a -n "${2}" ]; then
+  export ANDROID_SERIAL="${2}"
+  shift 2
+fi
+
+inFastboot && die "device in fastboot mode"
+inAdb || die "device not in adb mode"
+isDebuggable || die "device not a debug build"
+
+# Do something
+adb_wait || die "wait for device failed"
+adb_sh ls -d /sys/module/overlay </dev/null || die "overlay module not present"
+adb_su ls /sys/module/overlay/parameters/override_creds </dev/null ||
+  die "overlay module can not be used on ANDROID"
+adb_root &&
+  adb_wait &&
+  D=`adb disable-verity 2>&1` ||
+    die "setup for overlay"
+echo "${D}"
+if [ X"${D}" != X"${D##*using overlayfs}" ]; then
+  echo "${GREEN}[       OK ]${NORMAL} using overlayfs" >&2
+fi
+if adb_sh ls -d /data/overlay </dev/null >/dev/null 2>&1; then
+  echo "/data/overlay setup, clearing out" >&2
+  adb_sh rm -rf /data/overlay </dev/null ||
+    die "/data/overlay removal"
+fi
+adb_sh ls -d /cache/overlay </dev/null >/dev/null 2>&1 ||
+  adb_sh ls -d /mnt/scratch/overlay </dev/null >/dev/null 2>&1 ||
+  die "overlay directory setup"
+adb_reboot &&
+  adb_wait &&
+  adb_sh df -k </dev/null | head -1 &&
+  adb_sh df -k </dev/null | grep "^overlay " &&
+  adb_sh df -k </dev/null | grep "^overlay .* /system\$" >/dev/null ||
+  echo "${ORANGE}[  WARNING ]${NORMAL} overlay takeover before remount not complete" >&2
+
+adb_root &&
+  adb_wait &&
+  adb remount &&
+  adb_sh df -k </dev/null | head -1 &&
+  adb_sh df -k </dev/null | grep "^overlay " &&
+  adb_sh df -k </dev/null | grep "^overlay .* /system\$" >/dev/null ||
+  die  "overlay takeover after remount"
+!(adb_sh grep "^overlay " /proc/mounts </dev/null | grep " overlay ro,") &&
+  !(adb_sh grep " rw," /proc/mounts </dev/null |
+  skip_administrative_mounts) ||
+    die "remount overlayfs missed a spot"
+
+adb_su "sed -n '1,/overlay \\/system/p' /proc/mounts" </dev/null |
+  skip_administrative_mounts |
+  grep -v ' \(squashfs\|ext4\|f2fs\) ' &&
+  echo "${ORANGE}[  WARNING ]${NORMAL} overlay takeover after first stage init" >&2 ||
+  echo "${GREEN}[       OK ]${NORMAL} overlay takeover in first stage init" >&2
+
+# Check something
+A="Hello World! $(date)"
+echo "${A}" | adb_sh "cat - > /system/hello"
+echo "${A}" | adb_sh "cat - > /vendor/hello"
+B="`adb_cat /system/hello`" ||
+  die "sytem hello"
+check_eq "${A}" "${B}" system before reboot
+B="`adb_cat /vendor/hello`" ||
+  die "vendor hello"
+check_eq "${A}" "${B}" vendor before reboot
+adb_reboot &&
+  adb_wait &&
+  B="`adb_cat /system/hello`" ||
+  die "re-read system hello after reboot"
+check_eq "${A}" "${B}" system after reboot
+# Only root can read vendor if sepolicy permissions are as expected
+B="`adb_cat /vendor/hello`" &&
+  die "re-read vendor hello after reboot w/o root"
+check_eq "cat: /vendor/hello: Permission denied" "${B}" vendor after reboot w/o root
+adb_root &&
+  adb_wait &&
+  B="`adb_cat /vendor/hello`" ||
+  die "re-read vendor hello after reboot"
+check_eq "${A}" "${B}" vendor after reboot
+adb remount &&
+  adb_sh rm /system/hello /vendor/hello </dev/null ||
+  die "cleanup hello"
+B="`adb_cat /system/hello`" &&
+  die "re-read system hello after rm"
+check_eq "cat: /system/hello: No such file or directory" "${B}" after flash rm
+B="`adb_cat /vendor/hello`" &&
+  die "re-read vendor hello after rm"
+check_eq "cat: /vendor/hello: No such file or directory" "${B}" after flash rm
+
+echo "${GREEN}[  PASSED  ]${NORMAL} adb remount" >&2
diff --git a/rootdir/Android.mk b/rootdir/Android.mk
index 393e204..aad00ad 100644
--- a/rootdir/Android.mk
+++ b/rootdir/Android.mk
@@ -146,13 +146,7 @@
 
 include $(BUILD_SYSTEM)/base_rules.mk
 
-# Regenerate init.environ.rc if PRODUCT_BOOTCLASSPATH has changed.
-bcp_md5 := $(word 1, $(shell echo $(PRODUCT_BOOTCLASSPATH) $(PRODUCT_SYSTEM_SERVER_CLASSPATH) | $(MD5SUM)))
-bcp_dep := $(intermediates)/$(bcp_md5).bcp.dep
-$(bcp_dep) :
-	$(hide) mkdir -p $(dir $@) && rm -rf $(dir $@)*.bcp.dep && touch $@
-
-$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/init.environ.rc.in $(bcp_dep)
+$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/init.environ.rc.in
 	@echo "Generate: $< -> $@"
 	@mkdir -p $(dir $@)
 	$(hide) sed -e 's?%BOOTCLASSPATH%?$(PRODUCT_BOOTCLASSPATH)?g' $< >$@
@@ -161,9 +155,6 @@
 	$(hide) sed -i -e 's?%EXPORT_GLOBAL_GCOV_OPTIONS%?$(EXPORT_GLOBAL_GCOV_OPTIONS)?g' $@
 	$(hide) sed -i -e 's?%EXPORT_GLOBAL_HWASAN_OPTIONS%?$(EXPORT_GLOBAL_HWASAN_OPTIONS)?g' $@
 
-bcp_md5 :=
-bcp_dep :=
-
 # Append PLATFORM_VNDK_VERSION to base name.
 define append_vndk_version
 $(strip \