Merge "Unify some text format in task_profiles.json"
diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp
index 2bf2924..16fa215 100644
--- a/adb/client/adb_install.cpp
+++ b/adb/client/adb_install.cpp
@@ -638,39 +638,43 @@
         fprintf(stdout, "Created child session ID %d.\n", session_id);
         session_ids.push_back(session_id);
 
-        struct stat sb;
-        if (stat(file, &sb) == -1) {
-            fprintf(stderr, "adb: failed to stat %s: %s\n", file, strerror(errno));
-            goto finalize_multi_package_session;
+        // Support splitAPKs by allowing the notation split1.apk:split2.apk:split3.apk as argument.
+        std::vector<std::string> splits = android::base::Split(file, ":");
+
+        for (const std::string& split : splits) {
+            struct stat sb;
+            if (stat(split.c_str(), &sb) == -1) {
+                fprintf(stderr, "adb: failed to stat %s: %s\n", split.c_str(), strerror(errno));
+                goto finalize_multi_package_session;
+            }
+
+            std::string cmd = android::base::StringPrintf(
+                    "%s install-write -S %" PRIu64 " %d %d_%s -", install_cmd.c_str(),
+                    static_cast<uint64_t>(sb.st_size), session_id, i,
+                    android::base::Basename(split).c_str());
+
+            unique_fd local_fd(adb_open(split.c_str(), O_RDONLY | O_CLOEXEC));
+            if (local_fd < 0) {
+                fprintf(stderr, "adb: failed to open %s: %s\n", split.c_str(), strerror(errno));
+                goto finalize_multi_package_session;
+            }
+
+            std::string error;
+            unique_fd remote_fd(adb_connect(cmd, &error));
+            if (remote_fd < 0) {
+                fprintf(stderr, "adb: connect error for write: %s\n", error.c_str());
+                goto finalize_multi_package_session;
+            }
+
+            copy_to_file(local_fd.get(), remote_fd.get());
+            read_status_line(remote_fd.get(), buf, sizeof(buf));
+
+            if (strncmp("Success", buf, 7)) {
+                fprintf(stderr, "adb: failed to write %s\n", split.c_str());
+                fputs(buf, stderr);
+                goto finalize_multi_package_session;
+            }
         }
-
-        std::string cmd =
-                android::base::StringPrintf("%s install-write -S %" PRIu64 " %d %d_%s -",
-                                            install_cmd.c_str(), static_cast<uint64_t>(sb.st_size),
-                                            session_id, i, android::base::Basename(file).c_str());
-
-        unique_fd local_fd(adb_open(file, O_RDONLY | O_CLOEXEC));
-        if (local_fd < 0) {
-            fprintf(stderr, "adb: failed to open %s: %s\n", file, strerror(errno));
-            goto finalize_multi_package_session;
-        }
-
-        std::string error;
-        unique_fd remote_fd(adb_connect(cmd, &error));
-        if (remote_fd < 0) {
-            fprintf(stderr, "adb: connect error for write: %s\n", error.c_str());
-            goto finalize_multi_package_session;
-        }
-
-        copy_to_file(local_fd.get(), remote_fd.get());
-        read_status_line(remote_fd.get(), buf, sizeof(buf));
-
-        if (strncmp("Success", buf, 7)) {
-            fprintf(stderr, "adb: failed to write %s\n", file);
-            fputs(buf, stderr);
-            goto finalize_multi_package_session;
-        }
-
         all_session_ids += android::base::StringPrintf(" %d", session_id);
     }
 
@@ -718,6 +722,7 @@
         fputs(buf, stderr);
     }
 
+    session_ids.push_back(parent_session_id);
     // try to abandon all remaining sessions
     for (std::size_t i = 0; i < session_ids.size(); i++) {
         service = android::base::StringPrintf("%s install-abandon %d", install_cmd.c_str(),
diff --git a/adb/daemon/usb.cpp b/adb/daemon/usb.cpp
index f0e2861..83ff221 100644
--- a/adb/daemon/usb.cpp
+++ b/adb/daemon/usb.cpp
@@ -29,6 +29,7 @@
 #include <linux/usb/functionfs.h>
 #include <sys/eventfd.h>
 
+#include <algorithm>
 #include <array>
 #include <future>
 #include <memory>
@@ -60,6 +61,7 @@
 static constexpr size_t kUsbReadSize = 16384;
 
 static constexpr size_t kUsbWriteQueueDepth = 16;
+static constexpr size_t kUsbWriteSize = 16 * PAGE_SIZE;
 
 static const char* to_string(enum usb_functionfs_event_type type) {
     switch (type) {
@@ -115,7 +117,7 @@
 struct IoBlock {
     bool pending;
     struct iocb control;
-    Block payload;
+    std::shared_ptr<Block> payload;
 
     TransferId id() const { return TransferId::from_value(control.aio_data); }
 };
@@ -207,8 +209,20 @@
         std::lock_guard<std::mutex> lock(write_mutex_);
         write_requests_.push_back(CreateWriteBlock(std::move(header), next_write_id_++));
         if (!packet->payload.empty()) {
-            write_requests_.push_back(
-                    CreateWriteBlock(std::move(packet->payload), next_write_id_++));
+            // The kernel attempts to allocate a contiguous block of memory for each write,
+            // which can fail if the write is large and the kernel heap is fragmented.
+            // Split large writes into smaller chunks to avoid this.
+            std::shared_ptr<Block> payload = std::make_shared<Block>(std::move(packet->payload));
+            size_t offset = 0;
+            size_t len = payload->size();
+
+            while (len > 0) {
+                size_t write_size = std::min(kUsbWriteSize, len);
+                write_requests_.push_back(
+                        CreateWriteBlock(payload, offset, write_size, next_write_id_++));
+                len -= write_size;
+                offset += write_size;
+            }
         }
         SubmitWrites();
         return true;
@@ -367,10 +381,10 @@
 
     void PrepareReadBlock(IoBlock* block, uint64_t id) {
         block->pending = false;
-        block->payload.resize(kUsbReadSize);
+        block->payload = std::make_shared<Block>(kUsbReadSize);
         block->control.aio_data = static_cast<uint64_t>(TransferId::read(id));
-        block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload.data());
-        block->control.aio_nbytes = block->payload.size();
+        block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload->data());
+        block->control.aio_nbytes = block->payload->size();
     }
 
     IoBlock CreateReadBlock(uint64_t id) {
@@ -421,7 +435,7 @@
         uint64_t read_idx = id.id % kUsbReadQueueDepth;
         IoBlock* block = &read_requests_[read_idx];
         block->pending = false;
-        block->payload.resize(size);
+        block->payload->resize(size);
 
         // Notification for completed reads can be received out of order.
         if (block->id().id != needed_read_id_) {
@@ -442,16 +456,16 @@
     }
 
     void ProcessRead(IoBlock* block) {
-        if (!block->payload.empty()) {
+        if (!block->payload->empty()) {
             if (!incoming_header_.has_value()) {
-                CHECK_EQ(sizeof(amessage), block->payload.size());
+                CHECK_EQ(sizeof(amessage), block->payload->size());
                 amessage msg;
-                memcpy(&msg, block->payload.data(), sizeof(amessage));
+                memcpy(&msg, block->payload->data(), sizeof(amessage));
                 LOG(DEBUG) << "USB read:" << dump_header(&msg);
                 incoming_header_ = msg;
             } else {
                 size_t bytes_left = incoming_header_->data_length - incoming_payload_.size();
-                Block payload = std::move(block->payload);
+                Block payload = std::move(*block->payload);
                 CHECK_LE(payload.size(), bytes_left);
                 incoming_payload_.append(std::make_unique<Block>(std::move(payload)));
             }
@@ -506,7 +520,8 @@
         SubmitWrites();
     }
 
-    std::unique_ptr<IoBlock> CreateWriteBlock(Block payload, uint64_t id) {
+    std::unique_ptr<IoBlock> CreateWriteBlock(std::shared_ptr<Block> payload, size_t offset,
+                                              size_t len, uint64_t id) {
         auto block = std::make_unique<IoBlock>();
         block->payload = std::move(payload);
         block->control.aio_data = static_cast<uint64_t>(TransferId::write(id));
@@ -514,14 +529,20 @@
         block->control.aio_lio_opcode = IOCB_CMD_PWRITE;
         block->control.aio_reqprio = 0;
         block->control.aio_fildes = write_fd_.get();
-        block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload.data());
-        block->control.aio_nbytes = block->payload.size();
+        block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload->data() + offset);
+        block->control.aio_nbytes = len;
         block->control.aio_offset = 0;
         block->control.aio_flags = IOCB_FLAG_RESFD;
         block->control.aio_resfd = worker_event_fd_.get();
         return block;
     }
 
+    std::unique_ptr<IoBlock> CreateWriteBlock(Block payload, uint64_t id) {
+        std::shared_ptr<Block> block = std::make_shared<Block>(std::move(payload));
+        size_t len = block->size();
+        return CreateWriteBlock(std::move(block), 0, len, id);
+    }
+
     void SubmitWrites() REQUIRES(write_mutex_) {
         if (writes_submitted_ == kUsbWriteQueueDepth) {
             return;
diff --git a/fs_mgr/Android.bp b/fs_mgr/Android.bp
index 974e13e..4ee9624 100644
--- a/fs_mgr/Android.bp
+++ b/fs_mgr/Android.bp
@@ -111,3 +111,35 @@
         "libgsi_headers",
     ],
 }
+
+cc_binary {
+    name: "remount",
+    defaults: ["fs_mgr_defaults"],
+    static_libs: [
+        "libavb_user",
+    ],
+    shared_libs: [
+        "libbootloader_message",
+        "libbase",
+        "libcrypto",
+        "libfec",
+        "libfs_mgr",
+    ],
+    header_libs: [
+        "libcutils_headers",
+    ],
+    srcs: [
+        "fs_mgr_remount.cpp",
+    ],
+    cppflags: [
+        "-DALLOW_ADBD_DISABLE_VERITY=0",
+    ],
+    product_variables: {
+        debuggable: {
+            cppflags: [
+                "-UALLOW_ADBD_DISABLE_VERITY",
+                "-DALLOW_ADBD_DISABLE_VERITY=1",
+            ],
+        },
+    },
+}
diff --git a/fs_mgr/fs_mgr.cpp b/fs_mgr/fs_mgr.cpp
index 0c904c4..272190e 100644
--- a/fs_mgr/fs_mgr.cpp
+++ b/fs_mgr/fs_mgr.cpp
@@ -1089,7 +1089,7 @@
                 // Skips mounting the device.
                 continue;
             }
-        } else if (!current_entry.avb_key.empty()) {
+        } else if (!current_entry.avb_keys.empty()) {
             if (AvbHandle::SetUpStandaloneAvbHashtree(&current_entry) == AvbHashtreeResult::kFail) {
                 LERROR << "Failed to set up AVB on standalone partition: "
                        << current_entry.mount_point << ", skipping!";
@@ -1320,7 +1320,7 @@
                 // Skips mounting the device.
                 continue;
             }
-        } else if (!fstab_entry.avb_key.empty()) {
+        } else if (!fstab_entry.avb_keys.empty()) {
             if (AvbHandle::SetUpStandaloneAvbHashtree(&fstab_entry) == AvbHashtreeResult::kFail) {
                 LERROR << "Failed to set up AVB on standalone partition: "
                        << fstab_entry.mount_point << ", skipping!";
diff --git a/fs_mgr/fs_mgr_fstab.cpp b/fs_mgr/fs_mgr_fstab.cpp
index 82d9144..5d4a3cc 100644
--- a/fs_mgr/fs_mgr_fstab.cpp
+++ b/fs_mgr/fs_mgr_fstab.cpp
@@ -325,8 +325,8 @@
             }
         } else if (StartsWith(flag, "zram_backing_dev_path=")) {
             entry->zram_backing_dev_path = arg;
-        } else if (StartsWith(flag, "avb_key=")) {
-            entry->avb_key = arg;
+        } else if (StartsWith(flag, "avb_keys=")) {
+            entry->avb_keys = arg;
         } else {
             LWARNING << "Warning: unknown flag: " << flag;
         }
@@ -759,7 +759,8 @@
                          .fs_type = "ext4",
                          .flags = MS_RDONLY,
                          .fs_options = "barrier=1",
-                         .avb_key = "/gsi.avbpubkey",
+                         // could add more keys separated by ':'.
+                         .avb_keys = "/avb/gsi.avbpubkey:",
                          .logical_partition_name = "system"};
     system.fs_mgr_flags.wait = true;
     system.fs_mgr_flags.logical = true;
diff --git a/fs_mgr/fs_mgr_remount.cpp b/fs_mgr/fs_mgr_remount.cpp
new file mode 100644
index 0000000..c0e0ccd
--- /dev/null
+++ b/fs_mgr/fs_mgr_remount.cpp
@@ -0,0 +1,348 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <errno.h>
+#include <getopt.h>
+#include <libavb_user/libavb_user.h>
+#include <stdio.h>
+#include <sys/mount.h>
+#include <sys/types.h>
+#include <sys/vfs.h>
+#include <unistd.h>
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+#include <android-base/strings.h>
+#include <bootloader_message/bootloader_message.h>
+#include <cutils/android_reboot.h>
+#include <fec/io.h>
+#include <fs_mgr_overlayfs.h>
+#include <fs_mgr_priv.h>
+#include <fstab/fstab.h>
+
+namespace {
+
+[[noreturn]] void usage(int exit_status) {
+    LOG(INFO) << getprogname()
+              << " [-h] [-R] [-T fstab_file]\n"
+                 "\t-h --help\tthis help\n"
+                 "\t-R --reboot\tdisable verity & reboot to facilitate remount\n"
+                 "\t-T --fstab\tcustom fstab file location\n"
+                 "\n"
+                 "Remount all partitions read-write.\n"
+                 "-R notwithstanding, verity must be disabled.";
+
+    ::exit(exit_status);
+}
+
+bool remountable_partition(const android::fs_mgr::FstabEntry& entry) {
+    if (entry.fs_mgr_flags.vold_managed) return false;
+    if (entry.fs_mgr_flags.recovery_only) return false;
+    if (entry.fs_mgr_flags.slot_select_other) return false;
+    if (!(entry.flags & MS_RDONLY)) return false;
+    if (entry.fs_type == "vfat") return false;
+    return true;
+}
+
+const std::string system_mount_point(const android::fs_mgr::FstabEntry& entry) {
+    if (entry.mount_point == "/") return "/system";
+    return entry.mount_point;
+}
+
+const android::fs_mgr::FstabEntry* is_wrapped(const android::fs_mgr::Fstab& overlayfs_candidates,
+                                              const android::fs_mgr::FstabEntry& entry) {
+    auto mount_point = system_mount_point(entry);
+    auto it = std::find_if(overlayfs_candidates.begin(), overlayfs_candidates.end(),
+                           [&mount_point](const auto& entry) {
+                               return android::base::StartsWith(mount_point,
+                                                                system_mount_point(entry) + "/");
+                           });
+    if (it == overlayfs_candidates.end()) return nullptr;
+    return &(*it);
+}
+
+void try_unmount_bionic(android::fs_mgr::Fstab* mounts) {
+    static constexpr const char* kBionic = "/bionic";
+
+    auto entry = GetEntryForMountPoint(mounts, kBionic);
+    if (!entry) return;
+
+    struct statfs buf;
+    if (::statfs(kBionic, &buf) == -1) {
+        PLOG(ERROR) << "statfs of " << kBionic;
+        return;
+    }
+    if (buf.f_flags & MS_RDONLY) {
+        // /bionic is on a read-only partition; can happen for
+        // non-system-as-root-devices. Don' try to unmount.
+        return;
+    }
+    fs_mgr_set_blk_ro(entry->blk_device, false);
+    if (::mount(entry->blk_device.c_str(), entry->mount_point.c_str(), entry->fs_type.c_str(),
+                MS_REMOUNT, nullptr) == -1) {
+        PLOG(ERROR) << "remount of " << kBionic;
+    }
+}
+
+void MyLogger(android::base::LogId id, android::base::LogSeverity severity, const char* tag,
+              const char* file, unsigned int line, const char* message) {
+    static const char log_characters[] = "VD\0WEFF";
+    if (severity < sizeof(log_characters)) {
+        auto severity_char = log_characters[severity];
+        if (severity_char) fprintf(stderr, "%c ", severity_char);
+    }
+    fprintf(stderr, "%s\n", message);
+
+    static auto logd = android::base::LogdLogger();
+    logd(id, severity, tag, file, line, message);
+}
+
+[[noreturn]] void reboot(bool dedupe) {
+    if (dedupe) {
+        LOG(INFO) << "The device will now reboot to recovery and attempt un-deduplication.";
+    } else {
+        LOG(INFO) << "Successfully disabled verity\nrebooting device";
+    }
+    ::sync();
+    android::base::SetProperty(ANDROID_RB_PROPERTY, dedupe ? "reboot,recovery" : "reboot,remount");
+    ::sleep(60);
+    ::exit(0);  // SUCCESS
+}
+
+}  // namespace
+
+int main(int argc, char* argv[]) {
+    android::base::InitLogging(argv, MyLogger);
+
+    enum {
+        SUCCESS,
+        NOT_USERDEBUG,
+        BADARG,
+        NOT_ROOT,
+        NO_FSTAB,
+        VERITY_PARTITION,
+        BAD_OVERLAY,
+        NO_MOUNTS,
+        REMOUNT_FAILED,
+    } retval = SUCCESS;
+
+    // If somehow this executable is delivered on a "user" build, it can
+    // not function, so providing a clear message to the caller rather than
+    // letting if fall through and provide a lot of confusing failure messages.
+    if (!ALLOW_ADBD_DISABLE_VERITY || (android::base::GetProperty("ro.debuggable", "0") != "1")) {
+        LOG(ERROR) << "only functions on userdebug or eng builds";
+        return NOT_USERDEBUG;
+    }
+
+    const char* fstab_file = nullptr;
+    auto can_reboot = false;
+
+    struct option longopts[] = {
+            {"fstab", required_argument, nullptr, 'T'},
+            {"help", no_argument, nullptr, 'h'},
+            {"reboot", no_argument, nullptr, 'R'},
+            {0, 0, nullptr, 0},
+    };
+    for (int opt; (opt = ::getopt_long(argc, argv, "hRT:", longopts, nullptr)) != -1;) {
+        switch (opt) {
+            case 'R':
+                can_reboot = true;
+                break;
+            case 'T':
+                if (fstab_file) {
+                    LOG(ERROR) << "Cannot supply two fstabs: -T " << fstab_file << " -T" << optarg;
+                    usage(BADARG);
+                }
+                fstab_file = optarg;
+                break;
+            default:
+                LOG(ERROR) << "Bad Argument -" << char(opt);
+                usage(BADARG);
+                break;
+            case 'h':
+                usage(SUCCESS);
+                break;
+        }
+    }
+
+    if (argc > optind) {
+        LOG(ERROR) << "Bad Argument " << argv[optind];
+        usage(BADARG);
+    }
+
+    // Make sure we are root.
+    if (::getuid() != 0) {
+        LOG(ERROR) << "must be run as root";
+        return NOT_ROOT;
+    }
+
+    // Read the selected fstab.
+    android::fs_mgr::Fstab fstab;
+    auto fstab_read = false;
+    if (fstab_file) {
+        fstab_read = android::fs_mgr::ReadFstabFromFile(fstab_file, &fstab);
+    } else {
+        fstab_read = android::fs_mgr::ReadDefaultFstab(&fstab);
+    }
+    if (!fstab_read || fstab.empty()) {
+        PLOG(ERROR) << "Failed to read fstab";
+        return NO_FSTAB;
+    }
+
+    // Generate the list of supported overlayfs mount points.
+    auto overlayfs_candidates = fs_mgr_overlayfs_candidate_list(fstab);
+
+    // Generate the all remountable partitions sub-list
+    android::fs_mgr::Fstab partitions;
+    for (auto const& entry : fstab) {
+        if (!remountable_partition(entry)) continue;
+        if (overlayfs_candidates.empty() ||
+            GetEntryForMountPoint(&overlayfs_candidates, entry.mount_point) ||
+            (is_wrapped(overlayfs_candidates, entry) == nullptr)) {
+            partitions.emplace_back(entry);
+        }
+    }
+
+    // Check verity and optionally setup overlayfs backing.
+    auto reboot_later = false;
+    for (auto it = partitions.begin(); it != partitions.end();) {
+        auto& entry = *it;
+        auto& mount_point = entry.mount_point;
+        if (fs_mgr_is_verity_enabled(entry)) {
+            LOG(WARNING) << "Verity enabled on " << mount_point;
+            if (can_reboot &&
+                (android::base::GetProperty("ro.boot.vbmeta.devices_state", "") != "locked")) {
+                if (AvbOps* ops = avb_ops_user_new()) {
+                    auto ret = avb_user_verity_set(
+                            ops, android::base::GetProperty("ro.boot.slot_suffix", "").c_str(),
+                            false);
+                    avb_ops_user_free(ops);
+                    if (ret) {
+                        if (fs_mgr_overlayfs_valid() == OverlayfsValidResult::kNotSupported) {
+                            retval = VERITY_PARTITION;
+                            // w/o overlayfs available, also check for dedupe
+                            reboot_later = true;
+                            ++it;
+                            continue;
+                        }
+                        reboot(false);
+                    } else if (fs_mgr_set_blk_ro(entry.blk_device, false)) {
+                        fec::io fh(entry.blk_device.c_str(), O_RDWR);
+                        if (fh && fh.set_verity_status(false)) reboot_later = true;
+                    }
+                }
+            }
+            LOG(ERROR) << "Skipping " << mount_point;
+            retval = VERITY_PARTITION;
+            it = partitions.erase(it);
+            continue;
+        }
+
+        auto change = false;
+        errno = 0;
+        if (fs_mgr_overlayfs_setup(nullptr, mount_point.c_str(), &change)) {
+            if (change) {
+                LOG(INFO) << "Using overlayfs for " << mount_point;
+            }
+        } else if (errno) {
+            PLOG(ERROR) << "Overlayfs setup for " << mount_point << " failed, skipping";
+            retval = BAD_OVERLAY;
+            it = partitions.erase(it);
+            continue;
+        }
+        ++it;
+    }
+
+    if (partitions.empty()) {
+        if (reboot_later) reboot(false);
+        LOG(WARNING) << "No partitions to remount";
+        return retval;
+    }
+
+    // Mount overlayfs.
+    if (!fs_mgr_overlayfs_mount_all(&partitions)) {
+        retval = BAD_OVERLAY;
+        PLOG(ERROR) << "Can not mount overlayfs for partitions";
+    }
+
+    // Get actual mounts _after_ overlayfs has been added.
+    android::fs_mgr::Fstab mounts;
+    if (!android::fs_mgr::ReadFstabFromFile("/proc/mounts", &mounts) || mounts.empty()) {
+        PLOG(ERROR) << "Failed to read /proc/mounts";
+        retval = NO_MOUNTS;
+    }
+
+    // Remount selected partitions.
+    for (auto& entry : partitions) {
+        // unlock the r/o key for the mount point device
+        if (entry.fs_mgr_flags.logical) {
+            fs_mgr_update_logical_partition(&entry);
+        }
+        auto blk_device = entry.blk_device;
+        auto mount_point = entry.mount_point;
+
+        for (auto it = mounts.rbegin(); it != mounts.rend(); ++it) {
+            auto& rentry = *it;
+            if (mount_point == rentry.mount_point) {
+                blk_device = rentry.blk_device;
+                break;
+            }
+            if ((mount_point == "/") && (rentry.mount_point == "/system")) {
+                if (blk_device != "/dev/root") blk_device = rentry.blk_device;
+                mount_point = "/system";
+                break;
+            }
+        }
+        fs_mgr_set_blk_ro(blk_device, false);
+
+        // Now remount!
+        if (::mount(blk_device.c_str(), mount_point.c_str(), entry.fs_type.c_str(), MS_REMOUNT,
+                    nullptr) == 0) {
+            continue;
+        }
+        if ((errno == EINVAL) && (mount_point != entry.mount_point)) {
+            mount_point = entry.mount_point;
+            if (::mount(blk_device.c_str(), mount_point.c_str(), entry.fs_type.c_str(), MS_REMOUNT,
+                        nullptr) == 0) {
+                continue;
+            }
+        }
+        PLOG(WARNING) << "failed to remount partition dev:" << blk_device << " mnt:" << mount_point;
+        // If errno = EROFS at this point, we are dealing with r/o
+        // filesystem types like squashfs, erofs or ext4 dedupe. We will
+        // consider such a device that does not have CONFIG_OVERLAY_FS
+        // in the kernel as a misconfigured; except for ext4 dedupe.
+        if ((errno == EROFS) && can_reboot) {
+            const std::vector<std::string> msg = {"--fsck_unshare_blocks"};
+            std::string err;
+            if (write_bootloader_message(msg, &err)) reboot(true);
+            LOG(ERROR) << "Failed to set bootloader message: " << err;
+            errno = EROFS;
+        }
+        retval = REMOUNT_FAILED;
+    }
+
+    if (reboot_later) reboot(false);
+
+    try_unmount_bionic(&mounts);
+
+    return retval;
+}
diff --git a/fs_mgr/include_fstab/fstab/fstab.h b/fs_mgr/include_fstab/fstab/fstab.h
index a3d9fdd..e811447 100644
--- a/fs_mgr/include_fstab/fstab/fstab.h
+++ b/fs_mgr/include_fstab/fstab/fstab.h
@@ -55,7 +55,7 @@
     std::string zram_loopback_path;
     uint64_t zram_loopback_size = 512 * 1024 * 1024;  // 512MB by default;
     std::string zram_backing_dev_path;
-    std::string avb_key;
+    std::string avb_keys;
 
     struct FsMgrFlags {
         bool wait : 1;
diff --git a/fs_mgr/libfiemap_writer/fiemap_writer.cpp b/fs_mgr/libfiemap_writer/fiemap_writer.cpp
index 5b8a9c2..6b742e6 100644
--- a/fs_mgr/libfiemap_writer/fiemap_writer.cpp
+++ b/fs_mgr/libfiemap_writer/fiemap_writer.cpp
@@ -126,7 +126,8 @@
     return DeviceMapperStackPop(bdev_next, bdev_raw);
 }
 
-static bool FileToBlockDevicePath(const std::string& file_path, std::string* bdev_path) {
+bool FiemapWriter::GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path,
+                                         bool* uses_dm) {
     struct stat sb;
     if (stat(file_path.c_str(), &sb)) {
         PLOG(ERROR) << "Failed to get stat for: " << file_path;
@@ -146,6 +147,10 @@
         return false;
     }
 
+    if (uses_dm) {
+        *uses_dm = (bdev_raw != bdev);
+    }
+
     LOG(DEBUG) << "Popped device (" << bdev_raw << ") from device mapper stack starting with ("
                << bdev << ")";
 
@@ -458,7 +463,7 @@
     }
 
     std::string bdev_path;
-    if (!FileToBlockDevicePath(abs_path, &bdev_path)) {
+    if (!GetBlockDeviceForFile(abs_path, &bdev_path)) {
         LOG(ERROR) << "Failed to get block dev path for file: " << file_path;
         cleanup(abs_path, create);
         return nullptr;
diff --git a/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h b/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h
index edbae77..b5472c9 100644
--- a/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h
+++ b/fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h
@@ -57,6 +57,13 @@
     // FiemapWriter::Open).
     static bool HasPinnedExtents(const std::string& file_path);
 
+    // Returns the underlying block device of a file. This will look past device-mapper layers.
+    // If an intermediate device-mapper layer would not maintain a 1:1 mapping (i.e. is a non-
+    // trivial dm-linear), then this will fail. If device-mapper nodes are encountered, then
+    // |uses_dm| will be set to true.
+    static bool GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path,
+                                      bool* uses_dm = nullptr);
+
     // The counter part of Write(). It is an error for the offset to be unaligned with
     // the block device's block size.
     // In case of error, the contents of buffer MUST be discarded.
diff --git a/fs_mgr/libfs_avb/avb_util.cpp b/fs_mgr/libfs_avb/avb_util.cpp
index 80fa5c4..7d89902 100644
--- a/fs_mgr/libfs_avb/avb_util.cpp
+++ b/fs_mgr/libfs_avb/avb_util.cpp
@@ -28,6 +28,7 @@
 #include "util.h"
 
 using android::base::Basename;
+using android::base::ReadFileToString;
 using android::base::StartsWith;
 using android::base::unique_fd;
 
@@ -311,7 +312,8 @@
     return footer;
 }
 
-bool VerifyPublicKeyBlob(const uint8_t* key, size_t length, const std::string& expected_key_blob) {
+bool ValidatePublicKeyBlob(const uint8_t* key, size_t length,
+                           const std::string& expected_key_blob) {
     if (expected_key_blob.empty()) {  // no expectation of the key, return true.
         return true;
     }
@@ -324,6 +326,21 @@
     return false;
 }
 
+bool ValidatePublicKeyBlob(const std::string key_blob_to_validate,
+                           const std::vector<std::string>& allowed_key_paths) {
+    std::string allowed_key_blob;
+    if (key_blob_to_validate.empty()) {
+        LWARNING << "Failed to validate an empty key";
+        return false;
+    }
+    for (const auto& path : allowed_key_paths) {
+        if (ReadFileToString(path, &allowed_key_blob)) {
+            if (key_blob_to_validate == allowed_key_blob) return true;
+        }
+    }
+    return false;
+}
+
 VBMetaVerifyResult VerifyVBMetaSignature(const VBMetaData& vbmeta,
                                          const std::string& expected_public_key_blob,
                                          std::string* out_public_key_data) {
@@ -347,7 +364,7 @@
                        << ": Error verifying vbmeta image: failed to get public key";
                 return VBMetaVerifyResult::kError;
             }
-            if (!VerifyPublicKeyBlob(pk_data, pk_len, expected_public_key_blob)) {
+            if (!ValidatePublicKeyBlob(pk_data, pk_len, expected_public_key_blob)) {
                 LERROR << vbmeta.partition() << ": Error verifying vbmeta image: public key used to"
                        << " sign data does not match key in chain descriptor";
                 return VBMetaVerifyResult::kErrorVerification;
diff --git a/fs_mgr/libfs_avb/avb_util.h b/fs_mgr/libfs_avb/avb_util.h
index 5f413e3..986a69a 100644
--- a/fs_mgr/libfs_avb/avb_util.h
+++ b/fs_mgr/libfs_avb/avb_util.h
@@ -78,7 +78,10 @@
                                          const std::string& expected_public_key_blob,
                                          std::string* out_public_key_data);
 
-bool VerifyPublicKeyBlob(const uint8_t* key, size_t length, const std::string& expected_key_blob);
+bool ValidatePublicKeyBlob(const uint8_t* key, size_t length, const std::string& expected_key_blob);
+
+bool ValidatePublicKeyBlob(const std::string key_blob_to_validate,
+                           const std::vector<std::string>& expected_key_paths);
 
 // Detects if whether a partition contains a rollback image.
 bool RollbackDetected(const std::string& partition_name, uint64_t rollback_index);
diff --git a/fs_mgr/libfs_avb/fs_avb.cpp b/fs_mgr/libfs_avb/fs_avb.cpp
index 02902f0..f0767dc 100644
--- a/fs_mgr/libfs_avb/fs_avb.cpp
+++ b/fs_mgr/libfs_avb/fs_avb.cpp
@@ -29,6 +29,7 @@
 #include <android-base/file.h>
 #include <android-base/parseint.h>
 #include <android-base/stringprintf.h>
+#include <android-base/strings.h>
 #include <libavb/libavb.h>
 #include <libdm/dm.h>
 
@@ -40,6 +41,7 @@
 using android::base::Basename;
 using android::base::ParseUint;
 using android::base::ReadFileToString;
+using android::base::Split;
 using android::base::StringPrintf;
 
 namespace android {
@@ -264,8 +266,8 @@
 }
 
 AvbUniquePtr AvbHandle::LoadAndVerifyVbmeta(const FstabEntry& fstab_entry) {
-    if (fstab_entry.avb_key.empty()) {
-        LERROR << "avb_key=/path/to/key is missing for " << fstab_entry.mount_point;
+    if (fstab_entry.avb_keys.empty()) {
+        LERROR << "avb_keys=/path/to/key(s) is missing for " << fstab_entry.mount_point;
         return nullptr;
     }
 
@@ -273,24 +275,14 @@
     bool allow_verification_error = IsDeviceUnlocked();
     bool rollback_protection = !allow_verification_error;
 
-    std::string expected_key_blob;
-    if (!ReadFileToString(fstab_entry.avb_key, &expected_key_blob)) {
-        if (!allow_verification_error) {
-            LERROR << "Failed to load avb_key: " << fstab_entry.avb_key
-                   << " for mount point: " << fstab_entry.mount_point;
-            return nullptr;
-        }
-        LWARNING << "Allowing no expected key blob when verification error is permitted";
-        expected_key_blob.clear();
-    }
-
+    std::string public_key_data;
     bool verification_disabled = false;
     VBMetaVerifyResult verify_result = VBMetaVerifyResult::kError;
     std::unique_ptr<VBMetaData> vbmeta = LoadAndVerifyVbmetaByPath(
             fstab_entry.blk_device, "" /* partition_name, no need for a standalone path */,
-            expected_key_blob, allow_verification_error, rollback_protection,
-            false /* not is_chained_vbmeta */, nullptr /* out_public_key_data */,
-            &verification_disabled, &verify_result);
+            "" /* expected_public_key_blob, */, allow_verification_error, rollback_protection,
+            false /* not is_chained_vbmeta */, &public_key_data, &verification_disabled,
+            &verify_result);
 
     if (!vbmeta) {
         LERROR << "Failed to load vbmeta: " << fstab_entry.blk_device;
@@ -316,6 +308,15 @@
             return nullptr;
     }
 
+    if (!ValidatePublicKeyBlob(public_key_data, Split(fstab_entry.avb_keys, ":"))) {
+        avb_handle->status_ = AvbHandleStatus::kVerificationError;
+        LWARNING << "Found unknown public key used to sign " << fstab_entry.mount_point;
+        if (!allow_verification_error) {
+            LERROR << "Unknown public key is not allowed";
+            return nullptr;
+        }
+    }
+
     if (verification_disabled) {
         LINFO << "AVB verification disabled on: " << fstab_entry.mount_point;
         avb_handle->status_ = AvbHandleStatus::kVerificationDisabled;
diff --git a/fs_mgr/libfs_avb/tests/avb_util_test.cpp b/fs_mgr/libfs_avb/tests/avb_util_test.cpp
index e4213b7..0d342d3 100644
--- a/fs_mgr/libfs_avb/tests/avb_util_test.cpp
+++ b/fs_mgr/libfs_avb/tests/avb_util_test.cpp
@@ -35,9 +35,9 @@
 using android::fs_mgr::GetTotalSize;
 using android::fs_mgr::LoadAndVerifyVbmetaByPartition;
 using android::fs_mgr::LoadAndVerifyVbmetaByPath;
+using android::fs_mgr::ValidatePublicKeyBlob;
 using android::fs_mgr::VBMetaData;
 using android::fs_mgr::VBMetaVerifyResult;
-using android::fs_mgr::VerifyPublicKeyBlob;
 using android::fs_mgr::VerifyVBMetaData;
 using android::fs_mgr::VerifyVBMetaSignature;
 
@@ -415,7 +415,7 @@
     EXPECT_EQ(content_padding.size() - padding.size(), vbmeta_padding.size());
 }
 
-TEST_F(AvbUtilTest, VerifyPublicKeyBlob) {
+TEST_F(AvbUtilTest, ValidatePublicKeyBlob) {
     // Generates a raw key.bin
     const size_t key_size = 2048;
     base::FilePath key_path = GenerateImage("key.bin", key_size);
@@ -425,12 +425,12 @@
 
     std::string expected_key_blob;
     EXPECT_TRUE(base::ReadFileToString(key_path, &expected_key_blob));
-    EXPECT_TRUE(VerifyPublicKeyBlob(key_data, key_size, expected_key_blob));
+    EXPECT_TRUE(ValidatePublicKeyBlob(key_data, key_size, expected_key_blob));
 
     key_data[10] ^= 0x80;  // toggles a bit and expects a failure
-    EXPECT_FALSE(VerifyPublicKeyBlob(key_data, key_size, expected_key_blob));
+    EXPECT_FALSE(ValidatePublicKeyBlob(key_data, key_size, expected_key_blob));
     key_data[10] ^= 0x80;  // toggles the bit again, should pass
-    EXPECT_TRUE(VerifyPublicKeyBlob(key_data, key_size, expected_key_blob));
+    EXPECT_TRUE(ValidatePublicKeyBlob(key_data, key_size, expected_key_blob));
 }
 
 TEST_F(AvbUtilTest, VerifyEmptyPublicKeyBlob) {
@@ -442,7 +442,37 @@
     EXPECT_EQ(key_size, base::ReadFile(key_path, (char*)key_data, key_size));
 
     std::string expected_key_blob = "";  // empty means no expectation, thus return true.
-    EXPECT_TRUE(VerifyPublicKeyBlob(key_data, key_size, expected_key_blob));
+    EXPECT_TRUE(ValidatePublicKeyBlob(key_data, key_size, expected_key_blob));
+}
+
+TEST_F(AvbUtilTest, ValidatePublicKeyBlob_MultipleAllowedKeys) {
+    base::FilePath rsa2048_public_key =
+            ExtractPublicKeyAvb(data_dir_.Append("testkey_rsa2048.pem"));
+    base::FilePath rsa4096_public_key =
+            ExtractPublicKeyAvb(data_dir_.Append("testkey_rsa4096.pem"));
+    base::FilePath rsa8192_public_key =
+            ExtractPublicKeyAvb(data_dir_.Append("testkey_rsa8192.pem"));
+
+    std::vector<std::string> allowed_key_paths;
+    allowed_key_paths.push_back(rsa2048_public_key.value());
+    allowed_key_paths.push_back(rsa4096_public_key.value());
+
+    std::string expected_key_blob_2048;
+    EXPECT_TRUE(base::ReadFileToString(rsa2048_public_key, &expected_key_blob_2048));
+    std::string expected_key_blob_4096;
+    EXPECT_TRUE(base::ReadFileToString(rsa4096_public_key, &expected_key_blob_4096));
+    std::string expected_key_blob_8192;
+    EXPECT_TRUE(base::ReadFileToString(rsa8192_public_key, &expected_key_blob_8192));
+
+    EXPECT_TRUE(ValidatePublicKeyBlob(expected_key_blob_2048, allowed_key_paths));
+    EXPECT_TRUE(ValidatePublicKeyBlob(expected_key_blob_4096, allowed_key_paths));
+
+    EXPECT_FALSE(ValidatePublicKeyBlob(expected_key_blob_8192, allowed_key_paths));
+    EXPECT_FALSE(ValidatePublicKeyBlob("invalid_content", allowed_key_paths));
+    EXPECT_FALSE(ValidatePublicKeyBlob("", allowed_key_paths));
+
+    allowed_key_paths.push_back(rsa8192_public_key.value());
+    EXPECT_TRUE(ValidatePublicKeyBlob(expected_key_blob_8192, allowed_key_paths));
 }
 
 TEST_F(AvbUtilTest, VerifyVBMetaSignature) {
diff --git a/fs_mgr/libfs_avb/tests/fs_avb_device_test.cpp b/fs_mgr/libfs_avb/tests/fs_avb_device_test.cpp
index 4631330..c8605d7 100644
--- a/fs_mgr/libfs_avb/tests/fs_avb_device_test.cpp
+++ b/fs_mgr/libfs_avb/tests/fs_avb_device_test.cpp
@@ -114,8 +114,8 @@
     // Use the 2nd fstab entry, which is for physical system_other partition.
     FstabEntry* system_other_entry = &fstab[1];
     // Assign the default key if it's not specified in the fstab.
-    if (system_other_entry->avb_key.empty()) {
-        system_other_entry->avb_key = "/system/etc/security/avb/system_other.avbpubkey";
+    if (system_other_entry->avb_keys.empty()) {
+        system_other_entry->avb_keys = "/system/etc/security/avb/system_other.avbpubkey";
     }
     auto avb_handle = AvbHandle::LoadAndVerifyVbmeta(*system_other_entry);
     EXPECT_NE(nullptr, avb_handle) << "Failed to load system_other vbmeta. Try 'adb root'?";
diff --git a/fs_mgr/tests/adb-remount-test.sh b/fs_mgr/tests/adb-remount-test.sh
index fd53ed4..1ded954 100755
--- a/fs_mgr/tests/adb-remount-test.sh
+++ b/fs_mgr/tests/adb-remount-test.sh
@@ -70,19 +70,37 @@
 Returns: true if device is in adb mode" ]
 inAdb() {
   adb devices |
-    grep -v -e 'List of devices attached' -e '^$' |
+    grep -v -e 'List of devices attached' -e '^$' -e "[${SPACE}${TAB}]recovery\$" |
     if [ -n "${ANDROID_SERIAL}" ]; then
       grep "^${ANDROID_SERIAL}[${SPACE}${TAB}]" > /dev/null
     else
       wc -l | grep '^1$' >/dev/null
     fi
 }
+[ "USAGE: inRecovery
+
+Returns: true if device is in recovery mode" ]
+inRecovery() {
+  local list="`adb devices |
+              grep -v -e 'List of devices attached' -e '^$'`"
+  if [ -n "${ANDROID_SERIAL}" ]; then
+    echo "${list}" |
+      grep "^${ANDROID_SERIAL}[${SPACE}${TAB}][${SPACE}${TAB}]*recovery\$" >/dev/null
+    return ${?}
+  fi
+  if echo "${list}" | wc -l | grep '^1$' >/dev/null; then
+    echo "${list}" |
+      grep "[${SPACE}${TAB}]recovery\$" >/dev/null
+    return ${?}
+  fi
+  false
+}
 
 [ "USAGE: adb_sh <commands> </dev/stdin >/dev/stdout 2>/dev/stderr
 
 Returns: true if the command succeeded" ]
 adb_sh() {
-  args=
+  local args=
   for i in "${@}"; do
     [ -z "${args}" ] || args="${args} "
     if [ X"${i}" != X"${i#\'}" ]; then
@@ -143,10 +161,10 @@
 Returns: content of file to stdout with carriage returns skipped,
          true of the file exists" ]
 adb_cat() {
-    OUTPUT="`adb_sh cat ${1} </dev/null 2>&1`"
-    retval=${?}
+    local OUTPUT="`adb_sh cat ${1} </dev/null 2>&1`"
+    local ret=${?}
     echo "${OUTPUT}" | tr -d '\r'
-    return ${retval}
+    return ${ret}
 }
 
 [ "USAGE: adb_reboot
@@ -165,7 +183,7 @@
     echo unknown
     return
   fi
-  duration="${1}"
+  local duration="${1}"
   if [ X"${duration}" != X"${duration%s}" ]; then
     duration=${duration%s}
   elif [ X"${duration}" != X"${duration%m}" ]; then
@@ -175,9 +193,9 @@
   elif [ X"${duration}" != X"${duration%d}" ]; then
     duration=`expr ${duration%d} \* 86400`
   fi
-  seconds=`expr ${duration} % 60`
-  minutes=`expr \( ${duration} / 60 \) % 60`
-  hours=`expr ${duration} / 3600`
+  local seconds=`expr ${duration} % 60`
+  local minutes=`expr \( ${duration} / 60 \) % 60`
+  local hours=`expr ${duration} / 3600`
   if [ 0 -eq ${minutes} -a 0 -eq ${hours} ]; then
     if [ 1 -eq ${duration} ]; then
       echo 1 second
@@ -205,10 +223,10 @@
 adb_wait() {
   if [ -n "${1}" ]; then
     echo -n ". . . waiting `format_duration ${1}`" ${ANDROID_SERIAL} ${USB_ADDRESS} "${CR}"
-    timeout --preserve-status --signal=KILL ${1} adb wait-for-device
-    retval=${?}
+    timeout --preserve-status --signal=KILL ${1} adb wait-for-device 2>/dev/null
+    local ret=${?}
     echo -n "                                                                             ${CR}"
-    return ${retval}
+    return ${ret}
   else
     adb wait-for-device
   fi
@@ -216,12 +234,15 @@
 
 [ "USAGE: usb_status > stdout
 
-If adb_wait failed, check if device is in fastboot mode and report status
+If adb_wait failed, check if device is in adb, recovery or fastboot mode
+and report status string.
 
 Returns: \"(USB stack borken?)\", \"(In fastboot mode)\" or \"(in adb mode)\"" ]
 usb_status() {
   if inFastboot; then
     echo "(In fastboot mode)"
+  elif inRecovery; then
+    echo "(In recovery mode)"
   elif inAdb; then
     echo "(In adb mode)"
   else
@@ -238,15 +259,47 @@
   if [ -n "${1}" ]; then
     echo -n ". . . waiting `format_duration ${1}`" ${ANDROID_SERIAL} ${USB_ADDRESS} "${CR}"
     timeout --preserve-status --signal=KILL ${1} fastboot wait-for-device >/dev/null 2>/dev/null
-    retval=${?}
+    local ret=${?}
     echo -n "                                                                             ${CR}"
-    ( exit ${retval} )
+    ( exit ${ret} )
   else
     fastboot wait-for-device >/dev/null 2>/dev/null
   fi ||
     inFastboot
 }
 
+[ "USAGE: recovery_wait [timeout]
+
+Returns: waits until the device has returned for recovery or optional timeout" ]
+recovery_wait() {
+  if [ -n "${1}" ]; then
+    echo -n ". . . waiting `format_duration ${1}`" ${ANDROID_SERIAL} ${USB_ADDRESS} "${CR}"
+    timeout --preserve-status --signal=KILL ${1} adb wait-for-recovery 2>/dev/null
+    local ret=${?}
+    echo -n "                                                                             ${CR}"
+    return ${ret}
+  else
+    adb wait-for-recovery
+  fi
+}
+
+[ "any_wait [timeout]
+
+Returns: waits until a device has returned or optional timeout" ]
+any_wait() {
+  (
+    adb_wait ${1} &
+    adb_pid=${!}
+    fastboot_wait ${1} &
+    fastboot_pid=${!}
+    recovery_wait ${1} &
+    recovery_pid=${!}
+    wait -n
+    kill "${adb_pid}" "${fastboot_pid}" "${recovery_pid}"
+  ) >/dev/null 2>/dev/null
+  inFastboot || inAdb || inRecovery
+}
+
 [ "USAGE: adb_root
 
 NB: This can be flakey on devices due to USB state
@@ -277,11 +330,11 @@
 
 Returns: true if var output matches expected" ]
 fastboot_getvar() {
-  O=`fastboot getvar ${1} 2>&1`
-  err=${?}
+  local O=`fastboot getvar ${1} 2>&1`
+  local ret=${?}
   O="${O#< waiting for * >?}"
   O="${O%%?Finished. Total time: *}"
-  if [ 0 -ne ${err} ]; then
+  if [ 0 -ne ${ret} ]; then
     echo ${O} >&2
     false
     return
@@ -325,7 +378,7 @@
     echo "${BLUE}[     INFO ]${NORMAL} end `date`"
     [ -n "${start_time}" ] || return
     end_time=`date +%s`
-    diff_time=`expr ${end_time} - ${start_time}`
+    local diff_time=`expr ${end_time} - ${start_time}`
     echo "${BLUE}[     INFO ]${NORMAL} duration `format_duration ${diff_time}`"
   fi >&2
 }
@@ -358,8 +411,8 @@
 
 Returns true if (regex) lval matches rval" ]
 EXPECT_EQ() {
-  lval="${1}"
-  rval="${2}"
+  local lval="${1}"
+  local rval="${2}"
   shift 2
   if ! ( echo X"${rval}" | grep '^X'"${lval}"'$' >/dev/null 2>/dev/null ); then
     if [ `echo ${lval}${rval}${*} | wc -c` -gt 50 -o "${rval}" != "${rval%
@@ -404,10 +457,10 @@
 
 Exits if (regex) lval mismatches rval" ]
 check_eq() {
-  left="${1}"
-  right="${2}"
+  local lval="${1}"
+  local rval="${2}"
   shift 2
-  EXPECT_EQ "${left}" "${right}" ||
+  EXPECT_EQ "${lval}" "${rval}" ||
     die "${@}"
 }
 
@@ -498,15 +551,16 @@
 fi
 
 inFastboot && die "device in fastboot mode"
+inRecovery && die "device in recovery mode"
 if ! inAdb; then
-  echo "${ORANGE}[  WARNING ]${NORMAL} device not in adb mode"
+  echo "${ORANGE}[  WARNING ]${NORMAL} device not in adb mode" >&2
   adb_wait 2m
 fi
 inAdb || die "specified device not in adb mode"
 isDebuggable || die "device not a debug build"
 enforcing=true
 if ! adb_su getenforce </dev/null | grep 'Enforcing' >/dev/null; then
-  echo "${ORANGE}[  WARNING ]${NORMAL} device does not have sepolicy in enforcing mode"
+  echo "${ORANGE}[  WARNING ]${NORMAL} device does not have sepolicy in enforcing mode" >&2
   enforcing=false
 fi
 
@@ -563,6 +617,19 @@
       adb_reboot &&
       adb_wait 2m
   }
+
+  echo "${GREEN}[ RUN      ]${NORMAL} Testing adb shell su root remount -R command" >&2
+
+  adb_su remount -R </dev/null || true
+  sleep 2
+  adb_wait 2m ||
+    die "waiting for device after remount -R `usb_status`"
+  if [ "orange" != "`get_property ro.boot.verifiedbootstate`" -o \
+       "2" = "`get_property partition.system.verified`" ]; then
+    die "remount -R command failed"
+  fi
+
+  echo "${GREEN}[       OK ]${NORMAL} adb shell su root remount -R command" >&2
 fi
 
 echo "${GREEN}[ RUN      ]${NORMAL} Testing kernel support for overlayfs" >&2
@@ -577,7 +644,7 @@
   ) ||
   overlayfs_supported=false
 if ${overlayfs_supported}; then
-  adb_su ls /sys/module/overlay/parameters/override_creds </dev/null >/dev/null &&
+  adb_su ls /sys/module/overlay/parameters/override_creds </dev/null >/dev/null 2>/dev/null &&
     echo "${GREEN}[       OK ]${NORMAL} overlay module supports override_creds" >&2 ||
     case `adb_sh uname -r </dev/null` in
       4.[456789].* | 4.[1-9][0-9]* | [56789].*)
@@ -857,21 +924,20 @@
     echo "${GREEN}[       OK ]${NORMAL} overlay takeover in first stage init" >&2
 fi
 
-B="`adb_cat /system/hello`" ||
-  die "re-read /system/hello after reboot"
-check_eq "${A}" "${B}" /system after reboot
-echo "${GREEN}[       OK ]${NORMAL} /system content remains after reboot" >&2
-# Only root can read vendor if sepolicy permissions are as expected.
 if ${enforcing}; then
-  adb_unroot
-  B="`adb_cat /vendor/hello`" &&
-    die "re-read /vendor/hello after reboot w/o root"
+  adb_unroot ||
+    die "device not in unroot'd state"
+  B="`adb_cat /vendor/hello 2>&1`"
   check_eq "cat: /vendor/hello: Permission denied" "${B}" vendor after reboot w/o root
   echo "${GREEN}[       OK ]${NORMAL} /vendor content correct MAC after reboot" >&2
 fi
-adb_root &&
-  B="`adb_cat /vendor/hello`" ||
-  die "re-read /vendor/hello after reboot"
+B="`adb_cat /system/hello`"
+check_eq "${A}" "${B}" /system after reboot
+echo "${GREEN}[       OK ]${NORMAL} /system content remains after reboot" >&2
+# Only root can read vendor if sepolicy permissions are as expected.
+adb_root ||
+  die "adb root"
+B="`adb_cat /vendor/hello`"
 check_eq "${A}" "${B}" vendor after reboot
 echo "${GREEN}[       OK ]${NORMAL} /vendor content remains after reboot" >&2
 
@@ -901,8 +967,9 @@
 else
   adb reboot-fastboot ||
     die "fastbootd not supported (wrong adb in path?)"
-  fastboot_wait 2m ||
-    die "reboot into fastboot to flash vendor `usb_status`"
+  any_wait 2m &&
+    inFastboot ||
+    die "reboot into fastboot to flash vendor `usb_status` (bad bootloader?)"
   fastboot flash vendor ||
     ( fastboot reboot && false) ||
     die "fastboot flash vendor"
@@ -956,12 +1023,11 @@
       if ${is_userspace_fastboot}; then
         die  "overlay supposed to be minus /vendor takeover after flash vendor"
       else
-        echo "${ORANGE}[  WARNING ]${NORMAL} user fastboot missing, ignoring a failure"
-        ( die  "overlay supposed to be minus /vendor takeover after flash vendor" )
+        echo "${ORANGE}[  WARNING ]${NORMAL} user fastboot missing required to invalidate, ignoring a failure" >&2
+        echo "${ORANGE}[  WARNING ]${NORMAL} overlay supposed to be minus /vendor takeover after flash vendor" >&2
       fi
   fi
-  B="`adb_cat /system/hello`" ||
-    die "re-read /system/hello after flash vendor"
+  B="`adb_cat /system/hello`"
   check_eq "${A}" "${B}" system after flash vendor
   adb_root ||
     die "adb root"
@@ -969,13 +1035,21 @@
     if ${is_userspace_fastboot} || ! ${overlayfs_needed}; then
       die "re-read /vendor/hello after flash vendor"
     else
-      echo "${ORANGE}[  WARNING ]${NORMAL} user fastboot missing, ignoring a failure"
-      ( die "re-read /vendor/hello after flash vendor" )
+      echo "${ORANGE}[  WARNING ]${NORMAL} user fastboot missing required to invalidate, ignoring a failure" >&2
+      echo "${ORANGE}[  WARNING ]${NORMAL} re-read /vendor/hello after flash vendor" >&2
     fi
   if ${is_userspace_fastboot} || ! ${overlayfs_needed}; then
-    check_eq "cat: /vendor/hello: No such file or directory" "${B}" vendor after flash vendor
+    check_eq "cat: /vendor/hello: No such file or directory" "${B}" \
+             vendor content after flash vendor
   else
-    ( check_eq "cat: /vendor/hello: No such file or directory" "${B}" vendor after flash vendor )
+    (
+      echo "${ORANGE}[  WARNING ]${NORMAL} user fastboot missing required to invalidate, ignoring a failure" >&2
+      restore() {
+        true
+      }
+      check_eq "cat: /vendor/hello: No such file or directory" "${B}" \
+               vendor content after flash vendor
+    )
   fi
 fi
 
@@ -1003,12 +1077,10 @@
   adb_sh rm /system/hello </dev/null ||
   ( [ -n "${L}" ] && echo "${L}" && false ) ||
   die -t ${T} "cleanup hello"
-B="`adb_cat /system/hello`" &&
-  die "re-read /system/hello after rm"
-check_eq "cat: /system/hello: No such file or directory" "${B}" after flash rm
-B="`adb_cat /vendor/hello`" &&
-  die "re-read /vendor/hello after rm"
-check_eq "cat: /vendor/hello: No such file or directory" "${B}" after flash rm
+B="`adb_cat /system/hello`"
+check_eq "cat: /system/hello: No such file or directory" "${B}" after rm
+B="`adb_cat /vendor/hello`"
+check_eq "cat: /vendor/hello: No such file or directory" "${B}" after rm
 
 if [ -n "${scratch_partition}" ]; then
 
@@ -1081,9 +1153,56 @@
   die "/vendor is not read-write"
 echo "${GREEN}[       OK ]${NORMAL} mount -o rw,remount command works" >&2
 
+# Prerequisite is a prepped device from above.
+adb_reboot &&
+  adb_wait 2m ||
+  die "lost device after reboot to ro state (USB stack broken?)"
+adb_sh grep " /vendor .* rw," /proc/mounts >/dev/null </dev/null &&
+  die "/vendor is not read-only"
+adb_su remount </dev/null ||
+  die "remount command"
+adb_sh grep " /vendor .* rw," /proc/mounts >/dev/null </dev/null ||
+  die "/vendor is not read-write"
+echo "${GREEN}[       OK ]${NORMAL} remount command works from setup" >&2
+
+# Prerequisite is an overlayfs deconstructed device but with verity disabled.
+# This also saves a lot of 'noise' from the command doing a mkfs on backing
+# storage and all the related tuning and adjustment.
+for d in ${OVERLAYFS_BACKING}; do
+  adb_su rm -rf /${d}/overlay </dev/null ||
+    die "/${d}/overlay wipe"
+done
+adb_reboot &&
+  adb_wait 2m ||
+  die "lost device after reboot after wipe (USB stack broken?)"
+adb_sh grep " /vendor .* rw," /proc/mounts >/dev/null </dev/null &&
+  die "/vendor is not read-only"
+adb_su remount </dev/null ||
+  die "remount command"
+adb_sh grep " /vendor .* rw," /proc/mounts >/dev/null </dev/null ||
+  die "/vendor is not read-write"
+echo "${GREEN}[       OK ]${NORMAL} remount command works from scratch" >&2
+
 restore
 err=${?}
 
+if [ ${err} = 0 ] && ${overlayfs_supported}; then
+  echo "${GREEN}[ RUN      ]${NORMAL} test 'adb remount -R'" >&2
+  adb_root &&
+    adb remount -R &&
+    adb_wait 2m ||
+    die "adb remount -R"
+  if [ "orange" != "`get_property ro.boot.verifiedbootstate`" -o \
+       "2" = "`get_property partition.system.verified`" ]; then
+    die "remount -R command failed to disable verity"
+  fi
+
+  echo "${GREEN}[       OK ]${NORMAL} 'adb remount -R' command" >&2
+
+  restore
+  err=${?}
+fi
+
 restore() {
   true
 }
diff --git a/init/first_stage_mount.cpp b/init/first_stage_mount.cpp
index 6e55c11a..17cd470 100644
--- a/init/first_stage_mount.cpp
+++ b/init/first_stage_mount.cpp
@@ -760,7 +760,7 @@
         if (!InitAvbHandle()) return false;
         hashtree_result =
                 avb_handle_->SetUpAvbHashtree(fstab_entry, false /* wait_for_verity_dev */);
-    } else if (!fstab_entry->avb_key.empty()) {
+    } else if (!fstab_entry->avb_keys.empty()) {
         if (!InitAvbHandle()) return false;
         // Checks if hashtree should be disabled from the top-level /vbmeta.
         if (avb_handle_->status() == AvbHandleStatus::kHashtreeDisabled ||