Merge "liblp: Add a VIRTUAL_AB_DEVICE flag for LpMetadataHeader."
diff --git a/adb/adb.cpp b/adb/adb.cpp
index 9b663be..460ddde 100644
--- a/adb/adb.cpp
+++ b/adb/adb.cpp
@@ -1167,7 +1167,7 @@
std::string host;
int port = DEFAULT_ADB_LOCAL_TRANSPORT_PORT;
std::string error;
- if (address.starts_with("vsock:")) {
+ if (address.starts_with("vsock:") || address.starts_with("localfilesystem:")) {
serial = address;
} else if (!android::base::ParseNetAddress(address, &host, &port, &serial, &error)) {
SendFail(reply_fd, android::base::StringPrintf("couldn't parse '%s': %s",
diff --git a/adb/adb.h b/adb/adb.h
index c6cb06a..e7fcc91 100644
--- a/adb/adb.h
+++ b/adb/adb.h
@@ -200,7 +200,7 @@
#define ADB_SUBCLASS 0x42
#define ADB_PROTOCOL 0x1
-void local_init(int port);
+void local_init(const std::string& addr);
bool local_connect(int port);
int local_connect_arbitrary_ports(int console_port, int adb_port, std::string* error);
diff --git a/adb/client/main.cpp b/adb/client/main.cpp
index 0c5c28f..e5ffe4c 100644
--- a/adb/client/main.cpp
+++ b/adb/client/main.cpp
@@ -129,7 +129,7 @@
}
if (!getenv("ADB_EMU") || strcmp(getenv("ADB_EMU"), "0") != 0) {
- local_init(DEFAULT_ADB_LOCAL_TRANSPORT_PORT);
+ local_init(android::base::StringPrintf("tcp:%d", DEFAULT_ADB_LOCAL_TRANSPORT_PORT));
}
std::string error;
diff --git a/adb/daemon/main.cpp b/adb/daemon/main.cpp
index 7277cc8..3322574 100644
--- a/adb/daemon/main.cpp
+++ b/adb/daemon/main.cpp
@@ -32,11 +32,13 @@
#include <sys/prctl.h>
#include <memory>
+#include <vector>
#include <android-base/logging.h>
#include <android-base/macros.h>
#include <android-base/properties.h>
#include <android-base/stringprintf.h>
+#include <android-base/strings.h>
#if defined(__ANDROID__)
#include <libminijail.h>
@@ -51,6 +53,7 @@
#include "adb_auth.h"
#include "adb_listeners.h"
#include "adb_utils.h"
+#include "socket_spec.h"
#include "transport.h"
#include "mdns.h"
@@ -179,12 +182,26 @@
}
#endif
-static void setup_port(int port) {
- LOG(INFO) << "adbd listening on port " << port;
- local_init(port);
+static void setup_adb(const std::vector<std::string>& addrs) {
#if defined(__ANDROID__)
+ // Get the first valid port from addrs and setup mDNS.
+ int port = -1;
+ std::string error;
+ for (const auto& addr : addrs) {
+ port = get_host_socket_spec_port(addr, &error);
+ if (port != -1) {
+ break;
+ }
+ }
+ if (port == -1) {
+ port = DEFAULT_ADB_LOCAL_TRANSPORT_PORT;
+ }
setup_mdns(port);
#endif
+ for (const auto& addr : addrs) {
+ LOG(INFO) << "adbd listening on " << addr;
+ local_init(addr);
+ }
}
int adbd_main(int server_port) {
@@ -248,25 +265,38 @@
// If one of these properties is set, also listen on that port.
// If one of the properties isn't set and we couldn't listen on usb, listen
// on the default port.
- std::string prop_port = android::base::GetProperty("service.adb.tcp.port", "");
- if (prop_port.empty()) {
- prop_port = android::base::GetProperty("persist.adb.tcp.port", "");
- }
+ std::vector<std::string> addrs;
+ std::string prop_addr = android::base::GetProperty("service.adb.listen_addrs", "");
+ if (prop_addr.empty()) {
+ std::string prop_port = android::base::GetProperty("service.adb.tcp.port", "");
+ if (prop_port.empty()) {
+ prop_port = android::base::GetProperty("persist.adb.tcp.port", "");
+ }
#if !defined(__ANDROID__)
- if (prop_port.empty() && getenv("ADBD_PORT")) {
- prop_port = getenv("ADBD_PORT");
- }
+ if (prop_port.empty() && getenv("ADBD_PORT")) {
+ prop_port = getenv("ADBD_PORT");
+ }
#endif
- int port;
- if (sscanf(prop_port.c_str(), "%d", &port) == 1 && port > 0) {
- D("using port=%d", port);
- // Listen on TCP port specified by service.adb.tcp.port property.
- setup_port(port);
- } else if (!is_usb) {
- // Listen on default port.
- setup_port(DEFAULT_ADB_LOCAL_TRANSPORT_PORT);
+ int port;
+ if (sscanf(prop_port.c_str(), "%d", &port) == 1 && port > 0) {
+ D("using tcp port=%d", port);
+ // Listen on TCP and VSOCK port specified by service.adb.tcp.port property.
+ addrs.push_back(android::base::StringPrintf("tcp:%d", port));
+ addrs.push_back(android::base::StringPrintf("vsock:%d", port));
+ setup_adb(addrs);
+ } else if (!is_usb) {
+ // Listen on default port.
+ addrs.push_back(
+ android::base::StringPrintf("tcp:%d", DEFAULT_ADB_LOCAL_TRANSPORT_PORT));
+ addrs.push_back(
+ android::base::StringPrintf("vsock:%d", DEFAULT_ADB_LOCAL_TRANSPORT_PORT));
+ setup_adb(addrs);
+ }
+ } else {
+ addrs = android::base::Split(prop_addr, ",");
+ setup_adb(addrs);
}
D("adbd_main(): pre init_jdwp()");
diff --git a/adb/daemon/transport_qemu.cpp b/adb/daemon/transport_qemu.cpp
index aa760bc..901efee 100644
--- a/adb/daemon/transport_qemu.cpp
+++ b/adb/daemon/transport_qemu.cpp
@@ -18,6 +18,7 @@
#include <qemu_pipe.h>
#define TRACE_TAG TRANSPORT
+#include "socket_spec.h"
#include "sysdeps.h"
#include "transport.h"
@@ -55,7 +56,7 @@
* the transport registration is completed. That's why we need to send the
* 'start' request after the transport is registered.
*/
-void qemu_socket_thread(int port) {
+void qemu_socket_thread(std::string_view addr) {
/* 'accept' request to the adb QEMUD service. */
static const char _accept_req[] = "accept";
/* 'start' request to the adb QEMUD service. */
@@ -69,6 +70,12 @@
adb_thread_setname("qemu socket");
D("transport: qemu_socket_thread() starting");
+ std::string error;
+ int port = get_host_socket_spec_port(addr, &error);
+ if (port == -1) {
+ port = DEFAULT_ADB_LOCAL_TRANSPORT_PORT;
+ }
+
/* adb QEMUD service connection request. */
snprintf(con_name, sizeof(con_name), "pipe:qemud:adb:%d", port);
@@ -78,7 +85,7 @@
/* This could be an older version of the emulator, that doesn't
* implement adb QEMUD service. Fall back to the old TCP way. */
D("adb service is not available. Falling back to TCP socket.");
- std::thread(server_socket_thread, tcp_listen_inaddr_any, port).detach();
+ std::thread(server_socket_thread, adb_listen, addr).detach();
return;
}
diff --git a/adb/socket_spec.cpp b/adb/socket_spec.cpp
index 9ce443e..d17036c 100644
--- a/adb/socket_spec.cpp
+++ b/adb/socket_spec.cpp
@@ -122,6 +122,41 @@
return true;
}
+int get_host_socket_spec_port(std::string_view spec, std::string* error) {
+ int port;
+ if (spec.starts_with("tcp:")) {
+ if (!parse_tcp_socket_spec(spec, nullptr, &port, nullptr, error)) {
+ return -1;
+ }
+ } else if (spec.starts_with("vsock:")) {
+#if ADB_LINUX
+ std::string spec_str(spec);
+ std::vector<std::string> fragments = android::base::Split(spec_str, ":");
+ if (fragments.size() != 2) {
+ *error = "given vsock server socket string was invalid";
+ return -1;
+ }
+ if (!android::base::ParseInt(fragments[1], &port)) {
+ *error = "could not parse vsock port";
+ errno = EINVAL;
+ return -1;
+ }
+ if (port < 0) {
+ *error = "vsock port was negative.";
+ errno = EINVAL;
+ return -1;
+ }
+#else // ADB_LINUX
+ *error = "vsock is only supported on linux";
+ return -1;
+#endif // ADB_LINUX
+ } else {
+ *error = "given socket spec string was invalid";
+ return -1;
+ }
+ return port;
+}
+
static bool tcp_host_is_local(std::string_view hostname) {
// FIXME
return hostname.empty() || hostname == "localhost";
@@ -254,6 +289,14 @@
fd->reset(network_local_client(&address[prefix.length()], it.second.socket_namespace,
SOCK_STREAM, error));
+
+ if (fd->get() < 0) {
+ *error =
+ android::base::StringPrintf("could not connect to %s address '%s'",
+ it.first.c_str(), std::string(address).c_str());
+ return false;
+ }
+
if (serial) {
*serial = address;
}
@@ -275,7 +318,11 @@
}
int result;
+#if ADB_HOST
if (hostname.empty() && gListenAll) {
+#else
+ if (hostname.empty()) {
+#endif
result = network_inaddr_any_server(port, SOCK_STREAM, error);
} else if (tcp_host_is_local(hostname)) {
result = network_loopback_server(port, SOCK_STREAM, error, true);
diff --git a/adb/socket_spec.h b/adb/socket_spec.h
index 7cc2fac..94719c8 100644
--- a/adb/socket_spec.h
+++ b/adb/socket_spec.h
@@ -31,3 +31,5 @@
bool parse_tcp_socket_spec(std::string_view spec, std::string* hostname, int* port,
std::string* serial, std::string* error);
+
+int get_host_socket_spec_port(std::string_view spec, std::string* error);
diff --git a/adb/socket_spec_test.cpp b/adb/socket_spec_test.cpp
index 3a2f60c..e9d5270 100644
--- a/adb/socket_spec_test.cpp
+++ b/adb/socket_spec_test.cpp
@@ -18,6 +18,10 @@
#include <string>
+#include <unistd.h>
+
+#include <android-base/file.h>
+#include <android-base/stringprintf.h>
#include <gtest/gtest.h>
TEST(socket_spec, parse_tcp_socket_spec_just_port) {
@@ -88,3 +92,63 @@
EXPECT_FALSE(parse_tcp_socket_spec("tcp:[::1]:", &hostname, &port, &serial, &error));
EXPECT_FALSE(parse_tcp_socket_spec("tcp:[::1]:-1", &hostname, &port, &serial, &error));
}
+
+TEST(socket_spec, get_host_socket_spec_port) {
+ std::string error;
+ EXPECT_EQ(5555, get_host_socket_spec_port("tcp:5555", &error));
+ EXPECT_EQ(5555, get_host_socket_spec_port("tcp:localhost:5555", &error));
+ EXPECT_EQ(5555, get_host_socket_spec_port("tcp:[::1]:5555", &error));
+ EXPECT_EQ(5555, get_host_socket_spec_port("vsock:5555", &error));
+}
+
+TEST(socket_spec, get_host_socket_spec_port_no_port) {
+ std::string error;
+ EXPECT_EQ(5555, get_host_socket_spec_port("tcp:localhost", &error));
+ EXPECT_EQ(-1, get_host_socket_spec_port("vsock:localhost", &error));
+}
+
+TEST(socket_spec, get_host_socket_spec_port_bad_ports) {
+ std::string error;
+ EXPECT_EQ(-1, get_host_socket_spec_port("tcp:65536", &error));
+ EXPECT_EQ(-1, get_host_socket_spec_port("tcp:-5", &error));
+ EXPECT_EQ(-1, get_host_socket_spec_port("vsock:-5", &error));
+ EXPECT_EQ(-1, get_host_socket_spec_port("vsock:5:5555", &error));
+}
+
+TEST(socket_spec, get_host_socket_spec_port_bad_string) {
+ std::string error;
+ EXPECT_EQ(-1, get_host_socket_spec_port("tcpz:5555", &error));
+ EXPECT_EQ(-1, get_host_socket_spec_port("vsockz:5555", &error));
+ EXPECT_EQ(-1, get_host_socket_spec_port("abcd:5555", &error));
+ EXPECT_EQ(-1, get_host_socket_spec_port("abcd", &error));
+}
+
+TEST(socket_spec, socket_spec_listen_connect_tcp) {
+ std::string error, serial;
+ int port;
+ unique_fd server_fd, client_fd;
+ EXPECT_FALSE(socket_spec_connect(&client_fd, "tcp:localhost:7777", &port, &serial, &error));
+ server_fd.reset(socket_spec_listen("tcp:7777", &error, &port));
+ EXPECT_NE(server_fd.get(), -1);
+ EXPECT_TRUE(socket_spec_connect(&client_fd, "tcp:localhost:7777", &port, &serial, &error));
+ EXPECT_NE(client_fd.get(), -1);
+}
+
+TEST(socket_spec, socket_spec_listen_connect_localfilesystem) {
+ std::string error, serial;
+ int port;
+ unique_fd server_fd, client_fd;
+ TemporaryDir sock_dir;
+
+ // Only run this test if the created directory is writable.
+ int result = access(sock_dir.path, W_OK);
+ if (result == 0) {
+ std::string sock_addr =
+ android::base::StringPrintf("localfilesystem:%s/af_unix_socket", sock_dir.path);
+ EXPECT_FALSE(socket_spec_connect(&client_fd, sock_addr, &port, &serial, &error));
+ server_fd.reset(socket_spec_listen(sock_addr, &error, &port));
+ EXPECT_NE(server_fd.get(), -1);
+ EXPECT_TRUE(socket_spec_connect(&client_fd, sock_addr, &port, &serial, &error));
+ EXPECT_NE(client_fd.get(), -1);
+ }
+}
diff --git a/adb/sockets.cpp b/adb/sockets.cpp
index 7d5bf17..423af67 100644
--- a/adb/sockets.cpp
+++ b/adb/sockets.cpp
@@ -625,7 +625,8 @@
return true;
};
- static constexpr std::string_view prefixes[] = {"usb:", "product:", "model:", "device:"};
+ static constexpr std::string_view prefixes[] = {
+ "usb:", "product:", "model:", "device:", "localfilesystem:"};
for (std::string_view prefix : prefixes) {
if (command.starts_with(prefix)) {
consume(prefix.size());
diff --git a/adb/transport.h b/adb/transport.h
index ea77117..569e8bb 100644
--- a/adb/transport.h
+++ b/adb/transport.h
@@ -424,11 +424,12 @@
asocket* create_device_tracker(bool long_output);
#if !ADB_HOST
-unique_fd tcp_listen_inaddr_any(int port, std::string* error);
-void server_socket_thread(std::function<unique_fd(int, std::string*)> listen_func, int port);
+unique_fd adb_listen(std::string_view addr, std::string* error);
+void server_socket_thread(std::function<unique_fd(std::string_view, std::string*)> listen_func,
+ std::string_view addr);
#if defined(__ANDROID__)
-void qemu_socket_thread(int port);
+void qemu_socket_thread(std::string_view addr);
bool use_qemu_goldfish();
#endif
diff --git a/adb/transport_local.cpp b/adb/transport_local.cpp
index b9f738d..c726186 100644
--- a/adb/transport_local.cpp
+++ b/adb/transport_local.cpp
@@ -85,22 +85,6 @@
return local_connect_arbitrary_ports(port - 1, port, &dummy) == 0;
}
-std::tuple<unique_fd, int, std::string> tcp_connect(const std::string& address,
- std::string* response) {
- unique_fd fd;
- int port = DEFAULT_ADB_LOCAL_TRANSPORT_PORT;
- std::string serial;
- std::string prefix_addr = address.starts_with("vsock:") ? address : "tcp:" + address;
- if (socket_spec_connect(&fd, prefix_addr, &port, &serial, response)) {
- close_on_exec(fd);
- if (!set_tcp_keepalive(fd, 1)) {
- D("warning: failed to configure TCP keepalives (%s)", strerror(errno));
- }
- return std::make_tuple(std::move(fd), port, serial);
- }
- return std::make_tuple(unique_fd(), 0, serial);
-}
-
void connect_device(const std::string& address, std::string* response) {
if (address.empty()) {
*response = "empty address";
@@ -110,17 +94,25 @@
D("connection requested to '%s'", address.c_str());
unique_fd fd;
int port;
- std::string serial;
- std::tie(fd, port, serial) = tcp_connect(address, response);
+ std::string serial, prefix_addr;
+
+ // If address does not match any socket type, it should default to TCP.
+ if (address.starts_with("vsock:") || address.starts_with("localfilesystem:")) {
+ prefix_addr = address;
+ } else {
+ prefix_addr = "tcp:" + address;
+ }
+
+ socket_spec_connect(&fd, prefix_addr, &port, &serial, response);
if (fd.get() == -1) {
return;
}
- auto reconnect = [address](atransport* t) {
+ auto reconnect = [prefix_addr](atransport* t) {
std::string response;
unique_fd fd;
int port;
std::string serial;
- std::tie(fd, port, serial) = tcp_connect(address, &response);
+ socket_spec_connect(&fd, prefix_addr, &port, &serial, &response);
if (fd == -1) {
D("reconnect failed: %s", response.c_str());
return ReconnectResult::Retry;
@@ -203,7 +195,7 @@
std::mutex &retry_ports_lock = *new std::mutex;
std::condition_variable &retry_ports_cond = *new std::condition_variable;
-static void client_socket_thread(int) {
+static void client_socket_thread(std::string_view) {
adb_thread_setname("client_socket_thread");
D("transport: client_socket_thread() starting");
PollAllLocalPortsForEmulator();
@@ -248,7 +240,8 @@
#else // !ADB_HOST
-void server_socket_thread(std::function<unique_fd(int, std::string*)> listen_func, int port) {
+void server_socket_thread(std::function<unique_fd(std::string_view, std::string*)> listen_func,
+ std::string_view addr) {
adb_thread_setname("server socket");
unique_fd serverfd;
@@ -256,7 +249,7 @@
while (serverfd == -1) {
errno = 0;
- serverfd = listen_func(port, &error);
+ serverfd = listen_func(addr, &error);
if (errno == EAFNOSUPPORT || errno == EINVAL || errno == EPROTONOSUPPORT) {
D("unrecoverable error: '%s'", error.c_str());
return;
@@ -276,7 +269,9 @@
close_on_exec(fd.get());
disable_tcp_nagle(fd.get());
std::string serial = android::base::StringPrintf("host-%d", fd.get());
- register_socket_transport(std::move(fd), std::move(serial), port, 1,
+ // We don't care about port value in "register_socket_transport" as it is used
+ // only from ADB_HOST. "server_socket_thread" is never called from ADB_HOST.
+ register_socket_transport(std::move(fd), std::move(serial), 0, 1,
[](atransport*) { return ReconnectResult::Abort; });
}
}
@@ -285,38 +280,30 @@
#endif
-unique_fd tcp_listen_inaddr_any(int port, std::string* error) {
- return unique_fd{network_inaddr_any_server(port, SOCK_STREAM, error)};
-}
-
#if !ADB_HOST
-static unique_fd vsock_listen(int port, std::string* error) {
- return unique_fd{
- socket_spec_listen(android::base::StringPrintf("vsock:%d", port), error, nullptr)
- };
+unique_fd adb_listen(std::string_view addr, std::string* error) {
+ return unique_fd{socket_spec_listen(addr, error, nullptr)};
}
#endif
-void local_init(int port) {
+void local_init(const std::string& addr) {
#if ADB_HOST
D("transport: local client init");
- std::thread(client_socket_thread, port).detach();
+ std::thread(client_socket_thread, addr).detach();
adb_local_transport_max_port_env_override();
#elif !defined(__ANDROID__)
// Host adbd.
D("transport: local server init");
- std::thread(server_socket_thread, tcp_listen_inaddr_any, port).detach();
- std::thread(server_socket_thread, vsock_listen, port).detach();
+ std::thread(server_socket_thread, adb_listen, addr).detach();
#else
D("transport: local server init");
// For the adbd daemon in the system image we need to distinguish
// between the device, and the emulator.
- if (use_qemu_goldfish()) {
- std::thread(qemu_socket_thread, port).detach();
+ if (addr.starts_with("tcp:") && use_qemu_goldfish()) {
+ std::thread(qemu_socket_thread, addr).detach();
} else {
- std::thread(server_socket_thread, tcp_listen_inaddr_any, port).detach();
+ std::thread(server_socket_thread, adb_listen, addr).detach();
}
- std::thread(server_socket_thread, vsock_listen, port).detach();
#endif // !ADB_HOST
}
diff --git a/fs_mgr/fs_mgr.cpp b/fs_mgr/fs_mgr.cpp
index 7b686f0..5c0401e 100644
--- a/fs_mgr/fs_mgr.cpp
+++ b/fs_mgr/fs_mgr.cpp
@@ -1358,24 +1358,17 @@
return ret;
}
-static std::string GetBlockDeviceForMountPoint(const std::string& mount_point) {
- Fstab mounts;
- if (!ReadFstabFromFile("/proc/mounts", &mounts)) {
- LERROR << "Can't read /proc/mounts";
- return "";
- }
- auto entry = GetEntryForMountPoint(&mounts, mount_point);
- if (entry == nullptr) {
- LWARNING << mount_point << " is not mounted";
- return "";
- }
- return entry->blk_device;
-}
-
// TODO(b/143970043): return different error codes based on which step failed.
int fs_mgr_remount_userdata_into_checkpointing(Fstab* fstab) {
- std::string block_device = GetBlockDeviceForMountPoint("/data");
- if (block_device.empty()) {
+ Fstab proc_mounts;
+ if (!ReadFstabFromFile("/proc/mounts", &proc_mounts)) {
+ LERROR << "Can't read /proc/mounts";
+ return -1;
+ }
+ std::string block_device;
+ if (auto entry = GetEntryForMountPoint(&proc_mounts, "/data"); entry != nullptr) {
+ block_device = entry->blk_device;
+ } else {
LERROR << "/data is not mounted";
return -1;
}
@@ -1408,6 +1401,15 @@
}
} else {
LINFO << "Unmounting /data before remounting into checkpointing mode";
+ // First make sure that all the bind-mounts on top of /data are unmounted.
+ for (const auto& entry : proc_mounts) {
+ if (entry.blk_device == block_device && entry.mount_point != "/data") {
+ LINFO << "Unmounting bind-mount " << entry.mount_point;
+ if (umount2(entry.mount_point.c_str(), UMOUNT_NOFOLLOW) != 0) {
+ PWARNING << "Failed to unmount " << entry.mount_point;
+ }
+ }
+ }
if (umount2("/data", UMOUNT_NOFOLLOW) != 0) {
PERROR << "Failed to umount /data";
return -1;
diff --git a/fs_mgr/libfiemap/Android.bp b/fs_mgr/libfiemap/Android.bp
new file mode 100644
index 0000000..8dbbf4c
--- /dev/null
+++ b/fs_mgr/libfiemap/Android.bp
@@ -0,0 +1,135 @@
+//
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+cc_library_headers {
+ name: "libfiemap_headers",
+ recovery_available: true,
+ export_include_dirs: ["include"],
+}
+
+cc_defaults {
+ name: "libfiemap_defaults",
+ defaults: ["fs_mgr_defaults"],
+ cflags: [
+ "-D_FILE_OFFSET_BITS=64",
+ "-Wall",
+ "-Werror",
+ ],
+
+ srcs: [
+ "fiemap_writer.cpp",
+ "image_manager.cpp",
+ "metadata.cpp",
+ "split_fiemap_writer.cpp",
+ "utility.cpp",
+ ],
+
+ static_libs: [
+ "libdm",
+ "libext2_uuid",
+ "libext4_utils",
+ "liblp",
+ "libfs_mgr",
+ ],
+
+ shared_libs: [
+ "libbase",
+ ],
+
+ header_libs: [
+ "libfiemap_headers",
+ "liblog_headers",
+ ],
+
+ export_shared_lib_headers: [
+ "libbase",
+ ],
+
+ export_header_lib_headers: [
+ "libfiemap_headers",
+ ],
+}
+
+// Open up a binder IImageManager interface.
+cc_library_static {
+ name: "libfiemap_binder",
+ defaults: ["libfiemap_defaults"],
+ srcs: [
+ "binder.cpp",
+ ],
+ whole_static_libs: [
+ "gsi_aidl_interface-cpp",
+ "libgsi",
+ ],
+ shared_libs: [
+ "libbinder",
+ ],
+}
+
+// Open up a passthrough IImageManager interface. Use libfiemap_binder whenever
+// possible. This should only be used when binder is not available.
+cc_library_static {
+ name: "libfiemap_passthrough",
+ defaults: ["libfiemap_defaults"],
+ recovery_available: true,
+ srcs: [
+ "passthrough.cpp",
+ ],
+}
+
+cc_test {
+ name: "fiemap_writer_test",
+ defaults: ["libfiemap_defaults"],
+ static_libs: [
+ "libbase",
+ "libdm",
+ "liblog",
+ ],
+
+ data: [
+ "testdata/unaligned_file",
+ "testdata/file_4k",
+ "testdata/file_32k",
+ ],
+
+ srcs: [
+ "fiemap_writer_test.cpp",
+ ],
+}
+
+cc_test {
+ name: "fiemap_image_test",
+ defaults: ["libfiemap_defaults"],
+ static_libs: [
+ "libdm",
+ "libext4_utils",
+ "libfs_mgr",
+ "liblp",
+ ],
+ shared_libs: [
+ "libcrypto",
+ "libcrypto_utils",
+ "libcutils",
+ "liblog",
+ ],
+ srcs: [
+ "image_test.cpp",
+ ],
+}
+
+vts_config {
+ name: "VtsFiemapWriterTest",
+}
diff --git a/fs_mgr/libfiemap/AndroidTest.xml b/fs_mgr/libfiemap/AndroidTest.xml
new file mode 100644
index 0000000..44c80fc
--- /dev/null
+++ b/fs_mgr/libfiemap/AndroidTest.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2019 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Config for VTS VtsFiemapWriterTest">
+ <option name="config-descriptor:metadata" key="plan" value="vts-kernel" />
+ <target_preparer class="com.android.compatibility.common.tradefed.targetprep.VtsFilePusher">
+ <option name="abort-on-push-failure" value="false"/>
+ <option name="push-group" value="HostDrivenTest.push"/>
+ </target_preparer>
+ <test class="com.android.tradefed.testtype.VtsMultiDeviceTest">
+ <option name="test-module-name" value="VtsFiemapWriterTest"/>
+ <option name="binary-test-source" value="_32bit::DATA/nativetest/fiemap_writer_test/fiemap_writer_test" />
+ <option name="binary-test-source" value="_64bit::DATA/nativetest64/fiemap_writer_test/fiemap_writer_test" />
+ <option name="binary-test-type" value="gtest"/>
+ <option name="precondition-first-api-level" value="29" />
+ <option name="test-timeout" value="1m"/>
+ </test>
+</configuration>
diff --git a/fs_mgr/libfiemap/README.md b/fs_mgr/libfiemap/README.md
new file mode 100644
index 0000000..62d610a
--- /dev/null
+++ b/fs_mgr/libfiemap/README.md
@@ -0,0 +1,75 @@
+libfiemap
+=============
+
+`libfiemap` is a library for creating block-devices that are backed by
+storage in read-write partitions. It exists primary for gsid. Generally, the
+library works by using `libfiemap_writer` to allocate large files within
+filesystem, and then tracks their extents.
+
+There are three main uses for `libfiemap`:
+ - Creating images that will act as block devices. For example, gsid needs to
+ create a `system_gsi` image to store Dynamic System Updates.
+ - Mapping the image as a block device while /data is mounted. This is fairly
+ tricky and is described in more detail below.
+ - Mapping the image as a block device during first-stage init. This is simple
+ because it uses the same logic from dynamic partitions.
+
+Image creation is done through `SplitFiemap`. Depending on the file system,
+a large image may have to be split into multiple files. On Ext4 the limit is
+16GiB and on FAT32 it's 4GiB. Images are saved into `/data/gsi/<name>/`
+where `<name>` is chosen by the process requesting the image.
+
+At the same time, a file called `/metadata/gsi/<name>/lp_metadata` is created.
+This is a super partition header that allows first-stage init to create dynamic
+partitions from the image files. It also tracks the canonical size of the image,
+since the file size may be larger due to alignment.
+
+Mapping
+-------
+
+It is easy to make block devices out of blocks on `/data` when it is not
+mounted, so first-stage init has no issues mapping dynamic partitions from
+images. After `/data` is mounted however, there are two problems:
+ - `/data` is encrypted.
+ - `/dev/block/by-name/data` may be marked as in-use.
+
+We break the problem down into three scenarios.
+
+### FDE and Metadata Encrypted Devices
+
+When FDE or metadata encryption is used, `/data` is not mounted from
+`/dev/block/by-name/data`. Instead, it is mounted from an intermediate
+`dm-crypt` or `dm-default-key` device. This means the underlying device is
+not marked in use, and we can create new dm-linear devices on top of it.
+
+On these devices, a block device for an image will consist of a single
+device-mapper device with a `dm-linear` table entry for each extent in the
+backing file.
+
+### Unencrypted and FBE-encrypted Devices
+
+When a device is unencrypted, or is encrypted with FBE but not metadata
+encryption, we instead use a loop device with `LOOP_SET_DIRECT_IO` enabled.
+Since `/data/gsi` has encryption disabled, this means the raw blocks will be
+unencrypted as well.
+
+### Split Images
+
+If an image was too large to store a single file on the underlying filesystem,
+on an FBE/unencrypted device we will have multiple loop devices. In this case,
+we create a device-mapper device as well. For each loop device it will have one
+`dm-linear` table entry spanning the length of the device.
+
+State Tracking
+--------------
+
+It's important that we know whether or not an image is currently in-use by a
+block device. It could be catastrophic to write to a dm-linear device if the
+underlying blocks are no longer owned by the original file. Thus, when mapping
+an image, we create a property called `gsid.mapped_image.<name>` and set it to
+the path of the block device.
+
+Additionally, we create a `/metadata/gsi/<subdir>/<name>.status` file. Each
+line in this file denotes a dependency on either a device-mapper node or a loop
+device. When deleting a block device, this file is used to release all
+resources.
diff --git a/fs_mgr/libfiemap/binder.cpp b/fs_mgr/libfiemap/binder.cpp
new file mode 100644
index 0000000..49779f4
--- /dev/null
+++ b/fs_mgr/libfiemap/binder.cpp
@@ -0,0 +1,224 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#if !defined(__ANDROID_RECOVERY__)
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+#include <android/gsi/IGsiService.h>
+#include <android/gsi/IGsid.h>
+#include <binder/IServiceManager.h>
+#include <libfiemap/image_manager.h>
+#include <libgsi/libgsi.h>
+
+namespace android {
+namespace fiemap {
+
+using namespace android::gsi;
+using namespace std::chrono_literals;
+
+class ImageManagerBinder final : public IImageManager {
+ public:
+ ImageManagerBinder(android::sp<IGsiService>&& service, android::sp<IImageService>&& manager);
+ bool CreateBackingImage(const std::string& name, uint64_t size, int flags) override;
+ bool DeleteBackingImage(const std::string& name) override;
+ bool MapImageDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms,
+ std::string* path) override;
+ bool UnmapImageDevice(const std::string& name) override;
+ bool BackingImageExists(const std::string& name) override;
+ bool IsImageMapped(const std::string& name) override;
+ bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name,
+ std::string* dev) override;
+ bool ZeroFillNewImage(const std::string& name, uint64_t bytes) override;
+ bool RemoveAllImages() override;
+
+ std::vector<std::string> GetAllBackingImages() override;
+
+ private:
+ android::sp<IGsiService> service_;
+ android::sp<IImageService> manager_;
+};
+
+ImageManagerBinder::ImageManagerBinder(android::sp<IGsiService>&& service,
+ android::sp<IImageService>&& manager)
+ : service_(std::move(service)), manager_(std::move(manager)) {}
+
+bool ImageManagerBinder::CreateBackingImage(const std::string& name, uint64_t size, int flags) {
+ auto status = manager_->createBackingImage(name, size, flags);
+ if (!status.isOk()) {
+ LOG(ERROR) << __PRETTY_FUNCTION__
+ << " binder returned: " << status.exceptionMessage().string();
+ return false;
+ }
+ return true;
+}
+
+bool ImageManagerBinder::DeleteBackingImage(const std::string& name) {
+ auto status = manager_->deleteBackingImage(name);
+ if (!status.isOk()) {
+ LOG(ERROR) << __PRETTY_FUNCTION__
+ << " binder returned: " << status.exceptionMessage().string();
+ return false;
+ }
+ return true;
+}
+
+bool ImageManagerBinder::MapImageDevice(const std::string& name,
+ const std::chrono::milliseconds& timeout_ms,
+ std::string* path) {
+ int32_t timeout_ms_count =
+ static_cast<int32_t>(std::clamp<typename std::chrono::milliseconds::rep>(
+ timeout_ms.count(), INT32_MIN, INT32_MAX));
+ MappedImage map;
+ auto status = manager_->mapImageDevice(name, timeout_ms_count, &map);
+ if (!status.isOk()) {
+ LOG(ERROR) << __PRETTY_FUNCTION__
+ << " binder returned: " << status.exceptionMessage().string();
+ return false;
+ }
+ *path = map.path;
+ return true;
+}
+
+bool ImageManagerBinder::UnmapImageDevice(const std::string& name) {
+ auto status = manager_->unmapImageDevice(name);
+ if (!status.isOk()) {
+ LOG(ERROR) << __PRETTY_FUNCTION__
+ << " binder returned: " << status.exceptionMessage().string();
+ return false;
+ }
+ return true;
+}
+
+bool ImageManagerBinder::BackingImageExists(const std::string& name) {
+ bool retval;
+ auto status = manager_->backingImageExists(name, &retval);
+ if (!status.isOk()) {
+ LOG(ERROR) << __PRETTY_FUNCTION__
+ << " binder returned: " << status.exceptionMessage().string();
+ return false;
+ }
+ return retval;
+}
+
+bool ImageManagerBinder::IsImageMapped(const std::string& name) {
+ bool retval;
+ auto status = manager_->isImageMapped(name, &retval);
+ if (!status.isOk()) {
+ LOG(ERROR) << __PRETTY_FUNCTION__
+ << " binder returned: " << status.exceptionMessage().string();
+ return false;
+ }
+ return retval;
+}
+
+bool ImageManagerBinder::MapImageWithDeviceMapper(const IPartitionOpener& opener,
+ const std::string& name, std::string* dev) {
+ (void)opener;
+ (void)name;
+ (void)dev;
+ LOG(ERROR) << "MapImageWithDeviceMapper is not available over binder.";
+ return false;
+}
+
+std::vector<std::string> ImageManagerBinder::GetAllBackingImages() {
+ std::vector<std::string> retval;
+ auto status = manager_->getAllBackingImages(&retval);
+ if (!status.isOk()) {
+ LOG(ERROR) << __PRETTY_FUNCTION__
+ << " binder returned: " << status.exceptionMessage().string();
+ }
+ return retval;
+}
+
+bool ImageManagerBinder::ZeroFillNewImage(const std::string& name, uint64_t bytes) {
+ auto status = manager_->zeroFillNewImage(name, bytes);
+ if (!status.isOk()) {
+ LOG(ERROR) << __PRETTY_FUNCTION__
+ << " binder returned: " << status.exceptionMessage().string();
+ return false;
+ }
+ return true;
+}
+
+bool ImageManagerBinder::RemoveAllImages() {
+ auto status = manager_->removeAllImages();
+ if (!status.isOk()) {
+ LOG(ERROR) << __PRETTY_FUNCTION__
+ << " binder returned: " << status.exceptionMessage().string();
+ return false;
+ }
+ return true;
+}
+
+static android::sp<IGsid> AcquireIGsid(const std::chrono::milliseconds& timeout_ms) {
+ if (android::base::GetProperty("init.svc.gsid", "") != "running") {
+ if (!android::base::SetProperty("ctl.start", "gsid") ||
+ !android::base::WaitForProperty("init.svc.gsid", "running", timeout_ms)) {
+ LOG(ERROR) << "Could not start the gsid service";
+ return nullptr;
+ }
+ // Sleep for 250ms to give the service time to register.
+ usleep(250 * 1000);
+ }
+ auto sm = android::defaultServiceManager();
+ auto name = android::String16(kGsiServiceName);
+ auto service = sm->checkService(name);
+ return android::interface_cast<IGsid>(service);
+}
+
+static android::sp<IGsid> GetGsiService(const std::chrono::milliseconds& timeout_ms) {
+ auto start_time = std::chrono::steady_clock::now();
+
+ std::chrono::milliseconds elapsed = std::chrono::milliseconds::zero();
+ do {
+ if (auto gsid = AcquireIGsid(timeout_ms - elapsed); gsid != nullptr) {
+ return gsid;
+ }
+ auto now = std::chrono::steady_clock::now();
+ elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - start_time);
+ } while (elapsed <= timeout_ms);
+
+ LOG(ERROR) << "Timed out trying to acquire IGsid interface";
+ return nullptr;
+}
+
+std::unique_ptr<IImageManager> IImageManager::Open(const std::string& dir,
+ const std::chrono::milliseconds& timeout_ms) {
+ auto gsid = GetGsiService(timeout_ms);
+ if (!gsid) {
+ return nullptr;
+ }
+
+ android::sp<IGsiService> service;
+ auto status = gsid->getClient(&service);
+ if (!status.isOk() || !service) {
+ LOG(ERROR) << "Could not acquire IGsiService";
+ return nullptr;
+ }
+
+ android::sp<IImageService> manager;
+ status = service->openImageService(dir, &manager);
+ if (!status.isOk() || !manager) {
+ LOG(ERROR) << "Could not acquire IImageManager: " << status.exceptionMessage().string();
+ return nullptr;
+ }
+ return std::make_unique<ImageManagerBinder>(std::move(service), std::move(manager));
+}
+
+} // namespace fiemap
+} // namespace android
+
+#endif // __ANDROID_RECOVERY__
diff --git a/fs_mgr/libfiemap/fiemap_writer.cpp b/fs_mgr/libfiemap/fiemap_writer.cpp
new file mode 100644
index 0000000..b5794d3
--- /dev/null
+++ b/fs_mgr/libfiemap/fiemap_writer.cpp
@@ -0,0 +1,795 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <libfiemap/fiemap_writer.h>
+
+#include <dirent.h>
+#include <fcntl.h>
+#include <linux/fs.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/sysmacros.h>
+#include <sys/types.h>
+#include <sys/vfs.h>
+#include <unistd.h>
+
+#include <limits>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/stringprintf.h>
+#include <android-base/strings.h>
+#include <android-base/unique_fd.h>
+#include <libdm/dm.h>
+#include "utility.h"
+
+namespace android {
+namespace fiemap {
+
+using namespace android::dm;
+
+// We cap the maximum number of extents as a sanity measure.
+static constexpr uint32_t kMaxExtents = 50000;
+
+// TODO: Fallback to using fibmap if FIEMAP_EXTENT_MERGED is set.
+static constexpr const uint32_t kUnsupportedExtentFlags =
+ FIEMAP_EXTENT_UNKNOWN | FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_DELALLOC |
+ FIEMAP_EXTENT_NOT_ALIGNED | FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_DATA_TAIL |
+ FIEMAP_EXTENT_UNWRITTEN | FIEMAP_EXTENT_SHARED | FIEMAP_EXTENT_MERGED;
+
+// Large file support must be enabled.
+static_assert(sizeof(off_t) == sizeof(uint64_t));
+
+static inline void cleanup(const std::string& file_path, bool created) {
+ if (created) {
+ unlink(file_path.c_str());
+ }
+}
+
+static bool ValidateDmTarget(const DeviceMapper::TargetInfo& target) {
+ const auto& entry = target.spec;
+ if (entry.sector_start != 0) {
+ LOG(INFO) << "Stopping at target with non-zero starting sector";
+ return false;
+ }
+
+ auto target_type = DeviceMapper::GetTargetType(entry);
+ if (target_type == "bow" || target_type == "default-key" || target_type == "crypt") {
+ return true;
+ }
+ if (target_type == "linear") {
+ auto pieces = android::base::Split(target.data, " ");
+ if (pieces[1] != "0") {
+ LOG(INFO) << "Stopping at complex linear target with non-zero starting sector: "
+ << pieces[1];
+ return false;
+ }
+ return true;
+ }
+
+ LOG(INFO) << "Stopping at complex target type " << target_type;
+ return false;
+}
+
+static bool DeviceMapperStackPop(const std::string& bdev, std::string* bdev_raw) {
+ *bdev_raw = bdev;
+
+ if (!::android::base::StartsWith(bdev, "dm-")) {
+ // We are at the bottom of the device mapper stack.
+ return true;
+ }
+
+ // Get the device name.
+ auto dm_name_file = "/sys/block/" + bdev + "/dm/name";
+ std::string dm_name;
+ if (!android::base::ReadFileToString(dm_name_file, &dm_name)) {
+ PLOG(ERROR) << "Could not read file: " << dm_name_file;
+ return false;
+ }
+ dm_name = android::base::Trim(dm_name);
+
+ auto& dm = DeviceMapper::Instance();
+ std::vector<DeviceMapper::TargetInfo> table;
+ if (!dm.GetTableInfo(dm_name, &table)) {
+ LOG(ERROR) << "Could not read device-mapper table for " << dm_name << " at " << bdev;
+ return false;
+ }
+
+ // The purpose of libfiemap is to provide an extent-based view into
+ // a file. This is difficult if devices are not layered in a 1:1 manner;
+ // we would have to translate and break up extents based on the actual
+ // block mapping. Since this is too complex, we simply stop processing
+ // the device-mapper stack if we encounter a complex case.
+ //
+ // It is up to the caller to decide whether stopping at a virtual block
+ // device is allowable. In most cases it is not, because we want either
+ // "userdata" or an external volume. It is useful for tests however.
+ // Callers can check by comparing the device number to that of userdata,
+ // or by checking whether is a device-mapper node.
+ if (table.size() > 1) {
+ LOG(INFO) << "Stopping at complex table for " << dm_name << " at " << bdev;
+ return true;
+ }
+ if (!ValidateDmTarget(table[0])) {
+ return true;
+ }
+
+ auto dm_leaf_dir = "/sys/block/" + bdev + "/slaves";
+ auto d = std::unique_ptr<DIR, decltype(&closedir)>(opendir(dm_leaf_dir.c_str()), closedir);
+ if (d == nullptr) {
+ PLOG(ERROR) << "Failed to open: " << dm_leaf_dir;
+ return false;
+ }
+
+ struct dirent* de;
+ uint32_t num_leaves = 0;
+ std::string bdev_next = "";
+ while ((de = readdir(d.get())) != nullptr) {
+ if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) {
+ continue;
+ }
+
+ // We set the first name we find here
+ if (bdev_next.empty()) {
+ bdev_next = de->d_name;
+ }
+ num_leaves++;
+ }
+
+ // if we have more than one leaves, we return immediately. We can't continue to create the
+ // file since we don't know how to write it out using fiemap, so it will be readable via the
+ // underlying block devices later. The reader will also have to construct the same device mapper
+ // target in order read the file out.
+ if (num_leaves > 1) {
+ LOG(ERROR) << "Found " << num_leaves << " leaf block devices under device mapper device "
+ << bdev;
+ return false;
+ }
+
+ // recursively call with the block device we found in order to pop the device mapper stack.
+ return DeviceMapperStackPop(bdev_next, bdev_raw);
+}
+
+bool FiemapWriter::GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path,
+ bool* uses_dm) {
+ struct stat sb;
+ if (stat(file_path.c_str(), &sb)) {
+ PLOG(ERROR) << "Failed to get stat for: " << file_path;
+ return false;
+ }
+
+ std::string bdev;
+ if (!BlockDeviceToName(major(sb.st_dev), minor(sb.st_dev), &bdev)) {
+ LOG(ERROR) << "Failed to get block device name for " << major(sb.st_dev) << ":"
+ << minor(sb.st_dev);
+ return false;
+ }
+
+ std::string bdev_raw;
+ if (!DeviceMapperStackPop(bdev, &bdev_raw)) {
+ LOG(ERROR) << "Failed to get the bottom of the device mapper stack for device: " << bdev;
+ return false;
+ }
+
+ if (uses_dm) {
+ *uses_dm = (bdev_raw != bdev);
+ }
+
+ LOG(DEBUG) << "Popped device (" << bdev_raw << ") from device mapper stack starting with ("
+ << bdev << ")";
+
+ *bdev_path = ::android::base::StringPrintf("/dev/block/%s", bdev_raw.c_str());
+
+ // Make sure we are talking to a block device before calling it a success.
+ if (stat(bdev_path->c_str(), &sb)) {
+ PLOG(ERROR) << "Failed to get stat for block device: " << *bdev_path;
+ return false;
+ }
+
+ if ((sb.st_mode & S_IFMT) != S_IFBLK) {
+ PLOG(ERROR) << "File: " << *bdev_path << " is not a block device";
+ return false;
+ }
+
+ return true;
+}
+
+static bool GetBlockDeviceSize(int bdev_fd, const std::string& bdev_path, uint64_t* bdev_size) {
+ uint64_t size_in_bytes = 0;
+ if (ioctl(bdev_fd, BLKGETSIZE64, &size_in_bytes)) {
+ PLOG(ERROR) << "Failed to get total size for: " << bdev_path;
+ return false;
+ }
+
+ *bdev_size = size_in_bytes;
+
+ return true;
+}
+
+static uint64_t GetFileSize(const std::string& file_path) {
+ struct stat sb;
+ if (stat(file_path.c_str(), &sb)) {
+ PLOG(ERROR) << "Failed to get size for file: " << file_path;
+ return 0;
+ }
+
+ return sb.st_size;
+}
+
+static bool PerformFileChecks(const std::string& file_path, uint64_t file_size, uint64_t* blocksz,
+ uint32_t* fs_type) {
+ struct statfs64 sfs;
+ if (statfs64(file_path.c_str(), &sfs)) {
+ PLOG(ERROR) << "Failed to read file system status at: " << file_path;
+ return false;
+ }
+
+ if (!sfs.f_bsize) {
+ LOG(ERROR) << "Unsupported block size: " << sfs.f_bsize;
+ return false;
+ }
+
+ // Check if the filesystem is of supported types.
+ // Only ext4, f2fs, and vfat are tested and supported.
+ switch (sfs.f_type) {
+ case EXT4_SUPER_MAGIC:
+ case F2FS_SUPER_MAGIC:
+ case MSDOS_SUPER_MAGIC:
+ break;
+ default:
+ LOG(ERROR) << "Unsupported file system type: 0x" << std::hex << sfs.f_type;
+ return false;
+ }
+
+ uint64_t available_bytes = sfs.f_bsize * sfs.f_bavail;
+ if (access(file_path.c_str(), F_OK) != 0 && available_bytes <= file_size) {
+ LOG(ERROR) << "Not enough free space in file system to create file of size : " << file_size;
+ return false;
+ }
+
+ *blocksz = sfs.f_bsize;
+ *fs_type = sfs.f_type;
+ return true;
+}
+
+static bool FallocateFallback(int file_fd, uint64_t block_size, uint64_t file_size,
+ const std::string& file_path,
+ const std::function<bool(uint64_t, uint64_t)>& on_progress) {
+ // Even though this is much faster than writing zeroes, it is still slow
+ // enough that we need to fire the progress callback periodically. To
+ // easily achieve this, we seek in chunks. We use 1000 chunks since
+ // normally we only fire the callback on 1/1000th increments.
+ uint64_t bytes_per_chunk = std::max(file_size / 1000, block_size);
+
+ // Seek just to the end of each chunk and write a single byte, causing
+ // the filesystem to allocate blocks.
+ off_t cursor = 0;
+ off_t end = static_cast<off_t>(file_size);
+ while (cursor < end) {
+ cursor = std::min(static_cast<off_t>(cursor + bytes_per_chunk), end);
+ auto rv = TEMP_FAILURE_RETRY(lseek(file_fd, cursor - 1, SEEK_SET));
+ if (rv < 0) {
+ PLOG(ERROR) << "Failed to lseek " << file_path;
+ return false;
+ }
+ if (rv != cursor - 1) {
+ LOG(ERROR) << "Seek returned wrong offset " << rv << " for file " << file_path;
+ return false;
+ }
+ char buffer[] = {0};
+ if (!android::base::WriteFully(file_fd, buffer, 1)) {
+ PLOG(ERROR) << "Write failed: " << file_path;
+ return false;
+ }
+ if (on_progress && !on_progress(cursor, file_size)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// F2FS-specific ioctl
+// It requires the below kernel commit merged in v4.16-rc1.
+// 1ad71a27124c ("f2fs: add an ioctl to disable GC for specific file")
+// In android-4.4,
+// 56ee1e817908 ("f2fs: updates on v4.16-rc1")
+// In android-4.9,
+// 2f17e34672a8 ("f2fs: updates on v4.16-rc1")
+// In android-4.14,
+// ce767d9a55bc ("f2fs: updates on v4.16-rc1")
+#ifndef F2FS_IOC_SET_PIN_FILE
+#ifndef F2FS_IOCTL_MAGIC
+#define F2FS_IOCTL_MAGIC 0xf5
+#endif
+#define F2FS_IOC_GET_PIN_FILE _IOR(F2FS_IOCTL_MAGIC, 14, __u32)
+#define F2FS_IOC_SET_PIN_FILE _IOW(F2FS_IOCTL_MAGIC, 13, __u32)
+#endif
+
+static bool IsFilePinned(int file_fd, const std::string& file_path, uint32_t fs_type) {
+ if (fs_type != F2FS_SUPER_MAGIC) {
+ // No pinning necessary for ext4 or vfat. The blocks, once allocated,
+ // are expected to be fixed.
+ return true;
+ }
+
+ // f2fs: export FS_NOCOW_FL flag to user
+ uint32_t flags;
+ int error = ioctl(file_fd, FS_IOC_GETFLAGS, &flags);
+ if (error < 0) {
+ if ((errno == ENOTTY) || (errno == ENOTSUP)) {
+ PLOG(ERROR) << "Failed to get flags, not supported by kernel: " << file_path;
+ } else {
+ PLOG(ERROR) << "Failed to get flags: " << file_path;
+ }
+ return false;
+ }
+ if (!(flags & FS_NOCOW_FL)) {
+ return false;
+ }
+
+ // F2FS_IOC_GET_PIN_FILE returns the number of blocks moved.
+ uint32_t moved_blocks_nr;
+ error = ioctl(file_fd, F2FS_IOC_GET_PIN_FILE, &moved_blocks_nr);
+ if (error < 0) {
+ if ((errno == ENOTTY) || (errno == ENOTSUP)) {
+ PLOG(ERROR) << "Failed to get file pin status, not supported by kernel: " << file_path;
+ } else {
+ PLOG(ERROR) << "Failed to get file pin status: " << file_path;
+ }
+ return false;
+ }
+
+ if (moved_blocks_nr) {
+ LOG(WARNING) << moved_blocks_nr << " blocks moved in file " << file_path;
+ }
+ return moved_blocks_nr == 0;
+}
+
+static bool PinFile(int file_fd, const std::string& file_path, uint32_t fs_type) {
+ if (IsFilePinned(file_fd, file_path, fs_type)) {
+ return true;
+ }
+ if (fs_type != F2FS_SUPER_MAGIC) {
+ // No pinning necessary for ext4/msdos. The blocks, once allocated, are
+ // expected to be fixed.
+ return true;
+ }
+
+ uint32_t pin_status = 1;
+ int error = ioctl(file_fd, F2FS_IOC_SET_PIN_FILE, &pin_status);
+ if (error < 0) {
+ if ((errno == ENOTTY) || (errno == ENOTSUP)) {
+ PLOG(ERROR) << "Failed to pin file, not supported by kernel: " << file_path;
+ } else {
+ PLOG(ERROR) << "Failed to pin file: " << file_path;
+ }
+ return false;
+ }
+
+ return true;
+}
+
+// write zeroes in 'blocksz' byte increments until we reach file_size to make sure the data
+// blocks are actually written to by the file system and thus getting rid of the holes in the
+// file.
+static bool WriteZeroes(int file_fd, const std::string& file_path, size_t blocksz,
+ uint64_t file_size,
+ const std::function<bool(uint64_t, uint64_t)>& on_progress) {
+ auto buffer = std::unique_ptr<void, decltype(&free)>(calloc(1, blocksz), free);
+ if (buffer == nullptr) {
+ LOG(ERROR) << "failed to allocate memory for writing file";
+ return false;
+ }
+
+ off64_t offset = lseek64(file_fd, 0, SEEK_SET);
+ if (offset < 0) {
+ PLOG(ERROR) << "Failed to seek at the beginning of : " << file_path;
+ return false;
+ }
+
+ int permille = -1;
+ while (offset < file_size) {
+ if (!::android::base::WriteFully(file_fd, buffer.get(), blocksz)) {
+ PLOG(ERROR) << "Failed to write" << blocksz << " bytes at offset" << offset
+ << " in file " << file_path;
+ return false;
+ }
+
+ offset += blocksz;
+
+ // Don't invoke the callback every iteration - wait until a significant
+ // chunk (here, 1/1000th) of the data has been processed.
+ int new_permille = (static_cast<uint64_t>(offset) * 1000) / file_size;
+ if (new_permille != permille && static_cast<uint64_t>(offset) != file_size) {
+ if (on_progress && !on_progress(offset, file_size)) {
+ return false;
+ }
+ permille = new_permille;
+ }
+ }
+
+ if (lseek64(file_fd, 0, SEEK_SET) < 0) {
+ PLOG(ERROR) << "Failed to reset offset at the beginning of : " << file_path;
+ return false;
+ }
+ return true;
+}
+
+// Reserve space for the file on the file system and write it out to make sure the extents
+// don't come back unwritten. Return from this function with the kernel file offset set to 0.
+// If the filesystem is f2fs, then we also PIN the file on disk to make sure the blocks
+// aren't moved around.
+static bool AllocateFile(int file_fd, const std::string& file_path, uint64_t blocksz,
+ uint64_t file_size, unsigned int fs_type,
+ std::function<bool(uint64_t, uint64_t)> on_progress) {
+ bool need_explicit_writes = true;
+ switch (fs_type) {
+ case EXT4_SUPER_MAGIC:
+ break;
+ case F2FS_SUPER_MAGIC: {
+ bool supported;
+ if (!F2fsPinBeforeAllocate(file_fd, &supported)) {
+ return false;
+ }
+ if (supported) {
+ if (!PinFile(file_fd, file_path, fs_type)) {
+ return false;
+ }
+ need_explicit_writes = false;
+ }
+ break;
+ }
+ case MSDOS_SUPER_MAGIC:
+ // fallocate() is not supported, and not needed, since VFAT does not support holes.
+ // Instead we can perform a much faster allocation.
+ return FallocateFallback(file_fd, blocksz, file_size, file_path, on_progress);
+ default:
+ LOG(ERROR) << "Missing fallocate() support for file system " << fs_type;
+ return false;
+ }
+
+ if (fallocate(file_fd, 0, 0, file_size)) {
+ PLOG(ERROR) << "Failed to allocate space for file: " << file_path << " size: " << file_size;
+ return false;
+ }
+
+ if (need_explicit_writes && !WriteZeroes(file_fd, file_path, blocksz, file_size, on_progress)) {
+ return false;
+ }
+
+ // flush all writes here ..
+ if (fsync(file_fd)) {
+ PLOG(ERROR) << "Failed to synchronize written file:" << file_path;
+ return false;
+ }
+
+ // Send one last progress notification.
+ if (on_progress && !on_progress(file_size, file_size)) {
+ return false;
+ }
+ return true;
+}
+
+bool FiemapWriter::HasPinnedExtents(const std::string& file_path) {
+ android::base::unique_fd fd(open(file_path.c_str(), O_NOFOLLOW | O_CLOEXEC | O_RDONLY));
+ if (fd < 0) {
+ PLOG(ERROR) << "open: " << file_path;
+ return false;
+ }
+
+ struct statfs64 sfs;
+ if (fstatfs64(fd, &sfs)) {
+ PLOG(ERROR) << "fstatfs64: " << file_path;
+ return false;
+ }
+ return IsFilePinned(fd, file_path, sfs.f_type);
+}
+
+static bool CountFiemapExtents(int file_fd, const std::string& file_path, uint32_t* num_extents) {
+ struct fiemap fiemap = {};
+ fiemap.fm_start = 0;
+ fiemap.fm_length = UINT64_MAX;
+ fiemap.fm_flags = FIEMAP_FLAG_SYNC;
+ fiemap.fm_extent_count = 0;
+
+ if (ioctl(file_fd, FS_IOC_FIEMAP, &fiemap)) {
+ PLOG(ERROR) << "Failed to get FIEMAP from the kernel for file: " << file_path;
+ return false;
+ }
+
+ if (num_extents) {
+ *num_extents = fiemap.fm_mapped_extents;
+ }
+ return true;
+}
+
+static bool IsValidExtent(const fiemap_extent* extent, std::string_view file_path) {
+ if (extent->fe_flags & kUnsupportedExtentFlags) {
+ LOG(ERROR) << "Extent at location " << extent->fe_logical << " of file " << file_path
+ << " has unsupported flags";
+ return false;
+ }
+ return true;
+}
+
+static bool IsLastExtent(const fiemap_extent* extent) {
+ if (!(extent->fe_flags & FIEMAP_EXTENT_LAST)) {
+ LOG(ERROR) << "Extents are being received out-of-order";
+ return false;
+ }
+ return true;
+}
+
+static bool FiemapToExtents(struct fiemap* fiemap, std::vector<struct fiemap_extent>* extents,
+ uint32_t num_extents, std::string_view file_path) {
+ if (num_extents == 0) return false;
+
+ const struct fiemap_extent* last_extent = &fiemap->fm_extents[num_extents - 1];
+ if (!IsLastExtent(last_extent)) {
+ LOG(ERROR) << "FIEMAP did not return a final extent for file: " << file_path;
+ return false;
+ }
+
+ // Iterate through each extent, read and make sure its valid before adding it to the vector
+ // merging contiguous extents.
+ fiemap_extent* prev = &fiemap->fm_extents[0];
+ if (!IsValidExtent(prev, file_path)) return false;
+
+ for (uint32_t i = 1; i < num_extents; i++) {
+ fiemap_extent* next = &fiemap->fm_extents[i];
+
+ // Make sure extents are returned in order
+ if (next != last_extent && IsLastExtent(next)) return false;
+
+ // Check if extent's flags are valid
+ if (!IsValidExtent(next, file_path)) return false;
+
+ // Check if the current extent is contiguous with the previous one.
+ // An extent can be combined with its predecessor only if:
+ // 1. There is no physical space between the previous and the current
+ // extent, and
+ // 2. The physical distance between the previous and current extent
+ // corresponds to their logical distance (contiguous mapping).
+ if (prev->fe_physical + prev->fe_length == next->fe_physical &&
+ next->fe_physical - prev->fe_physical == next->fe_logical - prev->fe_logical) {
+ prev->fe_length += next->fe_length;
+ } else {
+ extents->emplace_back(*prev);
+ prev = next;
+ }
+ }
+ extents->emplace_back(*prev);
+
+ return true;
+}
+
+static bool ReadFiemap(int file_fd, const std::string& file_path,
+ std::vector<struct fiemap_extent>* extents) {
+ uint32_t num_extents;
+ if (!CountFiemapExtents(file_fd, file_path, &num_extents)) {
+ return false;
+ }
+ if (num_extents == 0) {
+ LOG(ERROR) << "File " << file_path << " has zero extents";
+ return false;
+ }
+ if (num_extents > kMaxExtents) {
+ LOG(ERROR) << "File has " << num_extents << ", maximum is " << kMaxExtents << ": "
+ << file_path;
+ return false;
+ }
+
+ uint64_t fiemap_size =
+ sizeof(struct fiemap_extent) + num_extents * sizeof(struct fiemap_extent);
+ auto buffer = std::unique_ptr<void, decltype(&free)>(calloc(1, fiemap_size), free);
+ if (buffer == nullptr) {
+ LOG(ERROR) << "Failed to allocate memory for fiemap";
+ return false;
+ }
+
+ struct fiemap* fiemap = reinterpret_cast<struct fiemap*>(buffer.get());
+ fiemap->fm_start = 0;
+ fiemap->fm_length = UINT64_MAX;
+ // make sure file is synced to disk before we read the fiemap
+ fiemap->fm_flags = FIEMAP_FLAG_SYNC;
+ fiemap->fm_extent_count = num_extents;
+
+ if (ioctl(file_fd, FS_IOC_FIEMAP, fiemap)) {
+ PLOG(ERROR) << "Failed to get FIEMAP from the kernel for file: " << file_path;
+ return false;
+ }
+ if (fiemap->fm_mapped_extents != num_extents) {
+ LOG(ERROR) << "FIEMAP returned unexpected extent count (" << num_extents
+ << " expected, got " << fiemap->fm_mapped_extents << ") for file: " << file_path;
+ return false;
+ }
+
+ return FiemapToExtents(fiemap, extents, num_extents, file_path);
+}
+
+static bool ReadFibmap(int file_fd, const std::string& file_path,
+ std::vector<struct fiemap_extent>* extents) {
+ struct stat s;
+ if (fstat(file_fd, &s)) {
+ PLOG(ERROR) << "Failed to stat " << file_path;
+ return false;
+ }
+
+ unsigned int blksize;
+ if (ioctl(file_fd, FIGETBSZ, &blksize) < 0) {
+ PLOG(ERROR) << "Failed to get FIGETBSZ for " << file_path;
+ return false;
+ }
+ if (!blksize) {
+ LOG(ERROR) << "Invalid filesystem block size: " << blksize;
+ return false;
+ }
+
+ uint64_t num_blocks = (s.st_size + blksize - 1) / blksize;
+ if (num_blocks > std::numeric_limits<uint32_t>::max()) {
+ LOG(ERROR) << "Too many blocks for FIBMAP (" << num_blocks << ")";
+ return false;
+ }
+
+ for (uint32_t last_block, block_number = 0; block_number < num_blocks; block_number++) {
+ uint32_t block = block_number;
+ if (ioctl(file_fd, FIBMAP, &block)) {
+ PLOG(ERROR) << "Failed to get FIBMAP for file " << file_path;
+ return false;
+ }
+ if (!block) {
+ LOG(ERROR) << "Logical block " << block_number << " is a hole, which is not supported";
+ return false;
+ }
+
+ if (!extents->empty() && block == last_block + 1) {
+ extents->back().fe_length += blksize;
+ } else {
+ extents->push_back(fiemap_extent{.fe_logical = block_number,
+ .fe_physical = static_cast<uint64_t>(block) * blksize,
+ .fe_length = static_cast<uint64_t>(blksize),
+ .fe_flags = 0});
+ if (extents->size() > kMaxExtents) {
+ LOG(ERROR) << "File has more than " << kMaxExtents << "extents: " << file_path;
+ return false;
+ }
+ }
+ last_block = block;
+ }
+ return true;
+}
+
+FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_size, bool create,
+ std::function<bool(uint64_t, uint64_t)> progress) {
+ // if 'create' is false, open an existing file and do not truncate.
+ int open_flags = O_RDWR | O_CLOEXEC;
+ if (create) {
+ if (access(file_path.c_str(), F_OK) == 0) {
+ LOG(WARNING) << "File " << file_path << " already exists, truncating";
+ }
+ open_flags |= O_CREAT | O_TRUNC;
+ }
+ ::android::base::unique_fd file_fd(
+ TEMP_FAILURE_RETRY(open(file_path.c_str(), open_flags, S_IRUSR | S_IWUSR)));
+ if (file_fd < 0) {
+ PLOG(ERROR) << "Failed to create file at: " << file_path;
+ return nullptr;
+ }
+
+ std::string abs_path;
+ if (!::android::base::Realpath(file_path, &abs_path)) {
+ PLOG(ERROR) << "Invalid file path: " << file_path;
+ cleanup(file_path, create);
+ return nullptr;
+ }
+
+ std::string bdev_path;
+ if (!GetBlockDeviceForFile(abs_path, &bdev_path)) {
+ LOG(ERROR) << "Failed to get block dev path for file: " << file_path;
+ cleanup(abs_path, create);
+ return nullptr;
+ }
+
+ ::android::base::unique_fd bdev_fd(
+ TEMP_FAILURE_RETRY(open(bdev_path.c_str(), O_RDONLY | O_CLOEXEC)));
+ if (bdev_fd < 0) {
+ PLOG(ERROR) << "Failed to open block device: " << bdev_path;
+ cleanup(file_path, create);
+ return nullptr;
+ }
+
+ uint64_t bdevsz;
+ if (!GetBlockDeviceSize(bdev_fd, bdev_path, &bdevsz)) {
+ LOG(ERROR) << "Failed to get block device size for : " << bdev_path;
+ cleanup(file_path, create);
+ return nullptr;
+ }
+
+ if (!create) {
+ file_size = GetFileSize(abs_path);
+ if (file_size == 0) {
+ LOG(ERROR) << "Invalid file size of zero bytes for file: " << abs_path;
+ return nullptr;
+ }
+ }
+
+ uint64_t blocksz;
+ uint32_t fs_type;
+ if (!PerformFileChecks(abs_path, file_size, &blocksz, &fs_type)) {
+ LOG(ERROR) << "Failed to validate file or file system for file:" << abs_path;
+ cleanup(abs_path, create);
+ return nullptr;
+ }
+
+ // Align up to the nearest block size.
+ if (file_size % blocksz) {
+ file_size += blocksz - (file_size % blocksz);
+ }
+
+ if (create) {
+ if (!AllocateFile(file_fd, abs_path, blocksz, file_size, fs_type, std::move(progress))) {
+ LOG(ERROR) << "Failed to allocate file: " << abs_path << " of size: " << file_size
+ << " bytes";
+ cleanup(abs_path, create);
+ return nullptr;
+ }
+ }
+
+ // f2fs may move the file blocks around.
+ if (!PinFile(file_fd, abs_path, fs_type)) {
+ cleanup(abs_path, create);
+ LOG(ERROR) << "Failed to pin the file in storage";
+ return nullptr;
+ }
+
+ // now allocate the FiemapWriter and start setting it up
+ FiemapUniquePtr fmap(new FiemapWriter());
+ switch (fs_type) {
+ case EXT4_SUPER_MAGIC:
+ case F2FS_SUPER_MAGIC:
+ if (!ReadFiemap(file_fd, abs_path, &fmap->extents_)) {
+ LOG(ERROR) << "Failed to read fiemap of file: " << abs_path;
+ cleanup(abs_path, create);
+ return nullptr;
+ }
+ break;
+ case MSDOS_SUPER_MAGIC:
+ if (!ReadFibmap(file_fd, abs_path, &fmap->extents_)) {
+ LOG(ERROR) << "Failed to read fibmap of file: " << abs_path;
+ cleanup(abs_path, create);
+ return nullptr;
+ }
+ break;
+ }
+
+ fmap->file_path_ = abs_path;
+ fmap->bdev_path_ = bdev_path;
+ fmap->file_size_ = file_size;
+ fmap->bdev_size_ = bdevsz;
+ fmap->fs_type_ = fs_type;
+ fmap->block_size_ = blocksz;
+
+ LOG(VERBOSE) << "Successfully created FiemapWriter for file " << abs_path << " on block device "
+ << bdev_path;
+ return fmap;
+}
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/fiemap_writer_test.cpp b/fs_mgr/libfiemap/fiemap_writer_test.cpp
new file mode 100644
index 0000000..4ac7161
--- /dev/null
+++ b/fs_mgr/libfiemap/fiemap_writer_test.cpp
@@ -0,0 +1,541 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <fcntl.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/vfs.h>
+#include <unistd.h>
+
+#include <string>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/stringprintf.h>
+#include <android-base/unique_fd.h>
+#include <gtest/gtest.h>
+#include <libdm/loop_control.h>
+#include <libfiemap/fiemap_writer.h>
+#include <libfiemap/split_fiemap_writer.h>
+
+#include "utility.h"
+
+namespace android {
+namespace fiemap {
+
+using namespace std;
+using namespace std::string_literals;
+using namespace android::fiemap;
+using unique_fd = android::base::unique_fd;
+using LoopDevice = android::dm::LoopDevice;
+
+std::string gTestDir;
+uint64_t testfile_size = 536870912; // default of 512MiB
+size_t gBlockSize = 0;
+
+class FiemapWriterTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
+ testfile = gTestDir + "/"s + tinfo->name();
+ }
+
+ void TearDown() override { unlink(testfile.c_str()); }
+
+ // name of the file we use for testing
+ std::string testfile;
+};
+
+class SplitFiemapTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
+ testfile = gTestDir + "/"s + tinfo->name();
+ }
+
+ void TearDown() override {
+ std::string message;
+ if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) {
+ cerr << "Could not remove all split files: " << message;
+ }
+ }
+
+ // name of the file we use for testing
+ std::string testfile;
+};
+
+TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) {
+ // Try creating a file of size ~100TB but aligned to
+ // 512 byte to make sure block alignment tests don't
+ // fail.
+ FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1099511627997184);
+ EXPECT_EQ(fptr, nullptr);
+ EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
+ EXPECT_EQ(errno, ENOENT);
+}
+
+TEST_F(FiemapWriterTest, CreateUnalignedFile) {
+ // Try creating a file of size 4097 bytes which is guaranteed
+ // to be unaligned to all known block sizes.
+ FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1);
+ ASSERT_NE(fptr, nullptr);
+ ASSERT_EQ(fptr->size(), gBlockSize * 2);
+}
+
+TEST_F(FiemapWriterTest, CheckFilePath) {
+ FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
+ ASSERT_NE(fptr, nullptr);
+ EXPECT_EQ(fptr->size(), gBlockSize);
+ EXPECT_EQ(fptr->file_path(), testfile);
+ EXPECT_EQ(access(testfile.c_str(), F_OK), 0);
+}
+
+TEST_F(FiemapWriterTest, CheckFileSize) {
+ // Create a large-ish file and test that the expected size matches.
+ FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1024 * 1024 * 16);
+ ASSERT_NE(fptr, nullptr);
+
+ struct stat s;
+ ASSERT_EQ(stat(testfile.c_str(), &s), 0);
+ EXPECT_EQ(static_cast<uint64_t>(s.st_size), fptr->size());
+}
+
+TEST_F(FiemapWriterTest, CheckProgress) {
+ std::vector<uint64_t> expected;
+ size_t invocations = 0;
+ auto callback = [&](uint64_t done, uint64_t total) -> bool {
+ if (invocations >= expected.size()) {
+ return false;
+ }
+ EXPECT_EQ(done, expected[invocations]);
+ EXPECT_EQ(total, gBlockSize);
+ invocations++;
+ return true;
+ };
+
+ expected.push_back(gBlockSize);
+
+ auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
+ EXPECT_NE(ptr, nullptr);
+ EXPECT_EQ(invocations, expected.size());
+}
+
+TEST_F(FiemapWriterTest, CheckPinning) {
+ auto ptr = FiemapWriter::Open(testfile, 4096);
+ ASSERT_NE(ptr, nullptr);
+ EXPECT_TRUE(FiemapWriter::HasPinnedExtents(testfile));
+}
+
+TEST_F(FiemapWriterTest, CheckBlockDevicePath) {
+ FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
+ EXPECT_EQ(fptr->size(), gBlockSize);
+ EXPECT_EQ(fptr->bdev_path().find("/dev/block/"), size_t(0));
+ EXPECT_EQ(fptr->bdev_path().find("/dev/block/dm-"), string::npos);
+}
+
+TEST_F(FiemapWriterTest, CheckFileCreated) {
+ FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 32768);
+ ASSERT_NE(fptr, nullptr);
+ unique_fd fd(open(testfile.c_str(), O_RDONLY));
+ EXPECT_GT(fd, -1);
+}
+
+TEST_F(FiemapWriterTest, CheckFileSizeActual) {
+ FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
+ ASSERT_NE(fptr, nullptr);
+
+ struct stat sb;
+ ASSERT_EQ(stat(testfile.c_str(), &sb), 0);
+ EXPECT_GE(sb.st_size, testfile_size);
+}
+
+TEST_F(FiemapWriterTest, CheckFileExtents) {
+ FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
+ ASSERT_NE(fptr, nullptr);
+ EXPECT_GT(fptr->extents().size(), 0);
+}
+
+TEST_F(FiemapWriterTest, ExistingFile) {
+ // Create the file.
+ { ASSERT_NE(FiemapWriter::Open(testfile, gBlockSize), nullptr); }
+ // Test that we can still open it.
+ {
+ auto ptr = FiemapWriter::Open(testfile, 0, false);
+ ASSERT_NE(ptr, nullptr);
+ EXPECT_GT(ptr->extents().size(), 0);
+ }
+}
+
+TEST_F(FiemapWriterTest, FileDeletedOnError) {
+ auto callback = [](uint64_t, uint64_t) -> bool { return false; };
+ auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
+ EXPECT_EQ(ptr, nullptr);
+ EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
+ EXPECT_EQ(errno, ENOENT);
+}
+
+TEST_F(FiemapWriterTest, MaxBlockSize) {
+ ASSERT_GT(DetermineMaximumFileSize(testfile), 0);
+}
+
+TEST_F(FiemapWriterTest, FibmapBlockAddressing) {
+ FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
+ ASSERT_NE(fptr, nullptr);
+
+ switch (fptr->fs_type()) {
+ case F2FS_SUPER_MAGIC:
+ case EXT4_SUPER_MAGIC:
+ // Skip the test for FIEMAP supported filesystems. This is really
+ // because f2fs/ext4 have caches that seem to defeat reading back
+ // directly from the block device, and writing directly is too
+ // dangerous.
+ std::cout << "Skipping test, filesystem does not use FIBMAP\n";
+ return;
+ }
+
+ bool uses_dm;
+ std::string bdev_path;
+ ASSERT_TRUE(FiemapWriter::GetBlockDeviceForFile(testfile, &bdev_path, &uses_dm));
+
+ if (uses_dm) {
+ // We could use a device-mapper wrapper here to bypass encryption, but
+ // really this test is for FIBMAP correctness on VFAT (where encryption
+ // is never used), so we don't bother.
+ std::cout << "Skipping test, block device is metadata encrypted\n";
+ return;
+ }
+
+ std::string data(fptr->size(), '\0');
+ for (size_t i = 0; i < data.size(); i++) {
+ data[i] = 'A' + static_cast<char>(data.size() % 26);
+ }
+
+ {
+ unique_fd fd(open(testfile.c_str(), O_WRONLY | O_CLOEXEC));
+ ASSERT_GE(fd, 0);
+ ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size()));
+ ASSERT_EQ(fsync(fd), 0);
+ }
+
+ ASSERT_FALSE(fptr->extents().empty());
+ const auto& first_extent = fptr->extents()[0];
+
+ unique_fd bdev(open(fptr->bdev_path().c_str(), O_RDONLY | O_CLOEXEC));
+ ASSERT_GE(bdev, 0);
+
+ off_t where = first_extent.fe_physical;
+ ASSERT_EQ(lseek(bdev, where, SEEK_SET), where);
+
+ // Note: this will fail on encrypted folders.
+ std::string actual(data.size(), '\0');
+ ASSERT_GE(first_extent.fe_length, data.size());
+ ASSERT_TRUE(android::base::ReadFully(bdev, actual.data(), actual.size()));
+ EXPECT_EQ(memcmp(actual.data(), data.data(), data.size()), 0);
+}
+
+TEST_F(SplitFiemapTest, Create) {
+ auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
+ ASSERT_NE(ptr, nullptr);
+
+ auto extents = ptr->extents();
+
+ // Destroy the fiemap, closing file handles. This should not delete them.
+ ptr = nullptr;
+
+ std::vector<std::string> files;
+ ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files));
+ for (const auto& path : files) {
+ EXPECT_EQ(access(path.c_str(), F_OK), 0);
+ }
+
+ ASSERT_GE(extents.size(), files.size());
+}
+
+TEST_F(SplitFiemapTest, Open) {
+ {
+ auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
+ ASSERT_NE(ptr, nullptr);
+ }
+
+ auto ptr = SplitFiemap::Open(testfile);
+ ASSERT_NE(ptr, nullptr);
+
+ auto extents = ptr->extents();
+ ASSERT_GE(extents.size(), 24);
+}
+
+TEST_F(SplitFiemapTest, DeleteOnFail) {
+ auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 100, 1);
+ ASSERT_EQ(ptr, nullptr);
+
+ std::string first_file = testfile + ".0001";
+ ASSERT_NE(access(first_file.c_str(), F_OK), 0);
+ ASSERT_EQ(errno, ENOENT);
+ ASSERT_NE(access(testfile.c_str(), F_OK), 0);
+ ASSERT_EQ(errno, ENOENT);
+}
+
+static string ReadSplitFiles(const std::string& base_path, size_t num_files) {
+ std::string result;
+ for (int i = 0; i < num_files; i++) {
+ std::string path = base_path + android::base::StringPrintf(".%04d", i);
+ std::string data;
+ if (!android::base::ReadFileToString(path, &data)) {
+ return {};
+ }
+ result += data;
+ }
+ return result;
+}
+
+TEST_F(SplitFiemapTest, WriteWholeFile) {
+ static constexpr size_t kChunkSize = 32768;
+ static constexpr size_t kSize = kChunkSize * 3;
+ auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
+ ASSERT_NE(ptr, nullptr);
+
+ auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
+ for (size_t i = 0; i < kSize / sizeof(int); i++) {
+ buffer[i] = i;
+ }
+ ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
+
+ std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
+ auto actual = ReadSplitFiles(testfile, 3);
+ ASSERT_EQ(expected.size(), actual.size());
+ EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
+}
+
+TEST_F(SplitFiemapTest, WriteFileInChunks1) {
+ static constexpr size_t kChunkSize = 32768;
+ static constexpr size_t kSize = kChunkSize * 3;
+ auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
+ ASSERT_NE(ptr, nullptr);
+
+ auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
+ for (size_t i = 0; i < kSize / sizeof(int); i++) {
+ buffer[i] = i;
+ }
+
+ // Write in chunks of 1000 (so some writes straddle the boundary of two
+ // files).
+ size_t bytes_written = 0;
+ while (bytes_written < kSize) {
+ size_t to_write = std::min(kSize - bytes_written, (size_t)1000);
+ char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
+ ASSERT_TRUE(ptr->Write(data, to_write));
+ bytes_written += to_write;
+ }
+
+ std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
+ auto actual = ReadSplitFiles(testfile, 3);
+ ASSERT_EQ(expected.size(), actual.size());
+ EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
+}
+
+TEST_F(SplitFiemapTest, WriteFileInChunks2) {
+ static constexpr size_t kChunkSize = 32768;
+ static constexpr size_t kSize = kChunkSize * 3;
+ auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
+ ASSERT_NE(ptr, nullptr);
+
+ auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
+ for (size_t i = 0; i < kSize / sizeof(int); i++) {
+ buffer[i] = i;
+ }
+
+ // Write in chunks of 32KiB so every write is exactly at the end of the
+ // current file.
+ size_t bytes_written = 0;
+ while (bytes_written < kSize) {
+ size_t to_write = std::min(kSize - bytes_written, kChunkSize);
+ char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
+ ASSERT_TRUE(ptr->Write(data, to_write));
+ bytes_written += to_write;
+ }
+
+ std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
+ auto actual = ReadSplitFiles(testfile, 3);
+ ASSERT_EQ(expected.size(), actual.size());
+ EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
+}
+
+TEST_F(SplitFiemapTest, WritePastEnd) {
+ static constexpr size_t kChunkSize = 32768;
+ static constexpr size_t kSize = kChunkSize * 3;
+ auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
+ ASSERT_NE(ptr, nullptr);
+
+ auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
+ for (size_t i = 0; i < kSize / sizeof(int); i++) {
+ buffer[i] = i;
+ }
+ ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
+ ASSERT_FALSE(ptr->Write(buffer.get(), kSize));
+}
+
+class VerifyBlockWritesExt4 : public ::testing::Test {
+ // 2GB Filesystem and 4k block size by default
+ static constexpr uint64_t block_size = 4096;
+ static constexpr uint64_t fs_size = 2147483648;
+
+ protected:
+ void SetUp() override {
+ fs_path = std::string(getenv("TMPDIR")) + "/ext4_2G.img";
+ uint64_t count = fs_size / block_size;
+ std::string dd_cmd =
+ ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
+ " count=%" PRIu64 " > /dev/null 2>&1",
+ fs_path.c_str(), block_size, count);
+ std::string mkfs_cmd =
+ ::android::base::StringPrintf("/system/bin/mkfs.ext4 -q %s", fs_path.c_str());
+ // create mount point
+ mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt";
+ ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0);
+ // create file for the file system
+ int ret = system(dd_cmd.c_str());
+ ASSERT_EQ(ret, 0);
+ // Get and attach a loop device to the filesystem we created
+ LoopDevice loop_dev(fs_path, 10s);
+ ASSERT_TRUE(loop_dev.valid());
+ // create file system
+ ret = system(mkfs_cmd.c_str());
+ ASSERT_EQ(ret, 0);
+
+ // mount the file system
+ ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "ext4", 0, nullptr), 0);
+ }
+
+ void TearDown() override {
+ umount(mntpoint.c_str());
+ rmdir(mntpoint.c_str());
+ unlink(fs_path.c_str());
+ }
+
+ std::string mntpoint;
+ std::string fs_path;
+};
+
+class VerifyBlockWritesF2fs : public ::testing::Test {
+ // 2GB Filesystem and 4k block size by default
+ static constexpr uint64_t block_size = 4096;
+ static constexpr uint64_t fs_size = 2147483648;
+
+ protected:
+ void SetUp() override {
+ fs_path = std::string(getenv("TMPDIR")) + "/f2fs_2G.img";
+ uint64_t count = fs_size / block_size;
+ std::string dd_cmd =
+ ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
+ " count=%" PRIu64 " > /dev/null 2>&1",
+ fs_path.c_str(), block_size, count);
+ std::string mkfs_cmd =
+ ::android::base::StringPrintf("/system/bin/make_f2fs -q %s", fs_path.c_str());
+ // create mount point
+ mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt";
+ ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0);
+ // create file for the file system
+ int ret = system(dd_cmd.c_str());
+ ASSERT_EQ(ret, 0);
+ // Get and attach a loop device to the filesystem we created
+ LoopDevice loop_dev(fs_path, 10s);
+ ASSERT_TRUE(loop_dev.valid());
+ // create file system
+ ret = system(mkfs_cmd.c_str());
+ ASSERT_EQ(ret, 0);
+
+ // mount the file system
+ ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "f2fs", 0, nullptr), 0);
+ }
+
+ void TearDown() override {
+ umount(mntpoint.c_str());
+ rmdir(mntpoint.c_str());
+ unlink(fs_path.c_str());
+ }
+
+ std::string mntpoint;
+ std::string fs_path;
+};
+
+bool DetermineBlockSize() {
+ struct statfs s;
+ if (statfs(gTestDir.c_str(), &s)) {
+ std::cerr << "Could not call statfs: " << strerror(errno) << "\n";
+ return false;
+ }
+ if (!s.f_bsize) {
+ std::cerr << "Invalid block size: " << s.f_bsize << "\n";
+ return false;
+ }
+
+ gBlockSize = s.f_bsize;
+ return true;
+}
+
+} // namespace fiemap
+} // namespace android
+
+using namespace android::fiemap;
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ if (argc > 1 && argv[1] == "-h"s) {
+ cerr << "Usage: [test_dir] [file_size]\n";
+ cerr << "\n";
+ cerr << "Note: test_dir must be a writable, unencrypted directory.\n";
+ exit(EXIT_FAILURE);
+ }
+ ::android::base::InitLogging(argv, ::android::base::StderrLogger);
+
+ std::string root_dir = "/data/local/unencrypted";
+ if (access(root_dir.c_str(), F_OK)) {
+ root_dir = "/data";
+ }
+
+ std::string tempdir = root_dir + "/XXXXXX"s;
+ if (!mkdtemp(tempdir.data())) {
+ cerr << "unable to create tempdir on " << root_dir << "\n";
+ exit(EXIT_FAILURE);
+ }
+ if (!android::base::Realpath(tempdir, &gTestDir)) {
+ cerr << "unable to find realpath for " << tempdir;
+ exit(EXIT_FAILURE);
+ }
+
+ if (argc > 2) {
+ testfile_size = strtoull(argv[2], NULL, 0);
+ if (testfile_size == ULLONG_MAX) {
+ testfile_size = 512 * 1024 * 1024;
+ }
+ }
+
+ if (!DetermineBlockSize()) {
+ exit(EXIT_FAILURE);
+ }
+
+ auto result = RUN_ALL_TESTS();
+
+ std::string cmd = "rm -rf " + gTestDir;
+ system(cmd.c_str());
+
+ return result;
+}
diff --git a/fs_mgr/libfiemap/image_manager.cpp b/fs_mgr/libfiemap/image_manager.cpp
new file mode 100644
index 0000000..fe2018d
--- /dev/null
+++ b/fs_mgr/libfiemap/image_manager.cpp
@@ -0,0 +1,674 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <libfiemap/image_manager.h>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+#include <android-base/strings.h>
+#include <android-base/unique_fd.h>
+#include <ext4_utils/ext4_utils.h>
+#include <fs_mgr/file_wait.h>
+#include <fs_mgr_dm_linear.h>
+#include <libdm/loop_control.h>
+#include <libfiemap/split_fiemap_writer.h>
+
+#include "metadata.h"
+#include "utility.h"
+
+namespace android {
+namespace fiemap {
+
+using namespace std::literals;
+using android::base::unique_fd;
+using android::dm::DeviceMapper;
+using android::dm::DmDeviceState;
+using android::dm::DmTable;
+using android::dm::DmTargetLinear;
+using android::dm::LoopControl;
+using android::fs_mgr::CreateLogicalPartition;
+using android::fs_mgr::CreateLogicalPartitionParams;
+using android::fs_mgr::DestroyLogicalPartition;
+using android::fs_mgr::GetPartitionName;
+
+static constexpr char kTestImageMetadataDir[] = "/metadata/gsi/test";
+
+std::unique_ptr<ImageManager> ImageManager::Open(const std::string& dir_prefix) {
+ auto metadata_dir = "/metadata/gsi/" + dir_prefix;
+ auto data_dir = "/data/gsi/" + dir_prefix;
+ return Open(metadata_dir, data_dir);
+}
+
+std::unique_ptr<ImageManager> ImageManager::Open(const std::string& metadata_dir,
+ const std::string& data_dir) {
+ return std::unique_ptr<ImageManager>(new ImageManager(metadata_dir, data_dir));
+}
+
+ImageManager::ImageManager(const std::string& metadata_dir, const std::string& data_dir)
+ : metadata_dir_(metadata_dir), data_dir_(data_dir) {
+ partition_opener_ = std::make_unique<android::fs_mgr::PartitionOpener>();
+}
+
+std::string ImageManager::GetImageHeaderPath(const std::string& name) {
+ return JoinPaths(data_dir_, name) + ".img";
+}
+
+// The status file has one entry per line, with each entry formatted as one of:
+// dm:<name>
+// loop:<path>
+//
+// This simplifies the process of tearing down a mapping, since we can simply
+// unmap each entry in the order it appears.
+std::string ImageManager::GetStatusFilePath(const std::string& image_name) {
+ return JoinPaths(metadata_dir_, image_name) + ".status";
+}
+
+static std::string GetStatusPropertyName(const std::string& image_name) {
+ // Note: we don't prefix |image_name|, because CreateLogicalPartition won't
+ // prefix the name either. There are no plans to change this at the moment,
+ // consumers of the image API must take care to use globally-unique image
+ // names.
+ return "gsid.mapped_image." + image_name;
+}
+
+void ImageManager::set_partition_opener(std::unique_ptr<IPartitionOpener>&& opener) {
+ partition_opener_ = std::move(opener);
+}
+
+bool ImageManager::IsImageMapped(const std::string& image_name) {
+ auto prop_name = GetStatusPropertyName(image_name);
+ if (android::base::GetProperty(prop_name, "").empty()) {
+ // If mapped in first-stage init, the dm-device will exist but not the
+ // property.
+ auto& dm = DeviceMapper::Instance();
+ return dm.GetState(image_name) != DmDeviceState::INVALID;
+ }
+ return true;
+}
+
+std::vector<std::string> ImageManager::GetAllBackingImages() {
+ std::vector<std::string> images;
+ if (!MetadataExists(metadata_dir_)) {
+ return images;
+ }
+ auto metadata = OpenMetadata(metadata_dir_);
+ if (metadata) {
+ for (auto&& partition : metadata->partitions) {
+ images.push_back(partition.name);
+ }
+ }
+ return images;
+}
+
+bool ImageManager::PartitionExists(const std::string& name) {
+ if (!MetadataExists(metadata_dir_)) {
+ return false;
+ }
+ auto metadata = OpenMetadata(metadata_dir_);
+ if (!metadata) {
+ return false;
+ }
+ return !!FindPartition(*metadata.get(), name);
+}
+
+bool ImageManager::BackingImageExists(const std::string& name) {
+ auto header_file = GetImageHeaderPath(name);
+ return access(header_file.c_str(), F_OK) == 0;
+}
+
+bool ImageManager::CreateBackingImage(const std::string& name, uint64_t size, int flags) {
+ return CreateBackingImage(name, size, flags, nullptr);
+}
+
+static bool IsUnreliablePinningAllowed(const std::string& path) {
+ return android::base::StartsWith(path, "/data/gsi/dsu/") ||
+ android::base::StartsWith(path, "/data/gsi/test/") ||
+ android::base::StartsWith(path, "/data/gsi/ota/test/");
+}
+
+bool ImageManager::CreateBackingImage(const std::string& name, uint64_t size, int flags,
+ std::function<bool(uint64_t, uint64_t)>&& on_progress) {
+ auto data_path = GetImageHeaderPath(name);
+ auto fw = SplitFiemap::Create(data_path, size, 0, on_progress);
+ if (!fw) {
+ return false;
+ }
+
+ bool reliable_pinning;
+ if (!FilesystemHasReliablePinning(data_path, &reliable_pinning)) {
+ return false;
+ }
+ if (!reliable_pinning && !IsUnreliablePinningAllowed(data_path)) {
+ // For historical reasons, we allow unreliable pinning for certain use
+ // cases (DSUs, testing) because the ultimate use case is either
+ // developer-oriented or ephemeral (the intent is to boot immediately
+ // into DSUs). For everything else - such as snapshots/OTAs or adb
+ // remount, we have a higher bar, and require the filesystem to support
+ // proper pinning.
+ LOG(ERROR) << "File system does not have reliable block pinning";
+ SplitFiemap::RemoveSplitFiles(data_path);
+ return false;
+ }
+
+ // Except for testing, we do not allow persisting metadata that references
+ // device-mapper devices. It just doesn't make sense, because the device
+ // numbering may change on reboot. We allow it for testing since the images
+ // are not meant to survive reboot. Outside of tests, this can only happen
+ // if device-mapper is stacked in some complex way not supported by
+ // FiemapWriter.
+ auto device_path = GetDevicePathForFile(fw.get());
+ if (android::base::StartsWith(device_path, "/dev/block/dm-") &&
+ !android::base::StartsWith(metadata_dir_, kTestImageMetadataDir)) {
+ LOG(ERROR) << "Cannot persist images against device-mapper device: " << device_path;
+
+ fw = {};
+ SplitFiemap::RemoveSplitFiles(data_path);
+ return false;
+ }
+
+ bool readonly = !!(flags & CREATE_IMAGE_READONLY);
+ if (!UpdateMetadata(metadata_dir_, name, fw.get(), size, readonly)) {
+ return false;
+ }
+
+ if (flags & CREATE_IMAGE_ZERO_FILL) {
+ if (!ZeroFillNewImage(name, 0)) {
+ DeleteBackingImage(name);
+ return false;
+ }
+ }
+ return true;
+}
+
+bool ImageManager::ZeroFillNewImage(const std::string& name, uint64_t bytes) {
+ auto data_path = GetImageHeaderPath(name);
+
+ // See the comment in MapImageDevice() about how this works.
+ std::string block_device;
+ bool can_use_devicemapper;
+ if (!FiemapWriter::GetBlockDeviceForFile(data_path, &block_device, &can_use_devicemapper)) {
+ LOG(ERROR) << "Could not determine block device for " << data_path;
+ return false;
+ }
+
+ if (!can_use_devicemapper) {
+ // We've backed with loop devices, and since we store files in an
+ // unencrypted folder, the initial zeroes we wrote will suffice.
+ return true;
+ }
+
+ // data is dm-crypt, or FBE + dm-default-key. This means the zeroes written
+ // by libfiemap were encrypted, so we need to map the image in and correct
+ // this.
+ auto device = MappedDevice::Open(this, 10s, name);
+ if (!device) {
+ return false;
+ }
+
+ static constexpr size_t kChunkSize = 4096;
+ std::string zeroes(kChunkSize, '\0');
+
+ uint64_t remaining;
+ if (bytes) {
+ remaining = bytes;
+ } else {
+ remaining = get_block_device_size(device->fd());
+ if (!remaining) {
+ PLOG(ERROR) << "Could not get block device size for " << device->path();
+ return false;
+ }
+ }
+ while (remaining) {
+ uint64_t to_write = std::min(static_cast<uint64_t>(zeroes.size()), remaining);
+ if (!android::base::WriteFully(device->fd(), zeroes.data(),
+ static_cast<size_t>(to_write))) {
+ PLOG(ERROR) << "write failed: " << device->path();
+ return false;
+ }
+ remaining -= to_write;
+ }
+ return true;
+}
+
+bool ImageManager::DeleteBackingImage(const std::string& name) {
+ // For dm-linear devices sitting on top of /data, we cannot risk deleting
+ // the file. The underlying blocks could be reallocated by the filesystem.
+ if (IsImageMapped(name)) {
+ LOG(ERROR) << "Backing image " << name << " is currently mapped to a block device";
+ return false;
+ }
+
+ std::string message;
+ auto header_file = GetImageHeaderPath(name);
+ if (!SplitFiemap::RemoveSplitFiles(header_file, &message)) {
+ // This is fatal, because we don't want to leave these files dangling.
+ LOG(ERROR) << "Error removing image " << name << ": " << message;
+ return false;
+ }
+
+ auto status_file = GetStatusFilePath(name);
+ if (!android::base::RemoveFileIfExists(status_file)) {
+ LOG(ERROR) << "Error removing " << status_file << ": " << message;
+ }
+ return RemoveImageMetadata(metadata_dir_, name);
+}
+
+// Create a block device for an image file, using its extents in its
+// lp_metadata.
+bool ImageManager::MapWithDmLinear(const IPartitionOpener& opener, const std::string& name,
+ const std::chrono::milliseconds& timeout_ms, std::string* path) {
+ // :TODO: refresh extents in metadata file until f2fs is fixed.
+ auto metadata = OpenMetadata(metadata_dir_);
+ if (!metadata) {
+ return false;
+ }
+
+ auto super = android::fs_mgr::GetMetadataSuperBlockDevice(*metadata.get());
+ auto block_device = android::fs_mgr::GetBlockDevicePartitionName(*super);
+
+ CreateLogicalPartitionParams params = {
+ .block_device = block_device,
+ .metadata = metadata.get(),
+ .partition_name = name,
+ .force_writable = true,
+ .timeout_ms = timeout_ms,
+ .partition_opener = &opener,
+ };
+ if (!CreateLogicalPartition(params, path)) {
+ LOG(ERROR) << "Error creating device-mapper node for image " << name;
+ return false;
+ }
+
+ auto status_string = "dm:" + name;
+ auto status_file = GetStatusFilePath(name);
+ if (!android::base::WriteStringToFile(status_string, status_file)) {
+ PLOG(ERROR) << "Could not write status file: " << status_file;
+ DestroyLogicalPartition(name);
+ return false;
+ }
+ return true;
+}
+
+// Helper to create a loop device for a file.
+static bool CreateLoopDevice(LoopControl& control, const std::string& file,
+ const std::chrono::milliseconds& timeout_ms, std::string* path) {
+ static constexpr int kOpenFlags = O_RDWR | O_NOFOLLOW | O_CLOEXEC;
+ android::base::unique_fd file_fd(open(file.c_str(), kOpenFlags));
+ if (file_fd < 0) {
+ PLOG(ERROR) << "Could not open file: " << file;
+ return false;
+ }
+ if (!control.Attach(file_fd, timeout_ms, path)) {
+ LOG(ERROR) << "Could not create loop device for: " << file;
+ return false;
+ }
+ LOG(INFO) << "Created loop device " << *path << " for file " << file;
+ return true;
+}
+
+class AutoDetachLoopDevices final {
+ public:
+ AutoDetachLoopDevices(LoopControl& control, const std::vector<std::string>& devices)
+ : control_(control), devices_(devices), commit_(false) {}
+
+ ~AutoDetachLoopDevices() {
+ if (commit_) return;
+ for (const auto& device : devices_) {
+ control_.Detach(device);
+ }
+ }
+
+ void Commit() { commit_ = true; }
+
+ private:
+ LoopControl& control_;
+ const std::vector<std::string>& devices_;
+ bool commit_;
+};
+
+// If an image is stored across multiple files, this takes a list of loop
+// devices and joins them together using device-mapper.
+bool ImageManager::MapWithLoopDeviceList(const std::vector<std::string>& device_list,
+ const std::string& name,
+ const std::chrono::milliseconds& timeout_ms,
+ std::string* path) {
+ auto metadata = OpenMetadata(metadata_dir_);
+ if (!metadata) {
+ return false;
+ }
+ auto partition = FindPartition(*metadata.get(), name);
+ if (!partition) {
+ LOG(ERROR) << "Could not find image in metadata: " << name;
+ return false;
+ }
+
+ // Since extent lengths are in sector units, the size should be a multiple
+ // of the sector size.
+ uint64_t partition_size = GetPartitionSize(*metadata.get(), *partition);
+ if (partition_size % LP_SECTOR_SIZE != 0) {
+ LOG(ERROR) << "Partition size not sector aligned: " << name << ", " << partition_size
+ << " bytes";
+ return false;
+ }
+
+ DmTable table;
+
+ uint64_t start_sector = 0;
+ uint64_t sectors_needed = partition_size / LP_SECTOR_SIZE;
+ for (const auto& block_device : device_list) {
+ // The final block device must be == partition_size, otherwise we
+ // can't find the AVB footer on verified partitions.
+ static constexpr int kOpenFlags = O_RDWR | O_NOFOLLOW | O_CLOEXEC;
+ unique_fd fd(open(block_device.c_str(), kOpenFlags));
+ if (fd < 0) {
+ PLOG(ERROR) << "Open failed: " << block_device;
+ return false;
+ }
+
+ uint64_t file_size = get_block_device_size(fd);
+ uint64_t file_sectors = file_size / LP_SECTOR_SIZE;
+ uint64_t segment_size = std::min(file_sectors, sectors_needed);
+
+ table.Emplace<DmTargetLinear>(start_sector, segment_size, block_device, 0);
+
+ start_sector += segment_size;
+ sectors_needed -= segment_size;
+ if (sectors_needed == 0) {
+ break;
+ }
+ }
+
+ auto& dm = DeviceMapper::Instance();
+ if (!dm.CreateDevice(name, table, path, timeout_ms)) {
+ LOG(ERROR) << "Could not create device-mapper device over loop set";
+ return false;
+ }
+
+ // Build the status file.
+ std::vector<std::string> lines;
+ lines.emplace_back("dm:" + name);
+ for (const auto& block_device : device_list) {
+ lines.emplace_back("loop:" + block_device);
+ }
+ auto status_message = android::base::Join(lines, "\n");
+ auto status_file = GetStatusFilePath(name);
+ if (!android::base::WriteStringToFile(status_message, status_file)) {
+ PLOG(ERROR) << "Write failed: " << status_file;
+ dm.DeleteDevice(name);
+ return false;
+ }
+ return true;
+}
+
+static bool OptimizeLoopDevices(const std::vector<std::string>& device_list) {
+ for (const auto& device : device_list) {
+ unique_fd fd(open(device.c_str(), O_RDWR | O_CLOEXEC | O_NOFOLLOW));
+ if (fd < 0) {
+ PLOG(ERROR) << "Open failed: " << device;
+ return false;
+ }
+ if (!LoopControl::EnableDirectIo(fd)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Helper to use one or more loop devices around image files.
+bool ImageManager::MapWithLoopDevice(const std::string& name,
+ const std::chrono::milliseconds& timeout_ms,
+ std::string* path) {
+ auto image_header = GetImageHeaderPath(name);
+
+ std::vector<std::string> file_list;
+ if (!SplitFiemap::GetSplitFileList(image_header, &file_list)) {
+ LOG(ERROR) << "Could not get image file list";
+ return false;
+ }
+
+ // Map each image file as a loopback device.
+ LoopControl control;
+ std::vector<std::string> loop_devices;
+ AutoDetachLoopDevices auto_detach(control, loop_devices);
+
+ auto start_time = std::chrono::steady_clock::now();
+ for (const auto& file : file_list) {
+ auto now = std::chrono::steady_clock::now();
+ auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - start_time);
+
+ std::string loop_device;
+ if (!CreateLoopDevice(control, file, timeout_ms - elapsed, &loop_device)) {
+ break;
+ }
+ loop_devices.emplace_back(loop_device);
+ }
+ if (loop_devices.size() != file_list.size()) {
+ // The number of devices will mismatch if CreateLoopDevice() failed.
+ return false;
+ }
+
+ // If OptimizeLoopDevices fails, we'd use double the memory.
+ if (!OptimizeLoopDevices(loop_devices)) {
+ return false;
+ }
+
+ // If there's only one loop device (by far the most common case, splits
+ // will normally only happen on sdcards with FAT32), then just return that
+ // as the block device. Otherwise, we need to use dm-linear to stitch
+ // together all the loop devices we just created.
+ if (loop_devices.size() > 1) {
+ if (!MapWithLoopDeviceList(loop_devices, name, timeout_ms, path)) {
+ return false;
+ }
+ }
+
+ auto status_message = "loop:" + loop_devices.back();
+ auto status_file = GetStatusFilePath(name);
+ if (!android::base::WriteStringToFile(status_message, status_file)) {
+ PLOG(ERROR) << "Write failed: " << status_file;
+ return false;
+ }
+
+ auto_detach.Commit();
+
+ *path = loop_devices.back();
+ return true;
+}
+
+bool ImageManager::MapImageDevice(const std::string& name,
+ const std::chrono::milliseconds& timeout_ms, std::string* path) {
+ if (IsImageMapped(name)) {
+ LOG(ERROR) << "Backing image " << name << " is already mapped";
+ return false;
+ }
+
+ auto image_header = GetImageHeaderPath(name);
+
+ // If there is a device-mapper node wrapping the block device, then we're
+ // able to create another node around it; the dm layer does not carry the
+ // exclusion lock down the stack when a mount occurs.
+ //
+ // If there is no intermediate device-mapper node, then partitions cannot be
+ // opened writable due to sepolicy and exclusivity of having a mounted
+ // filesystem. This should only happen on devices with no encryption, or
+ // devices with FBE and no metadata encryption. For these cases it suffices
+ // to perform normal file writes to /data/gsi (which is unencrypted).
+ std::string block_device;
+ bool can_use_devicemapper;
+ if (!FiemapWriter::GetBlockDeviceForFile(image_header, &block_device, &can_use_devicemapper)) {
+ LOG(ERROR) << "Could not determine block device for " << image_header;
+ return false;
+ }
+
+ if (can_use_devicemapper) {
+ if (!MapWithDmLinear(*partition_opener_.get(), name, timeout_ms, path)) {
+ return false;
+ }
+ } else if (!MapWithLoopDevice(name, timeout_ms, path)) {
+ return false;
+ }
+
+ // Set a property so we remember this is mapped.
+ auto prop_name = GetStatusPropertyName(name);
+ if (!android::base::SetProperty(prop_name, *path)) {
+ UnmapImageDevice(name, true);
+ return false;
+ }
+ return true;
+}
+
+bool ImageManager::MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name,
+ std::string* dev) {
+ std::string ignore_path;
+ if (!MapWithDmLinear(opener, name, {}, &ignore_path)) {
+ return false;
+ }
+
+ auto& dm = DeviceMapper::Instance();
+ if (!dm.GetDeviceString(name, dev)) {
+ return false;
+ }
+ return true;
+}
+
+bool ImageManager::UnmapImageDevice(const std::string& name) {
+ return UnmapImageDevice(name, false);
+}
+
+bool ImageManager::UnmapImageDevice(const std::string& name, bool force) {
+ if (!force && !IsImageMapped(name)) {
+ LOG(ERROR) << "Backing image " << name << " is not mapped";
+ return false;
+ }
+ auto& dm = DeviceMapper::Instance();
+ LoopControl loop;
+
+ std::string status;
+ auto status_file = GetStatusFilePath(name);
+ if (!android::base::ReadFileToString(status_file, &status)) {
+ PLOG(ERROR) << "Read failed: " << status_file;
+ return false;
+ }
+
+ auto lines = android::base::Split(status, "\n");
+ for (const auto& line : lines) {
+ auto pieces = android::base::Split(line, ":");
+ if (pieces.size() != 2) {
+ LOG(ERROR) << "Unknown status line";
+ continue;
+ }
+ if (pieces[0] == "dm") {
+ // Failure to remove a dm node is fatal, since we can't safely
+ // remove the file or loop devices.
+ const auto& name = pieces[1];
+ if (!dm.DeleteDeviceIfExists(name)) {
+ return false;
+ }
+ } else if (pieces[0] == "loop") {
+ // Failure to remove a loop device is not fatal, since we can still
+ // remove the backing file if we want.
+ loop.Detach(pieces[1]);
+ } else {
+ LOG(ERROR) << "Unknown status: " << pieces[0];
+ }
+ }
+
+ std::string message;
+ if (!android::base::RemoveFileIfExists(status_file, &message)) {
+ LOG(ERROR) << "Could not remove " << status_file << ": " << message;
+ }
+
+ auto status_prop = GetStatusPropertyName(name);
+ android::base::SetProperty(status_prop, "");
+ return true;
+}
+
+bool ImageManager::RemoveAllImages() {
+ if (!MetadataExists(metadata_dir_)) {
+ return true;
+ }
+ auto metadata = OpenMetadata(metadata_dir_);
+ if (!metadata) {
+ return RemoveAllMetadata(metadata_dir_);
+ }
+
+ bool ok = true;
+ for (const auto& partition : metadata->partitions) {
+ auto partition_name = GetPartitionName(partition);
+ ok &= DeleteBackingImage(partition_name);
+ }
+ return ok && RemoveAllMetadata(metadata_dir_);
+}
+
+bool ImageManager::Validate() {
+ auto metadata = OpenMetadata(metadata_dir_);
+ if (!metadata) {
+ return false;
+ }
+
+ for (const auto& partition : metadata->partitions) {
+ auto name = GetPartitionName(partition);
+ auto image_path = GetImageHeaderPath(name);
+ auto fiemap = SplitFiemap::Open(image_path);
+ if (!fiemap || !fiemap->HasPinnedExtents()) {
+ LOG(ERROR) << "Image is missing or was moved: " << image_path;
+ return false;
+ }
+ }
+ return true;
+}
+
+std::unique_ptr<MappedDevice> MappedDevice::Open(IImageManager* manager,
+ const std::chrono::milliseconds& timeout_ms,
+ const std::string& name) {
+ std::string path;
+ if (!manager->MapImageDevice(name, timeout_ms, &path)) {
+ return nullptr;
+ }
+
+ auto device = std::unique_ptr<MappedDevice>(new MappedDevice(manager, name, path));
+ if (device->fd() < 0) {
+ return nullptr;
+ }
+ return device;
+}
+
+MappedDevice::MappedDevice(IImageManager* manager, const std::string& name, const std::string& path)
+ : manager_(manager), name_(name), path_(path) {
+ // The device is already mapped; try and open it.
+ fd_.reset(open(path.c_str(), O_RDWR | O_CLOEXEC));
+}
+
+MappedDevice::~MappedDevice() {
+ fd_ = {};
+ manager_->UnmapImageDevice(name_);
+}
+
+bool IImageManager::UnmapImageIfExists(const std::string& name) {
+ // No lock is needed even though this seems to be vulnerable to TOCTOU. If process A
+ // calls MapImageDevice() while process B calls UnmapImageIfExists(), and MapImageDevice()
+ // happens after process B checks IsImageMapped(), it would be as if MapImageDevice() is called
+ // after process B finishes calling UnmapImageIfExists(), resulting the image to be mapped,
+ // which is a reasonable sequence.
+ if (!IsImageMapped(name)) {
+ return true;
+ }
+ return UnmapImageDevice(name);
+}
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/image_test.cpp b/fs_mgr/libfiemap/image_test.cpp
new file mode 100644
index 0000000..f05825c
--- /dev/null
+++ b/fs_mgr/libfiemap/image_test.cpp
@@ -0,0 +1,251 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <chrono>
+#include <iostream>
+#include <thread>
+
+#include <android-base/file.h>
+#include <android-base/properties.h>
+#include <android-base/strings.h>
+#include <android-base/unique_fd.h>
+#include <ext4_utils/ext4_utils.h>
+#include <fs_mgr/file_wait.h>
+#include <gtest/gtest.h>
+#include <libdm/dm.h>
+#include <libfiemap/image_manager.h>
+
+using namespace android::dm;
+using namespace std::literals;
+using android::base::unique_fd;
+using android::fiemap::ImageManager;
+using android::fs_mgr::BlockDeviceInfo;
+using android::fs_mgr::PartitionOpener;
+using android::fs_mgr::WaitForFile;
+
+static std::string gDataPath;
+static std::string gDataMountPath;
+static constexpr char kMetadataPath[] = "/metadata/gsi/test";
+
+static constexpr uint64_t kTestImageSize = 1024 * 1024;
+
+class TestPartitionOpener final : public PartitionOpener {
+ public:
+ android::base::unique_fd Open(const std::string& partition_name, int flags) const override {
+ return PartitionOpener::Open(GetPathForBlockDeviceName(partition_name), flags);
+ }
+ bool GetInfo(const std::string& partition_name, BlockDeviceInfo* info) const override {
+ return PartitionOpener::GetInfo(GetPathForBlockDeviceName(partition_name), info);
+ }
+ std::string GetDeviceString(const std::string& partition_name) const override {
+ return PartitionOpener::GetDeviceString(GetPathForBlockDeviceName(partition_name));
+ }
+
+ private:
+ static std::string GetPathForBlockDeviceName(const std::string& name) {
+ if (android::base::StartsWith(name, "loop") || android::base::StartsWith(name, "dm-")) {
+ return "/dev/block/"s + name;
+ }
+ return name;
+ }
+};
+
+// This fixture is for tests against the device's native configuration.
+class NativeTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ manager_ = ImageManager::Open(kMetadataPath, gDataPath);
+ ASSERT_NE(manager_, nullptr);
+
+ manager_->set_partition_opener(std::make_unique<TestPartitionOpener>());
+
+ const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
+ base_name_ = tinfo->name();
+ }
+
+ void TearDown() override {
+ manager_->UnmapImageDevice(base_name_);
+ manager_->DeleteBackingImage(base_name_);
+ }
+
+ std::string PropertyName() { return "gsid.mapped_image." + base_name_; }
+
+ std::unique_ptr<ImageManager> manager_;
+ std::string base_name_;
+};
+
+TEST_F(NativeTest, CreateAndMap) {
+ ASSERT_TRUE(manager_->CreateBackingImage(base_name_, kTestImageSize, false, nullptr));
+
+ std::string path;
+ ASSERT_TRUE(manager_->MapImageDevice(base_name_, 5s, &path));
+ ASSERT_TRUE(manager_->IsImageMapped(base_name_));
+ ASSERT_EQ(android::base::GetProperty(PropertyName(), ""), path);
+
+ {
+ unique_fd fd(open(path.c_str(), O_RDWR | O_NOFOLLOW | O_CLOEXEC));
+ ASSERT_GE(fd, 0);
+ ASSERT_EQ(get_block_device_size(fd), kTestImageSize);
+ }
+
+ ASSERT_TRUE(manager_->UnmapImageDevice(base_name_));
+ ASSERT_FALSE(manager_->IsImageMapped(base_name_));
+ ASSERT_EQ(android::base::GetProperty(PropertyName(), ""), "");
+}
+
+// This fixture is for tests against a simulated device environment. Rather
+// than use /data, we create an image and then layer a new filesystem within
+// it. Each test then decides how to mount and create layered images. This
+// allows us to test FBE vs FDE configurations.
+class ImageTest : public ::testing::Test {
+ public:
+ ImageTest() : dm_(DeviceMapper::Instance()) {}
+
+ void SetUp() override {
+ manager_ = ImageManager::Open(kMetadataPath, gDataPath);
+ ASSERT_NE(manager_, nullptr);
+
+ manager_->set_partition_opener(std::make_unique<TestPartitionOpener>());
+
+ submanager_ = ImageManager::Open(kMetadataPath + "/mnt"s, gDataPath + "/mnt"s);
+ ASSERT_NE(submanager_, nullptr);
+
+ submanager_->set_partition_opener(std::make_unique<TestPartitionOpener>());
+
+ // Ensure that metadata is cleared in between runs.
+ submanager_->RemoveAllImages();
+ manager_->RemoveAllImages();
+
+ const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
+ base_name_ = tinfo->name();
+ test_image_name_ = base_name_ + "-base";
+ wrapper_device_name_ = base_name_ + "-wrapper";
+
+ ASSERT_TRUE(manager_->CreateBackingImage(base_name_, kTestImageSize * 16, false, nullptr));
+ ASSERT_TRUE(manager_->MapImageDevice(base_name_, 5s, &base_device_));
+ }
+
+ void TearDown() override {
+ submanager_->UnmapImageDevice(test_image_name_);
+ umount(gDataMountPath.c_str());
+ dm_.DeleteDeviceIfExists(wrapper_device_name_);
+ manager_->UnmapImageDevice(base_name_);
+ manager_->DeleteBackingImage(base_name_);
+ }
+
+ protected:
+ bool DoFormat(const std::string& device) {
+ // clang-format off
+ std::vector<std::string> mkfs_args = {
+ "/system/bin/mke2fs",
+ "-F",
+ "-b 4096",
+ "-t ext4",
+ "-m 0",
+ "-O has_journal",
+ device,
+ ">/dev/null",
+ "2>/dev/null",
+ "</dev/null",
+ };
+ // clang-format on
+ auto command = android::base::Join(mkfs_args, " ");
+ return system(command.c_str()) == 0;
+ }
+
+ std::unique_ptr<ImageManager> manager_;
+ std::unique_ptr<ImageManager> submanager_;
+
+ DeviceMapper& dm_;
+ std::string base_name_;
+ std::string base_device_;
+ std::string test_image_name_;
+ std::string wrapper_device_name_;
+};
+
+TEST_F(ImageTest, DirectMount) {
+ ASSERT_TRUE(DoFormat(base_device_));
+ ASSERT_EQ(mount(base_device_.c_str(), gDataMountPath.c_str(), "ext4", 0, nullptr), 0);
+ ASSERT_TRUE(submanager_->CreateBackingImage(test_image_name_, kTestImageSize, false, nullptr));
+
+ std::string path;
+ ASSERT_TRUE(submanager_->MapImageDevice(test_image_name_, 5s, &path));
+ ASSERT_TRUE(android::base::StartsWith(path, "/dev/block/loop"));
+}
+
+TEST_F(ImageTest, IndirectMount) {
+ // Create a simple wrapper around the base device that we'll mount from
+ // instead. This will simulate the code paths for dm-crypt/default-key/bow
+ // and force us to use device-mapper rather than loop devices.
+ uint64_t device_size = 0;
+ {
+ unique_fd fd(open(base_device_.c_str(), O_RDWR | O_CLOEXEC));
+ ASSERT_GE(fd, 0);
+ device_size = get_block_device_size(fd);
+ ASSERT_EQ(device_size, kTestImageSize * 16);
+ }
+ uint64_t num_sectors = device_size / 512;
+
+ auto& dm = DeviceMapper::Instance();
+
+ DmTable table;
+ table.Emplace<DmTargetLinear>(0, num_sectors, base_device_, 0);
+ ASSERT_TRUE(dm.CreateDevice(wrapper_device_name_, table));
+
+ // Format and mount.
+ std::string wrapper_device;
+ ASSERT_TRUE(dm.GetDmDevicePathByName(wrapper_device_name_, &wrapper_device));
+ ASSERT_TRUE(WaitForFile(wrapper_device, 5s));
+ ASSERT_TRUE(DoFormat(wrapper_device));
+ ASSERT_EQ(mount(wrapper_device.c_str(), gDataMountPath.c_str(), "ext4", 0, nullptr), 0);
+
+ ASSERT_TRUE(submanager_->CreateBackingImage(test_image_name_, kTestImageSize, false, nullptr));
+
+ std::string path;
+ ASSERT_TRUE(submanager_->MapImageDevice(test_image_name_, 5s, &path));
+ ASSERT_TRUE(android::base::StartsWith(path, "/dev/block/dm-"));
+}
+
+bool Mkdir(const std::string& path) {
+ if (mkdir(path.c_str(), 0700) && errno != EEXIST) {
+ std::cerr << "Could not mkdir " << path << ": " << strerror(errno) << std::endl;
+ return false;
+ }
+ return true;
+}
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+
+ if (argc >= 2) {
+ gDataPath = argv[1];
+ } else {
+ gDataPath = "/data/gsi/test";
+ }
+ gDataMountPath = gDataPath + "/mnt"s;
+
+ if (!Mkdir(gDataPath) || !Mkdir(kMetadataPath) || !Mkdir(gDataMountPath) ||
+ !Mkdir(kMetadataPath + "/mnt"s)) {
+ return 1;
+ }
+ return RUN_ALL_TESTS();
+}
diff --git a/fs_mgr/libfiemap/include/libfiemap/fiemap_writer.h b/fs_mgr/libfiemap/include/libfiemap/fiemap_writer.h
new file mode 100644
index 0000000..c692265
--- /dev/null
+++ b/fs_mgr/libfiemap/include/libfiemap/fiemap_writer.h
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <linux/fiemap.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <functional>
+#include <string>
+#include <vector>
+
+#include <android-base/unique_fd.h>
+
+namespace android {
+namespace fiemap {
+
+class FiemapWriter;
+using FiemapUniquePtr = std::unique_ptr<FiemapWriter>;
+
+class FiemapWriter final {
+ public:
+ // Factory method for FiemapWriter.
+ // The method returns FiemapUniquePtr that contains all the data necessary to be able to write
+ // to the given file directly using raw block i/o. The optional progress callback will be
+ // invoked, if create is true, while the file is being initialized. It receives the bytes
+ // written and the number of total bytes. If the callback returns false, the operation will
+ // fail.
+ //
+ // Note: when create is true, the file size will be aligned up to the nearest file system
+ // block.
+ static FiemapUniquePtr Open(const std::string& file_path, uint64_t file_size,
+ bool create = true,
+ std::function<bool(uint64_t, uint64_t)> progress = {});
+
+ // Check that a file still has the same extents since it was last opened with FiemapWriter,
+ // assuming the file was not resized outside of FiemapWriter. Returns false either on error
+ // or if the file was not pinned.
+ //
+ // This will always return true on Ext4. On F2FS, it will return true if either of the
+ // following cases are true:
+ // - The file was never pinned.
+ // - The file is pinned and has not been moved by the GC.
+ // Thus, this method should only be called for pinned files (such as those returned by
+ // FiemapWriter::Open).
+ static bool HasPinnedExtents(const std::string& file_path);
+
+ // Returns the underlying block device of a file. This will look past device-mapper layers
+ // as long as each layer would not change block mappings (i.e., dm-crypt, dm-bow, and dm-
+ // default-key tables are okay; dm-linear is not). If a mapping such as dm-linear is found,
+ // it will be returned in place of any physical block device.
+ //
+ // It is the caller's responsibility to check whether the returned block device is acceptable.
+ // Gsid, for example, will only accept /dev/block/by-name/userdata as the bottom device.
+ // Callers can check the device name (dm- or loop prefix), inspect sysfs, or compare the major
+ // number against a boot device.
+ //
+ // If device-mapper nodes were encountered, then |uses_dm| will be set to true.
+ static bool GetBlockDeviceForFile(const std::string& file_path, std::string* bdev_path,
+ bool* uses_dm = nullptr);
+
+ ~FiemapWriter() = default;
+
+ const std::string& file_path() const { return file_path_; };
+ uint64_t size() const { return file_size_; };
+ const std::string& bdev_path() const { return bdev_path_; };
+ uint64_t block_size() const { return block_size_; };
+ const std::vector<struct fiemap_extent>& extents() { return extents_; };
+ uint32_t fs_type() const { return fs_type_; }
+
+ // Non-copyable & Non-movable
+ FiemapWriter(const FiemapWriter&) = delete;
+ FiemapWriter& operator=(const FiemapWriter&) = delete;
+ FiemapWriter& operator=(FiemapWriter&&) = delete;
+ FiemapWriter(FiemapWriter&&) = delete;
+
+ private:
+ // Name of the file managed by this class.
+ std::string file_path_;
+ // Block device on which we have created the file.
+ std::string bdev_path_;
+
+ // Size in bytes of the file this class is writing
+ uint64_t file_size_;
+
+ // total size in bytes of the block device
+ uint64_t bdev_size_;
+
+ // Filesystem type where the file is being created.
+ // See: <uapi/linux/magic.h> for filesystem magic numbers
+ uint32_t fs_type_;
+
+ // block size as reported by the kernel of the underlying block device;
+ uint64_t block_size_;
+
+ // This file's fiemap
+ std::vector<struct fiemap_extent> extents_;
+
+ FiemapWriter() = default;
+};
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/include/libfiemap/image_manager.h b/fs_mgr/libfiemap/include/libfiemap/image_manager.h
new file mode 100644
index 0000000..5ff4628
--- /dev/null
+++ b/fs_mgr/libfiemap/include/libfiemap/image_manager.h
@@ -0,0 +1,185 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#pragma once
+
+#include <stdint.h>
+
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <string>
+
+#include <android-base/unique_fd.h>
+#include <liblp/partition_opener.h>
+
+namespace android {
+namespace fiemap {
+
+class IImageManager {
+ public:
+ using IPartitionOpener = android::fs_mgr::IPartitionOpener;
+
+ virtual ~IImageManager() {}
+
+ // When linking to libfiemap_binder, the Open() call will use binder.
+ // Otherwise, the Open() call will use the ImageManager implementation
+ // below.
+ static std::unique_ptr<IImageManager> Open(const std::string& dir_prefix,
+ const std::chrono::milliseconds& timeout_ms);
+
+ // Flags for CreateBackingImage().
+ static constexpr int CREATE_IMAGE_DEFAULT = 0x0;
+ static constexpr int CREATE_IMAGE_READONLY = 0x1;
+ static constexpr int CREATE_IMAGE_ZERO_FILL = 0x2;
+
+ // Create an image that can be mapped as a block-device. If |force_zero_fill|
+ // is true, the image will be zero-filled. Otherwise, the initial content
+ // of the image is undefined. If zero-fill is requested, and the operation
+ // cannot be completed, the image will be deleted and this function will
+ // return false.
+ virtual bool CreateBackingImage(const std::string& name, uint64_t size, int flags) = 0;
+
+ // Delete an image created with CreateBackingImage. Its entry will be
+ // removed from the associated lp_metadata file.
+ virtual bool DeleteBackingImage(const std::string& name) = 0;
+
+ // Create a block device for an image previously created with
+ // CreateBackingImage. This will wait for at most |timeout_ms| milliseconds
+ // for |path| to be available, and will return false if not available in
+ // the requested time. If |timeout_ms| is zero, this is NOT guaranteed to
+ // return true. A timeout of 10s is recommended.
+ //
+ // Note that snapshots created with a readonly flag are always mapped
+ // writable. The flag is persisted in the lp_metadata file however, so if
+ // fs_mgr::CreateLogicalPartition(s) is used, the flag will be respected.
+ virtual bool MapImageDevice(const std::string& name,
+ const std::chrono::milliseconds& timeout_ms, std::string* path) = 0;
+
+ // Unmap a block device previously mapped with mapBackingImage.
+ virtual bool UnmapImageDevice(const std::string& name) = 0;
+
+ // Returns true whether the named backing image exists.
+ virtual bool BackingImageExists(const std::string& name) = 0;
+
+ // Returns true if the specified image is mapped to a device.
+ virtual bool IsImageMapped(const std::string& name) = 0;
+
+ // Map an image using device-mapper. This is not available over binder, and
+ // is intended only for first-stage init. The returned device is a major:minor
+ // device string.
+ virtual bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name,
+ std::string* dev) = 0;
+
+ // Get all backing image names.
+ virtual std::vector<std::string> GetAllBackingImages() = 0;
+
+ // Writes |bytes| zeros to |name| file. If |bytes| is 0, then the
+ // whole file if filled with zeros.
+ virtual bool ZeroFillNewImage(const std::string& name, uint64_t bytes) = 0;
+
+ // Find and remove all images and metadata for this manager.
+ virtual bool RemoveAllImages() = 0;
+
+ virtual bool UnmapImageIfExists(const std::string& name);
+};
+
+class ImageManager final : public IImageManager {
+ public:
+ // Return an ImageManager for the given metadata and data directories. Both
+ // directories must already exist.
+ static std::unique_ptr<ImageManager> Open(const std::string& metadata_dir,
+ const std::string& data_dir);
+
+ // Helper function that derives the metadata and data dirs given a single
+ // prefix.
+ static std::unique_ptr<ImageManager> Open(const std::string& dir_prefix);
+
+ // Methods that must be implemented from IImageManager.
+ bool CreateBackingImage(const std::string& name, uint64_t size, int flags) override;
+ bool DeleteBackingImage(const std::string& name) override;
+ bool MapImageDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms,
+ std::string* path) override;
+ bool UnmapImageDevice(const std::string& name) override;
+ bool BackingImageExists(const std::string& name) override;
+ bool IsImageMapped(const std::string& name) override;
+ bool MapImageWithDeviceMapper(const IPartitionOpener& opener, const std::string& name,
+ std::string* dev) override;
+ bool RemoveAllImages() override;
+
+ std::vector<std::string> GetAllBackingImages();
+ // Same as CreateBackingImage, but provides a progress notification.
+ bool CreateBackingImage(const std::string& name, uint64_t size, int flags,
+ std::function<bool(uint64_t, uint64_t)>&& on_progress);
+
+ // Returns true if the named partition exists. This does not check the
+ // consistency of the backing image/data file.
+ bool PartitionExists(const std::string& name);
+
+ // Validates that all images still have pinned extents. This will be removed
+ // once b/134588268 is fixed.
+ bool Validate();
+
+ void set_partition_opener(std::unique_ptr<IPartitionOpener>&& opener);
+
+ // Writes |bytes| zeros at the beginning of the passed image
+ bool ZeroFillNewImage(const std::string& name, uint64_t bytes);
+
+ private:
+ ImageManager(const std::string& metadata_dir, const std::string& data_dir);
+ std::string GetImageHeaderPath(const std::string& name);
+ std::string GetStatusFilePath(const std::string& image_name);
+ bool MapWithLoopDevice(const std::string& name, const std::chrono::milliseconds& timeout_ms,
+ std::string* path);
+ bool MapWithLoopDeviceList(const std::vector<std::string>& device_list, const std::string& name,
+ const std::chrono::milliseconds& timeout_ms, std::string* path);
+ bool MapWithDmLinear(const IPartitionOpener& opener, const std::string& name,
+ const std::chrono::milliseconds& timeout_ms, std::string* path);
+ bool UnmapImageDevice(const std::string& name, bool force);
+
+ ImageManager(const ImageManager&) = delete;
+ ImageManager& operator=(const ImageManager&) = delete;
+ ImageManager& operator=(ImageManager&&) = delete;
+ ImageManager(ImageManager&&) = delete;
+
+ std::string metadata_dir_;
+ std::string data_dir_;
+ std::unique_ptr<IPartitionOpener> partition_opener_;
+};
+
+// RAII helper class for mapping and opening devices with an ImageManager.
+class MappedDevice final {
+ public:
+ static std::unique_ptr<MappedDevice> Open(IImageManager* manager,
+ const std::chrono::milliseconds& timeout_ms,
+ const std::string& name);
+
+ ~MappedDevice();
+
+ int fd() const { return fd_; }
+ const std::string& path() const { return path_; }
+
+ protected:
+ MappedDevice(IImageManager* manager, const std::string& name, const std::string& path);
+
+ IImageManager* manager_;
+ std::string name_;
+ std::string path_;
+ android::base::unique_fd fd_;
+};
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/include/libfiemap/split_fiemap_writer.h b/fs_mgr/libfiemap/include/libfiemap/split_fiemap_writer.h
new file mode 100644
index 0000000..feffb3d
--- /dev/null
+++ b/fs_mgr/libfiemap/include/libfiemap/split_fiemap_writer.h
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <android-base/unique_fd.h>
+
+#include "fiemap_writer.h"
+
+namespace android {
+namespace fiemap {
+
+// Wrapper around FiemapWriter that is able to split images across files if
+// necessary.
+class SplitFiemap final {
+ public:
+ using ProgressCallback = std::function<bool(uint64_t, uint64_t)>;
+
+ // Create a new split fiemap file. If |max_piece_size| is 0, the number of
+ // pieces will be determined automatically by detecting the filesystem.
+ // Otherwise, the file will be split evenly (with the remainder in the
+ // final file).
+ static std::unique_ptr<SplitFiemap> Create(const std::string& file_path, uint64_t file_size,
+ uint64_t max_piece_size,
+ ProgressCallback progress = {});
+
+ // Open an existing split fiemap file.
+ static std::unique_ptr<SplitFiemap> Open(const std::string& file_path);
+
+ ~SplitFiemap();
+
+ // Return a list of all files created for a split file.
+ static bool GetSplitFileList(const std::string& file_path, std::vector<std::string>* list);
+
+ // Destroy all components of a split file. If the root file does not exist,
+ // this returns true and does not report an error.
+ static bool RemoveSplitFiles(const std::string& file_path, std::string* message = nullptr);
+
+ // Return whether all components of a split file still have pinned extents.
+ bool HasPinnedExtents() const;
+
+ // Helper method for writing data that spans files. Note there is no seek
+ // method (yet); this starts at 0 and increments the position by |bytes|.
+ bool Write(const void* data, uint64_t bytes);
+
+ // Flush all writes to all split files.
+ bool Flush();
+
+ const std::vector<struct fiemap_extent>& extents();
+ uint32_t block_size() const;
+ uint64_t size() const { return total_size_; }
+ const std::string& bdev_path() const;
+
+ // Non-copyable & Non-movable
+ SplitFiemap(const SplitFiemap&) = delete;
+ SplitFiemap& operator=(const SplitFiemap&) = delete;
+ SplitFiemap& operator=(SplitFiemap&&) = delete;
+ SplitFiemap(SplitFiemap&&) = delete;
+
+ private:
+ SplitFiemap() = default;
+ void AddFile(FiemapUniquePtr&& file);
+
+ bool creating_ = false;
+ std::string list_file_;
+ std::vector<FiemapUniquePtr> files_;
+ std::vector<struct fiemap_extent> extents_;
+ uint64_t total_size_ = 0;
+
+ // Most recently open file and position for Write().
+ size_t cursor_index_ = 0;
+ uint64_t cursor_file_pos_ = 0;
+ android::base::unique_fd cursor_fd_;
+};
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/metadata.cpp b/fs_mgr/libfiemap/metadata.cpp
new file mode 100644
index 0000000..597efe9
--- /dev/null
+++ b/fs_mgr/libfiemap/metadata.cpp
@@ -0,0 +1,196 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "metadata.h"
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <liblp/builder.h>
+
+#include "utility.h"
+
+namespace android {
+namespace fiemap {
+
+using namespace android::fs_mgr;
+
+static constexpr uint32_t kMaxMetadataSize = 256 * 1024;
+
+std::string GetMetadataFile(const std::string& metadata_dir) {
+ return JoinPaths(metadata_dir, "lp_metadata");
+}
+
+bool MetadataExists(const std::string& metadata_dir) {
+ auto metadata_file = GetMetadataFile(metadata_dir);
+ return access(metadata_file.c_str(), F_OK) == 0;
+}
+
+std::unique_ptr<LpMetadata> OpenMetadata(const std::string& metadata_dir) {
+ auto metadata_file = GetMetadataFile(metadata_dir);
+ auto metadata = ReadFromImageFile(metadata_file);
+ if (!metadata) {
+ LOG(ERROR) << "Could not read metadata file " << metadata_file;
+ return nullptr;
+ }
+ return metadata;
+}
+
+// :TODO: overwrite on create if open fails
+std::unique_ptr<MetadataBuilder> OpenOrCreateMetadata(const std::string& metadata_dir,
+ SplitFiemap* file) {
+ auto metadata_file = GetMetadataFile(metadata_dir);
+
+ PartitionOpener opener;
+ std::unique_ptr<MetadataBuilder> builder;
+ if (access(metadata_file.c_str(), R_OK)) {
+ if (errno != ENOENT) {
+ PLOG(ERROR) << "access " << metadata_file << " failed:";
+ return nullptr;
+ }
+
+ auto data_device = GetDevicePathForFile(file);
+
+ BlockDeviceInfo device_info;
+ if (!opener.GetInfo(data_device, &device_info)) {
+ LOG(ERROR) << "Could not read partition: " << data_device;
+ return nullptr;
+ }
+
+ std::vector<BlockDeviceInfo> block_devices = {device_info};
+ auto super_name = android::base::Basename(data_device);
+ builder = MetadataBuilder::New(block_devices, super_name, kMaxMetadataSize, 1);
+ } else {
+ auto metadata = OpenMetadata(metadata_dir);
+ if (!metadata) {
+ return nullptr;
+ }
+ builder = MetadataBuilder::New(*metadata.get(), &opener);
+ }
+
+ if (!builder) {
+ LOG(ERROR) << "Could not create metadata builder";
+ return nullptr;
+ }
+ return builder;
+}
+
+bool SaveMetadata(MetadataBuilder* builder, const std::string& metadata_dir) {
+ auto exported = builder->Export();
+ if (!exported) {
+ LOG(ERROR) << "Unable to export new metadata";
+ return false;
+ }
+
+ // If there are no more partitions in the metadata, just delete the file.
+ auto metadata_file = GetMetadataFile(metadata_dir);
+ if (exported->partitions.empty() && android::base::RemoveFileIfExists(metadata_file)) {
+ return true;
+ }
+ if (!WriteToImageFile(metadata_file, *exported.get())) {
+ LOG(ERROR) << "Unable to save new metadata";
+ return false;
+ }
+ return true;
+}
+
+bool RemoveAllMetadata(const std::string& dir) {
+ auto metadata_file = GetMetadataFile(dir);
+ return android::base::RemoveFileIfExists(metadata_file);
+}
+
+bool FillPartitionExtents(MetadataBuilder* builder, Partition* partition, SplitFiemap* file,
+ uint64_t partition_size) {
+ auto block_device = android::base::Basename(GetDevicePathForFile(file));
+
+ uint64_t sectors_needed = partition_size / LP_SECTOR_SIZE;
+ for (const auto& extent : file->extents()) {
+ if (extent.fe_length % LP_SECTOR_SIZE != 0) {
+ LOG(ERROR) << "Extent is not sector-aligned: " << extent.fe_length;
+ return false;
+ }
+ if (extent.fe_physical % LP_SECTOR_SIZE != 0) {
+ LOG(ERROR) << "Extent physical sector is not sector-aligned: " << extent.fe_physical;
+ return false;
+ }
+
+ uint64_t num_sectors =
+ std::min(static_cast<uint64_t>(extent.fe_length / LP_SECTOR_SIZE), sectors_needed);
+ if (!num_sectors || !sectors_needed) {
+ // This should never happen, but we include it just in case. It would
+ // indicate that the last filesystem block had multiple extents.
+ LOG(WARNING) << "FiemapWriter allocated extra blocks";
+ break;
+ }
+
+ uint64_t physical_sector = extent.fe_physical / LP_SECTOR_SIZE;
+ if (!builder->AddLinearExtent(partition, block_device, num_sectors, physical_sector)) {
+ LOG(ERROR) << "Could not add extent to lp metadata";
+ return false;
+ }
+
+ sectors_needed -= num_sectors;
+ }
+ return true;
+}
+
+bool RemoveImageMetadata(const std::string& metadata_dir, const std::string& partition_name) {
+ if (!MetadataExists(metadata_dir)) {
+ return true;
+ }
+ auto metadata = OpenMetadata(metadata_dir);
+ if (!metadata) {
+ return false;
+ }
+
+ PartitionOpener opener;
+ auto builder = MetadataBuilder::New(*metadata.get(), &opener);
+ if (!builder) {
+ return false;
+ }
+ builder->RemovePartition(partition_name);
+ return SaveMetadata(builder.get(), metadata_dir);
+}
+
+bool UpdateMetadata(const std::string& metadata_dir, const std::string& partition_name,
+ SplitFiemap* file, uint64_t partition_size, bool readonly) {
+ auto builder = OpenOrCreateMetadata(metadata_dir, file);
+ if (!builder) {
+ return false;
+ }
+ auto partition = builder->FindPartition(partition_name);
+ if (!partition) {
+ int attrs = 0;
+ if (readonly) attrs |= LP_PARTITION_ATTR_READONLY;
+
+ if ((partition = builder->AddPartition(partition_name, attrs)) == nullptr) {
+ LOG(ERROR) << "Could not add partition " << partition_name << " to metadata";
+ return false;
+ }
+ }
+ partition->RemoveExtents();
+
+ if (!FillPartitionExtents(builder.get(), partition, file, partition_size)) {
+ return false;
+ }
+ return SaveMetadata(builder.get(), metadata_dir);
+}
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/metadata.h b/fs_mgr/libfiemap/metadata.h
new file mode 100644
index 0000000..f0ce23e
--- /dev/null
+++ b/fs_mgr/libfiemap/metadata.h
@@ -0,0 +1,36 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <stdint.h>
+
+#include <memory>
+#include <string>
+
+#include <libfiemap/split_fiemap_writer.h>
+#include <liblp/liblp.h>
+
+namespace android {
+namespace fiemap {
+
+bool MetadataExists(const std::string& metadata_dir);
+std::unique_ptr<android::fs_mgr::LpMetadata> OpenMetadata(const std::string& metadata_dir);
+bool UpdateMetadata(const std::string& metadata_dir, const std::string& partition_name,
+ SplitFiemap* file, uint64_t partition_size, bool readonly);
+bool RemoveImageMetadata(const std::string& metadata_dir, const std::string& partition_name);
+bool RemoveAllMetadata(const std::string& dir);
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/passthrough.cpp b/fs_mgr/libfiemap/passthrough.cpp
new file mode 100644
index 0000000..1ccd9a0
--- /dev/null
+++ b/fs_mgr/libfiemap/passthrough.cpp
@@ -0,0 +1,29 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <libfiemap/image_manager.h>
+
+namespace android {
+namespace fiemap {
+
+std::unique_ptr<IImageManager> IImageManager::Open(const std::string& dir_prefix,
+ const std::chrono::milliseconds& timeout_ms) {
+ (void)timeout_ms;
+ return ImageManager::Open(dir_prefix);
+}
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/split_fiemap_writer.cpp b/fs_mgr/libfiemap/split_fiemap_writer.cpp
new file mode 100644
index 0000000..cc54f20
--- /dev/null
+++ b/fs_mgr/libfiemap/split_fiemap_writer.cpp
@@ -0,0 +1,303 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <libfiemap/split_fiemap_writer.h>
+
+#include <fcntl.h>
+#include <stdint.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/stringprintf.h>
+#include <android-base/strings.h>
+#include <android-base/unique_fd.h>
+
+#include "utility.h"
+
+namespace android {
+namespace fiemap {
+
+using android::base::unique_fd;
+
+// We use a four-digit suffix at the end of filenames.
+static const size_t kMaxFilePieces = 500;
+
+std::unique_ptr<SplitFiemap> SplitFiemap::Create(const std::string& file_path, uint64_t file_size,
+ uint64_t max_piece_size,
+ ProgressCallback progress) {
+ if (!file_size) {
+ LOG(ERROR) << "Cannot create a fiemap for a 0-length file: " << file_path;
+ return nullptr;
+ }
+
+ if (!max_piece_size) {
+ max_piece_size = DetermineMaximumFileSize(file_path);
+ if (!max_piece_size) {
+ LOG(ERROR) << "Could not determine maximum file size for " << file_path;
+ return nullptr;
+ }
+ }
+
+ // Remove any existing file.
+ RemoveSplitFiles(file_path);
+
+ // Call |progress| only when the total percentage would significantly change.
+ int permille = -1;
+ uint64_t total_bytes_written = 0;
+ auto on_progress = [&](uint64_t written, uint64_t) -> bool {
+ uint64_t actual_written = total_bytes_written + written;
+ int new_permille = (actual_written * 1000) / file_size;
+ if (new_permille != permille && actual_written < file_size) {
+ if (progress && !progress(actual_written, file_size)) {
+ return false;
+ }
+ permille = new_permille;
+ }
+ return true;
+ };
+
+ std::unique_ptr<SplitFiemap> out(new SplitFiemap());
+ out->creating_ = true;
+ out->list_file_ = file_path;
+
+ // Create the split files.
+ uint64_t remaining_bytes = file_size;
+ while (remaining_bytes) {
+ if (out->files_.size() >= kMaxFilePieces) {
+ LOG(ERROR) << "Requested size " << file_size << " created too many split files";
+ return nullptr;
+ }
+ std::string chunk_path =
+ android::base::StringPrintf("%s.%04d", file_path.c_str(), (int)out->files_.size());
+ uint64_t chunk_size = std::min(max_piece_size, remaining_bytes);
+ auto writer = FiemapWriter::Open(chunk_path, chunk_size, true, on_progress);
+ if (!writer) {
+ return nullptr;
+ }
+
+ // To make sure the alignment doesn't create too much inconsistency, we
+ // account the *actual* size, not the requested size.
+ total_bytes_written += writer->size();
+
+ // writer->size() is block size aligned and could be bigger than remaining_bytes
+ // If remaining_bytes is bigger, set remaining_bytes to 0 to avoid underflow error.
+ remaining_bytes = remaining_bytes > writer->size() ? (remaining_bytes - writer->size()) : 0;
+
+ out->AddFile(std::move(writer));
+ }
+
+ // Create the split file list.
+ unique_fd fd(open(out->list_file_.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0660));
+ if (fd < 0) {
+ PLOG(ERROR) << "Failed to open " << file_path;
+ return nullptr;
+ }
+
+ for (const auto& writer : out->files_) {
+ std::string line = android::base::Basename(writer->file_path()) + "\n";
+ if (!android::base::WriteFully(fd, line.data(), line.size())) {
+ PLOG(ERROR) << "Write failed " << file_path;
+ return nullptr;
+ }
+ }
+
+ // Unset this bit, so we don't unlink on destruction.
+ out->creating_ = false;
+ return out;
+}
+
+std::unique_ptr<SplitFiemap> SplitFiemap::Open(const std::string& file_path) {
+ std::vector<std::string> files;
+ if (!GetSplitFileList(file_path, &files)) {
+ return nullptr;
+ }
+
+ std::unique_ptr<SplitFiemap> out(new SplitFiemap());
+ out->list_file_ = file_path;
+
+ for (const auto& file : files) {
+ auto writer = FiemapWriter::Open(file, 0, false);
+ if (!writer) {
+ // Error was logged in Open().
+ return nullptr;
+ }
+ out->AddFile(std::move(writer));
+ }
+ return out;
+}
+
+bool SplitFiemap::GetSplitFileList(const std::string& file_path, std::vector<std::string>* list) {
+ // This is not the most efficient thing, but it is simple and recovering
+ // the fiemap/fibmap is much more expensive.
+ std::string contents;
+ if (!android::base::ReadFileToString(file_path, &contents, true)) {
+ PLOG(ERROR) << "Error reading file: " << file_path;
+ return false;
+ }
+
+ std::vector<std::string> names = android::base::Split(contents, "\n");
+ std::string dir = android::base::Dirname(file_path);
+ for (const auto& name : names) {
+ if (!name.empty()) {
+ list->emplace_back(dir + "/" + name);
+ }
+ }
+ return true;
+}
+
+bool SplitFiemap::RemoveSplitFiles(const std::string& file_path, std::string* message) {
+ // Early exit if this does not exist, and do not report an error.
+ if (access(file_path.c_str(), F_OK) && errno == ENOENT) {
+ return true;
+ }
+
+ bool ok = true;
+ std::vector<std::string> files;
+ if (GetSplitFileList(file_path, &files)) {
+ for (const auto& file : files) {
+ ok &= android::base::RemoveFileIfExists(file, message);
+ }
+ }
+ ok &= android::base::RemoveFileIfExists(file_path, message);
+ return ok;
+}
+
+bool SplitFiemap::HasPinnedExtents() const {
+ for (const auto& file : files_) {
+ if (!FiemapWriter::HasPinnedExtents(file->file_path())) {
+ return false;
+ }
+ }
+ return true;
+}
+
+const std::vector<struct fiemap_extent>& SplitFiemap::extents() {
+ if (extents_.empty()) {
+ for (const auto& file : files_) {
+ const auto& extents = file->extents();
+ extents_.insert(extents_.end(), extents.begin(), extents.end());
+ }
+ }
+ return extents_;
+}
+
+bool SplitFiemap::Write(const void* data, uint64_t bytes) {
+ // Open the current file.
+ FiemapWriter* file = files_[cursor_index_].get();
+
+ const uint8_t* data_ptr = reinterpret_cast<const uint8_t*>(data);
+ uint64_t bytes_remaining = bytes;
+ while (bytes_remaining) {
+ // How many bytes can we write into the current file?
+ uint64_t file_bytes_left = file->size() - cursor_file_pos_;
+ if (!file_bytes_left) {
+ if (cursor_index_ == files_.size() - 1) {
+ LOG(ERROR) << "write past end of file requested";
+ return false;
+ }
+
+ // No space left in the current file, but we have more files to
+ // use, so prep the next one.
+ cursor_fd_ = {};
+ cursor_file_pos_ = 0;
+ file = files_[++cursor_index_].get();
+ file_bytes_left = file->size();
+ }
+
+ // Open the current file if it's not open.
+ if (cursor_fd_ < 0) {
+ cursor_fd_.reset(open(file->file_path().c_str(), O_CLOEXEC | O_WRONLY));
+ if (cursor_fd_ < 0) {
+ PLOG(ERROR) << "open failed: " << file->file_path();
+ return false;
+ }
+ CHECK(cursor_file_pos_ == 0);
+ }
+
+ if (!FiemapWriter::HasPinnedExtents(file->file_path())) {
+ LOG(ERROR) << "file is no longer pinned: " << file->file_path();
+ return false;
+ }
+
+ uint64_t bytes_to_write = std::min(file_bytes_left, bytes_remaining);
+ if (!android::base::WriteFully(cursor_fd_, data_ptr, bytes_to_write)) {
+ PLOG(ERROR) << "write failed: " << file->file_path();
+ return false;
+ }
+ data_ptr += bytes_to_write;
+ bytes_remaining -= bytes_to_write;
+ cursor_file_pos_ += bytes_to_write;
+ }
+
+ // If we've reached the end of the current file, close it for sanity.
+ if (cursor_file_pos_ == file->size()) {
+ cursor_fd_ = {};
+ }
+ return true;
+}
+
+bool SplitFiemap::Flush() {
+ for (const auto& file : files_) {
+ unique_fd fd(open(file->file_path().c_str(), O_RDONLY | O_CLOEXEC));
+ if (fd < 0) {
+ PLOG(ERROR) << "open failed: " << file->file_path();
+ return false;
+ }
+ if (fsync(fd)) {
+ PLOG(ERROR) << "fsync failed: " << file->file_path();
+ return false;
+ }
+ }
+ return true;
+}
+
+SplitFiemap::~SplitFiemap() {
+ if (!creating_) {
+ return;
+ }
+
+ // We failed to finish creating, so unlink everything.
+ unlink(list_file_.c_str());
+ for (auto&& file : files_) {
+ std::string path = file->file_path();
+ file = nullptr;
+
+ unlink(path.c_str());
+ }
+}
+
+void SplitFiemap::AddFile(FiemapUniquePtr&& file) {
+ total_size_ += file->size();
+ files_.emplace_back(std::move(file));
+}
+
+uint32_t SplitFiemap::block_size() const {
+ return files_[0]->block_size();
+}
+
+const std::string& SplitFiemap::bdev_path() const {
+ return files_[0]->bdev_path();
+}
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/testdata/file_32k b/fs_mgr/libfiemap/testdata/file_32k
new file mode 100644
index 0000000..12f3be4
--- /dev/null
+++ b/fs_mgr/libfiemap/testdata/file_32k
Binary files differ
diff --git a/fs_mgr/libfiemap/testdata/file_4k b/fs_mgr/libfiemap/testdata/file_4k
new file mode 100644
index 0000000..08e7df1
--- /dev/null
+++ b/fs_mgr/libfiemap/testdata/file_4k
Binary files differ
diff --git a/fs_mgr/libfiemap/testdata/unaligned_file b/fs_mgr/libfiemap/testdata/unaligned_file
new file mode 100644
index 0000000..c107c26
--- /dev/null
+++ b/fs_mgr/libfiemap/testdata/unaligned_file
Binary files differ
diff --git a/fs_mgr/libfiemap/utility.cpp b/fs_mgr/libfiemap/utility.cpp
new file mode 100644
index 0000000..955e544
--- /dev/null
+++ b/fs_mgr/libfiemap/utility.cpp
@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "utility.h"
+
+#include <stdint.h>
+#include <sys/stat.h>
+#include <sys/sysmacros.h>
+#include <sys/types.h>
+#include <sys/vfs.h>
+#include <unistd.h>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/stringprintf.h>
+#include <android-base/strings.h>
+#include <libfiemap/fiemap_writer.h>
+
+namespace android {
+namespace fiemap {
+
+using namespace std::string_literals;
+using android::base::unique_fd;
+
+static constexpr char kUserdataDevice[] = "/dev/block/by-name/userdata";
+
+uint64_t DetermineMaximumFileSize(const std::string& file_path) {
+ // Create the smallest file possible (one block).
+ auto writer = FiemapWriter::Open(file_path, 1);
+ if (!writer) {
+ return 0;
+ }
+
+ uint64_t result = 0;
+ switch (writer->fs_type()) {
+ case EXT4_SUPER_MAGIC:
+ // The minimum is 16GiB, so just report that. If we wanted we could parse the
+ // superblock and figure out if 64-bit support is enabled.
+ result = 17179869184ULL;
+ break;
+ case F2FS_SUPER_MAGIC:
+ // Formula is from https://www.kernel.org/doc/Documentation/filesystems/f2fs.txt
+ // 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB.
+ result = 4329690886144ULL;
+ break;
+ case MSDOS_SUPER_MAGIC:
+ // 4GB-1, which we want aligned to the block size.
+ result = 4294967295;
+ result -= (result % writer->block_size());
+ break;
+ default:
+ LOG(ERROR) << "Unknown file system type: " << writer->fs_type();
+ break;
+ }
+
+ // Close and delete the temporary file.
+ writer = nullptr;
+ unlink(file_path.c_str());
+
+ return result;
+}
+
+// Given a SplitFiemap, this returns a device path that will work during first-
+// stage init (i.e., its path can be found by InitRequiredDevices).
+std::string GetDevicePathForFile(SplitFiemap* file) {
+ auto bdev_path = file->bdev_path();
+
+ struct stat userdata, given;
+ if (!stat(bdev_path.c_str(), &given) && !stat(kUserdataDevice, &userdata)) {
+ if (S_ISBLK(given.st_mode) && S_ISBLK(userdata.st_mode) &&
+ given.st_rdev == userdata.st_rdev) {
+ return kUserdataDevice;
+ }
+ }
+ return bdev_path;
+}
+
+std::string JoinPaths(const std::string& dir, const std::string& file) {
+ if (android::base::EndsWith(dir, "/")) {
+ return dir + file;
+ }
+ return dir + "/" + file;
+}
+
+bool F2fsPinBeforeAllocate(int file_fd, bool* supported) {
+ struct stat st;
+ if (fstat(file_fd, &st) < 0) {
+ PLOG(ERROR) << "stat failed";
+ return false;
+ }
+ std::string bdev;
+ if (!BlockDeviceToName(major(st.st_dev), minor(st.st_dev), &bdev)) {
+ LOG(ERROR) << "Failed to get block device name for " << major(st.st_dev) << ":"
+ << minor(st.st_dev);
+ return false;
+ }
+
+ std::string contents;
+ std::string feature_file = "/sys/fs/f2fs/" + bdev + "/features";
+ if (!android::base::ReadFileToString(feature_file, &contents)) {
+ PLOG(ERROR) << "read failed: " << feature_file;
+ return false;
+ }
+ contents = android::base::Trim(contents);
+
+ auto features = android::base::Split(contents, ", ");
+ auto iter = std::find(features.begin(), features.end(), "pin_file"s);
+ *supported = (iter != features.end());
+ return true;
+}
+
+bool BlockDeviceToName(uint32_t major, uint32_t minor, std::string* bdev_name) {
+ // The symlinks in /sys/dev/block point to the block device node under /sys/device/..
+ // The directory name in the target corresponds to the name of the block device. We use
+ // that to extract the block device name.
+ // e.g for block device name 'ram0', there exists a symlink named '1:0' in /sys/dev/block as
+ // follows.
+ // 1:0 -> ../../devices/virtual/block/ram0
+ std::string sysfs_path = ::android::base::StringPrintf("/sys/dev/block/%u:%u", major, minor);
+ std::string sysfs_bdev;
+
+ if (!::android::base::Readlink(sysfs_path, &sysfs_bdev)) {
+ PLOG(ERROR) << "Failed to read link at: " << sysfs_path;
+ return false;
+ }
+
+ *bdev_name = ::android::base::Basename(sysfs_bdev);
+ // Paranoid sanity check to make sure we just didn't get the
+ // input in return as-is.
+ if (sysfs_bdev == *bdev_name) {
+ LOG(ERROR) << "Malformed symlink for block device: " << sysfs_bdev;
+ return false;
+ }
+
+ return true;
+}
+
+bool FilesystemHasReliablePinning(const std::string& file, bool* supported) {
+ struct statfs64 sfs;
+ if (statfs64(file.c_str(), &sfs)) {
+ PLOG(ERROR) << "statfs failed: " << file;
+ return false;
+ }
+ if (sfs.f_type != F2FS_SUPER_MAGIC) {
+ *supported = true;
+ return true;
+ }
+
+ unique_fd fd(open(file.c_str(), O_RDONLY | O_CLOEXEC));
+ if (fd < 0) {
+ PLOG(ERROR) << "open failed: " << file;
+ return false;
+ }
+ return F2fsPinBeforeAllocate(fd, supported);
+}
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libfiemap/utility.h b/fs_mgr/libfiemap/utility.h
new file mode 100644
index 0000000..24ebc57
--- /dev/null
+++ b/fs_mgr/libfiemap/utility.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#include <string>
+
+#include <libfiemap/split_fiemap_writer.h>
+
+namespace android {
+namespace fiemap {
+
+// Given a file that will be created, determine the maximum size its containing
+// filesystem allows. Note this is a theoretical maximum size; free space is
+// ignored entirely.
+uint64_t DetermineMaximumFileSize(const std::string& file_path);
+
+// Given a SplitFiemap, this returns a device path that will work during first-
+// stage init (i.e., its path can be found by InitRequiredDevices).
+std::string GetDevicePathForFile(android::fiemap::SplitFiemap* file);
+
+// Combine two path components into a single path.
+std::string JoinPaths(const std::string& dir, const std::string& file);
+
+// Given a file within an F2FS filesystem, return whether or not the filesystem
+// supports the "pin_file" feature, which requires pinning before fallocation.
+bool F2fsPinBeforeAllocate(int file_fd, bool* supported);
+
+// Given a major/minor device number, return its canonical name such that
+// /dev/block/<name> resolves to the device.
+bool BlockDeviceToName(uint32_t major, uint32_t minor, std::string* bdev_name);
+
+// This is the same as F2fsPinBeforeAllocate, however, it will return true
+// (and supported = true) for non-f2fs filesystems. It is intended to be used
+// in conjunction with ImageManager to reject image requests for reliable use
+// cases (such as snapshots or adb remount).
+bool FilesystemHasReliablePinning(const std::string& file, bool* supported);
+
+} // namespace fiemap
+} // namespace android
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h
index 5738b96..445e6db 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h
@@ -198,6 +198,13 @@
// - other states indicating an error has occurred
UpdateState InitiateMergeAndWait();
+ // Wait for the merge if rebooted into the new slot. Does NOT initiate a
+ // merge. If the merge has not been initiated (but should be), wait.
+ // Returns:
+ // - true there is no merge or merge finishes
+ // - false indicating an error has occurred
+ bool WaitForMerge();
+
// Find the status of the current update, if any.
//
// |progress| depends on the returned status:
@@ -496,6 +503,10 @@
// as a sanity check.
bool EnsureNoOverflowSnapshot(LockedFile* lock);
+ enum class Slot { Unknown, Source, Target };
+ friend std::ostream& operator<<(std::ostream& os, SnapshotManager::Slot slot);
+ Slot GetCurrentSlot();
+
std::string gsid_dir_;
std::string metadata_dir_;
std::unique_ptr<IDeviceInfo> device_;
diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp
index f38db43..a0ec068 100644
--- a/fs_mgr/libsnapshot/snapshot.cpp
+++ b/fs_mgr/libsnapshot/snapshot.cpp
@@ -74,6 +74,7 @@
using namespace std::string_literals;
static constexpr char kBootIndicatorPath[] = "/metadata/ota/snapshot-boot";
+static constexpr auto kUpdateStateCheckInterval = 2s;
// Note: IImageManager is an incomplete type in the header, so the default
// destructor doesn't work.
@@ -171,14 +172,9 @@
if (state == UpdateState::Unverified) {
// We completed an update, but it can still be canceled if we haven't booted into it.
- auto boot_file = GetSnapshotBootIndicatorPath();
- std::string contents;
- if (!android::base::ReadFileToString(boot_file, &contents)) {
- PLOG(WARNING) << "Cannot read " << boot_file << ", proceed to canceling the update:";
- return RemoveAllUpdateState(file.get());
- }
- if (device_->GetSlotSuffix() == contents) {
- LOG(INFO) << "Canceling a previously completed update";
+ auto slot = GetCurrentSlot();
+ if (slot != Slot::Target) {
+ LOG(INFO) << "Canceling previously completed updates (if any)";
return RemoveAllUpdateState(file.get());
}
}
@@ -186,6 +182,19 @@
return true;
}
+SnapshotManager::Slot SnapshotManager::GetCurrentSlot() {
+ auto boot_file = GetSnapshotBootIndicatorPath();
+ std::string contents;
+ if (!android::base::ReadFileToString(boot_file, &contents)) {
+ PLOG(WARNING) << "Cannot read " << boot_file;
+ return Slot::Unknown;
+ }
+ if (device_->GetSlotSuffix() == contents) {
+ return Slot::Source;
+ }
+ return Slot::Target;
+}
+
bool SnapshotManager::RemoveAllUpdateState(LockedFile* lock) {
if (!RemoveAllSnapshots(lock)) {
LOG(ERROR) << "Could not remove all snapshots";
@@ -505,15 +514,9 @@
return false;
}
- std::string old_slot;
- auto boot_file = GetSnapshotBootIndicatorPath();
- if (!android::base::ReadFileToString(boot_file, &old_slot)) {
- LOG(ERROR) << "Could not determine the previous slot; aborting merge";
- return false;
- }
- auto new_slot = device_->GetSlotSuffix();
- if (new_slot == old_slot) {
- LOG(ERROR) << "Device cannot merge while booting off old slot " << old_slot;
+ auto slot = GetCurrentSlot();
+ if (slot != Slot::Target) {
+ LOG(ERROR) << "Device cannot merge while not booting from new slot";
return false;
}
@@ -729,7 +732,7 @@
// This wait is not super time sensitive, so we have a relatively
// low polling frequency.
- std::this_thread::sleep_for(2s);
+ std::this_thread::sleep_for(kUpdateStateCheckInterval);
}
}
@@ -1097,13 +1100,11 @@
}
bool SnapshotManager::HandleCancelledUpdate(LockedFile* lock) {
- std::string old_slot;
- auto boot_file = GetSnapshotBootIndicatorPath();
- if (!android::base::ReadFileToString(boot_file, &old_slot)) {
- PLOG(ERROR) << "Unable to read the snapshot indicator file: " << boot_file;
+ auto slot = GetCurrentSlot();
+ if (slot == Slot::Unknown) {
return false;
}
- if (device_->GetSlotSuffix() != old_slot) {
+ if (slot == Slot::Target) {
// We're booted into the target slot, which means we just rebooted
// after applying the update.
if (!HandleCancelledUpdateOnNewSlot(lock)) {
@@ -1271,14 +1272,9 @@
// ultimately we'll fail to boot. Why not make it a fatal error and have
// the reason be clearer? Because the indicator file still exists, and
// if this was FATAL, reverting to the old slot would be broken.
- std::string old_slot;
- auto boot_file = GetSnapshotBootIndicatorPath();
- if (!android::base::ReadFileToString(boot_file, &old_slot)) {
- PLOG(ERROR) << "Unable to read the snapshot indicator file: " << boot_file;
- return false;
- }
- if (device_->GetSlotSuffix() == old_slot) {
- LOG(INFO) << "Detected slot rollback, will not mount snapshots.";
+ auto slot = GetCurrentSlot();
+ if (slot != Slot::Target) {
+ LOG(INFO) << "Not booting from new slot. Will not mount snapshots.";
return false;
}
@@ -2156,6 +2152,17 @@
return ok;
}
+std::ostream& operator<<(std::ostream& os, SnapshotManager::Slot slot) {
+ switch (slot) {
+ case SnapshotManager::Slot::Unknown:
+ return os << "unknown";
+ case SnapshotManager::Slot::Source:
+ return os << "source";
+ case SnapshotManager::Slot::Target:
+ return os << "target";
+ }
+}
+
bool SnapshotManager::Dump(std::ostream& os) {
// Don't actually lock. Dump() is for debugging purposes only, so it is okay
// if it is racy.
@@ -2166,11 +2173,8 @@
ss << "Update state: " << ReadUpdateState(file.get()) << std::endl;
- auto boot_file = GetSnapshotBootIndicatorPath();
- std::string boot_indicator;
- if (android::base::ReadFileToString(boot_file, &boot_indicator)) {
- ss << "Boot indicator: old slot = " << boot_indicator << std::endl;
- }
+ ss << "Current slot: " << device_->GetSlotSuffix() << std::endl;
+ ss << "Boot indicator: booting from " << GetCurrentSlot() << " slot" << std::endl;
bool ok = true;
std::vector<std::string> snapshots;
@@ -2238,6 +2242,21 @@
return state;
}
+bool SnapshotManager::WaitForMerge() {
+ LOG(INFO) << "Waiting for any previous merge request to complete. "
+ << "This can take up to several minutes.";
+ while (true) {
+ auto state = ProcessUpdateState();
+ if (state == UpdateState::Unverified && GetCurrentSlot() == Slot::Target) {
+ LOG(INFO) << "Wait for merge to be initiated.";
+ std::this_thread::sleep_for(kUpdateStateCheckInterval);
+ continue;
+ }
+ LOG(INFO) << "Wait for merge exits with state " << state;
+ return state == UpdateState::None || state == UpdateState::MergeCompleted;
+ }
+}
+
bool SnapshotManager::HandleImminentDataWipe(const std::function<void()>& callback) {
if (!device_->IsRecovery()) {
LOG(ERROR) << "Data wipes are only allowed in recovery.";
@@ -2283,11 +2302,9 @@
//
// Since the rollback is inevitable, we don't treat a HAL failure
// as an error here.
- std::string old_slot;
- auto boot_file = GetSnapshotBootIndicatorPath();
- if (android::base::ReadFileToString(boot_file, &old_slot) &&
- device_->GetSlotSuffix() != old_slot) {
- LOG(ERROR) << "Reverting to slot " << old_slot << " since update will be deleted.";
+ auto slot = GetCurrentSlot();
+ if (slot == Slot::Target) {
+ LOG(ERROR) << "Reverting to old slot since update will be deleted.";
device_->SetSlotAsUnbootable(slot_number);
}
break;
diff --git a/fs_mgr/libsnapshot/snapshot_test.cpp b/fs_mgr/libsnapshot/snapshot_test.cpp
index 964b21a..0f5af14 100644
--- a/fs_mgr/libsnapshot/snapshot_test.cpp
+++ b/fs_mgr/libsnapshot/snapshot_test.cpp
@@ -921,6 +921,25 @@
return AssertionSuccess();
}
+ // Create fake install operations to grow the COW device size.
+ void AddOperation(PartitionUpdate* partition_update, uint64_t size_bytes = 0) {
+ auto e = partition_update->add_operations()->add_dst_extents();
+ e->set_start_block(0);
+ if (size_bytes == 0) {
+ size_bytes = GetSize(partition_update);
+ }
+ e->set_num_blocks(size_bytes / manifest_.block_size());
+ }
+
+ void AddOperationForPartitions(std::vector<PartitionUpdate*> partitions = {}) {
+ if (partitions.empty()) {
+ partitions = {sys_, vnd_, prd_};
+ }
+ for (auto* partition : partitions) {
+ AddOperation(partition);
+ }
+ }
+
std::unique_ptr<TestPartitionOpener> opener_;
DeltaArchiveManifest manifest_;
std::unique_ptr<MetadataBuilder> src_;
@@ -948,12 +967,7 @@
SetSize(vnd_, partition_size);
SetSize(prd_, partition_size);
- // Create fake install operations to grow the COW device size.
- for (auto& partition : {sys_, vnd_, prd_}) {
- auto e = partition->add_operations()->add_dst_extents();
- e->set_start_block(0);
- e->set_num_blocks(GetSize(partition) / manifest_.block_size());
- }
+ AddOperationForPartitions();
// Execute the update.
ASSERT_TRUE(sm->BeginUpdate());
@@ -1089,12 +1103,7 @@
ASSERT_TRUE(sm->BeginUpdate());
ASSERT_TRUE(sm->UnmapUpdateSnapshot("sys_b"));
- // Create fake install operations to grow the COW device size.
- for (auto& partition : {sys_, vnd_, prd_}) {
- auto e = partition->add_operations()->add_dst_extents();
- e->set_start_block(0);
- e->set_num_blocks(GetSize(partition) / manifest_.block_size());
- }
+ AddOperationForPartitions();
ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_));
@@ -1239,10 +1248,8 @@
group_->set_size(kRetrofitGroupSize);
for (auto* partition : {sys_, vnd_, prd_}) {
SetSize(partition, 2_MiB);
- auto* e = partition->add_operations()->add_dst_extents();
- e->set_start_block(0);
- e->set_num_blocks(2_MiB / manifest_.block_size());
}
+ AddOperationForPartitions();
ASSERT_TRUE(sm->BeginUpdate());
ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_));
@@ -1285,9 +1292,7 @@
}
// Add operations for sys. The whole device is written.
- auto e = sys_->add_operations()->add_dst_extents();
- e->set_start_block(0);
- e->set_num_blocks(GetSize(sys_) / manifest_.block_size());
+ AddOperation(sys_);
// Execute the update.
ASSERT_TRUE(sm->BeginUpdate());
@@ -1477,10 +1482,7 @@
const auto block_size = manifest_.block_size();
SetSize(sys_, partition_size);
-
- auto e = sys_->add_operations()->add_dst_extents();
- e->set_start_block(0);
- e->set_num_blocks(data_size / block_size);
+ AddOperation(sys_, data_size);
// Set hastree extents.
sys_->mutable_hash_tree_data_extent()->set_start_block(0);
@@ -1525,9 +1527,7 @@
const auto actual_write_size = GetSize(sys_);
const auto declared_write_size = actual_write_size - 1_MiB;
- auto e = sys_->add_operations()->add_dst_extents();
- e->set_start_block(0);
- e->set_num_blocks(declared_write_size / manifest_.block_size());
+ AddOperation(sys_, declared_write_size);
// Execute the update.
ASSERT_TRUE(sm->BeginUpdate());
@@ -1546,6 +1546,45 @@
<< "FinishedSnapshotWrites should detect overflow of CoW device.";
}
+TEST_F(SnapshotUpdateTest, WaitForMerge) {
+ AddOperationForPartitions();
+
+ // Execute the update.
+ ASSERT_TRUE(sm->BeginUpdate());
+ ASSERT_TRUE(sm->CreateUpdateSnapshots(manifest_));
+
+ // Write some data to target partitions.
+ for (const auto& name : {"sys_b", "vnd_b", "prd_b"}) {
+ ASSERT_TRUE(WriteSnapshotAndHash(name));
+ }
+
+ ASSERT_TRUE(sm->FinishedSnapshotWrites());
+
+ // Simulate shutting down the device.
+ ASSERT_TRUE(UnmapAll());
+
+ // After reboot, init does first stage mount.
+ {
+ auto init = SnapshotManager::NewForFirstStageMount(new TestDeviceInfo(fake_super, "_b"));
+ ASSERT_NE(nullptr, init);
+ ASSERT_TRUE(init->CreateLogicalAndSnapshotPartitions("super"));
+ }
+
+ auto new_sm = SnapshotManager::New(new TestDeviceInfo(fake_super, "_b"));
+ ASSERT_NE(nullptr, new_sm);
+
+ auto waiter = std::async(std::launch::async, [&new_sm] { return new_sm->WaitForMerge(); });
+ ASSERT_EQ(std::future_status::timeout, waiter.wait_for(1s))
+ << "WaitForMerge should block when not initiated";
+
+ auto merger =
+ std::async(std::launch::async, [&new_sm] { return new_sm->InitiateMergeAndWait(); });
+ // Small images, so should be merged pretty quickly.
+ ASSERT_EQ(std::future_status::ready, waiter.wait_for(3s)) << "WaitForMerge did not finish";
+ ASSERT_TRUE(waiter.get());
+ ASSERT_THAT(merger.get(), AnyOf(UpdateState::None, UpdateState::MergeCompleted));
+}
+
class FlashAfterUpdateTest : public SnapshotUpdateTest,
public WithParamInterface<std::tuple<uint32_t, bool>> {
public:
diff --git a/rootdir/init.rc b/rootdir/init.rc
index c2c9df3..b89c45e 100644
--- a/rootdir/init.rc
+++ b/rootdir/init.rc
@@ -421,9 +421,6 @@
# Once everything is setup, no need to modify /.
# The bind+remount combination allows this to work in containers.
mount rootfs rootfs / remount bind ro nodev
- # Mount default storage into root namespace
- mount none /mnt/runtime/default /storage bind rec
- mount none none /storage slave rec
# Make sure /sys/kernel/debug (if present) is labeled properly
# Note that tracefs may be mounted under debug, so we need to cross filesystems
@@ -721,6 +718,22 @@
chown root system /dev/fscklogs/log
chmod 0770 /dev/fscklogs/log
+# Switch between sdcardfs and FUSE depending on persist property
+# TODO: Move this to ro property before launch because FDE devices
+# interact with persistent properties differently during boot
+on zygote-start && property:persist.sys.fuse=true
+ # Mount default storage into root namespace
+ mount none /mnt/user/0 /storage bind rec
+ mount none none /storage slave rec
+on zygote-start && property:persist.sys.fuse=false
+ # Mount default storage into root namespace
+ mount none /mnt/runtime/default /storage bind rec
+ mount none none /storage slave rec
+on zygote-start && property:persist.sys.fuse=""
+ # Mount default storage into root namespace
+ mount none /mnt/runtime/default /storage bind rec
+ mount none none /storage slave rec
+
# It is recommended to put unnecessary data/ initialization from post-fs-data
# to start-zygote in device's init.rc to unblock zygote start.
on zygote-start && property:ro.crypto.state=unencrypted