Merge "adb: Use kernel aio for functionfs."
diff --git a/adb/Android.mk b/adb/Android.mk
index e195bec..d033d5a 100644
--- a/adb/Android.mk
+++ b/adb/Android.mk
@@ -116,7 +116,7 @@
# Even though we're building a static library (and thus there's no link step for
# this to take effect), this adds the includes to our path.
-LOCAL_STATIC_LIBRARIES := libcrypto_utils libcrypto libbase
+LOCAL_STATIC_LIBRARIES := libcrypto_utils libcrypto libbase libasyncio
include $(BUILD_STATIC_LIBRARY)
@@ -362,6 +362,7 @@
LOCAL_STRIP_MODULE := keep_symbols
LOCAL_STATIC_LIBRARIES := \
libadbd \
+ libasyncio \
libavb_user \
libbase \
libbootloader_message \
diff --git a/adb/adb.h b/adb/adb.h
index 88e13b6..6a9897f 100644
--- a/adb/adb.h
+++ b/adb/adb.h
@@ -31,8 +31,7 @@
#include "usb.h"
constexpr size_t MAX_PAYLOAD_V1 = 4 * 1024;
-constexpr size_t MAX_PAYLOAD_V2 = 256 * 1024;
-constexpr size_t MAX_PAYLOAD = MAX_PAYLOAD_V2;
+constexpr size_t MAX_PAYLOAD = 1024 * 1024;
constexpr size_t LINUX_MAX_SOCKET_SIZE = 4194304;
diff --git a/adb/daemon/usb.cpp b/adb/daemon/usb.cpp
index 0f92282..87ed3db 100644
--- a/adb/daemon/usb.cpp
+++ b/adb/daemon/usb.cpp
@@ -26,6 +26,7 @@
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
+#include <sys/mman.h>
#include <sys/types.h>
#include <unistd.h>
@@ -49,16 +50,11 @@
#define MAX_PACKET_SIZE_HS 512
#define MAX_PACKET_SIZE_SS 1024
-// Kernels before 3.3 have a 16KiB transfer limit That limit was replaced
-// with a 16MiB global limit in 3.3, but each URB submitted required a
-// contiguous kernel allocation, so you would get ENOMEM if you tried to
-// send something larger than the biggest available contiguous kernel
-// memory region. Large contiguous allocations could be unreliable
-// on a device kernel that has been running for a while fragmenting its
-// memory so we start with a larger allocation, and shrink the amount if
-// necessary.
#define USB_FFS_BULK_SIZE 16384
+// Number of buffers needed to fit MAX_PAYLOAD, with an extra for ZLPs.
+#define USB_FFS_NUM_BUFS ((MAX_PAYLOAD / USB_FFS_BULK_SIZE) + 1)
+
#define cpu_to_le16(x) htole16(x)
#define cpu_to_le32(x) htole32(x)
@@ -234,6 +230,26 @@
},
};
+static void aio_block_init(aio_block* aiob) {
+ aiob->iocb.resize(USB_FFS_NUM_BUFS);
+ aiob->iocbs.resize(USB_FFS_NUM_BUFS);
+ aiob->events.resize(USB_FFS_NUM_BUFS);
+ aiob->num_submitted = 0;
+ for (unsigned i = 0; i < USB_FFS_NUM_BUFS; i++) {
+ aiob->iocbs[i] = &aiob->iocb[i];
+ }
+}
+
+static int getMaxPacketSize(int ffs_fd) {
+ usb_endpoint_descriptor desc;
+ if (ioctl(ffs_fd, FUNCTIONFS_ENDPOINT_DESC, reinterpret_cast<unsigned long>(&desc))) {
+ D("[ could not get endpoint descriptor! (%d) ]", errno);
+ return MAX_PACKET_SIZE_HS;
+ } else {
+ return desc.wMaxPacketSize;
+ }
+}
+
bool init_functionfs(struct usb_handle* h) {
LOG(INFO) << "initializing functionfs";
@@ -301,6 +317,14 @@
goto err;
}
+ if (io_setup(USB_FFS_NUM_BUFS, &h->read_aiob.ctx) ||
+ io_setup(USB_FFS_NUM_BUFS, &h->write_aiob.ctx)) {
+ D("[ aio: got error on io_setup (%d) ]", errno);
+ }
+
+ h->read_aiob.fd = h->bulk_out;
+ h->write_aiob.fd = h->bulk_in;
+
h->max_rw = MAX_PAYLOAD;
while (h->max_rw >= USB_FFS_BULK_SIZE && retries < ENDPOINT_ALLOC_RETRIES) {
int ret_in = ioctl(h->bulk_in, FUNCTIONFS_ENDPOINT_ALLOC, static_cast<__u32>(h->max_rw));
@@ -410,6 +434,65 @@
return 0;
}
+static int usb_ffs_do_aio(usb_handle* h, const void* data, int len, bool read) {
+ aio_block* aiob = read ? &h->read_aiob : &h->write_aiob;
+ bool zero_packet = false;
+
+ int num_bufs = len / USB_FFS_BULK_SIZE + (len % USB_FFS_BULK_SIZE == 0 ? 0 : 1);
+ const char* cur_data = reinterpret_cast<const char*>(data);
+ int packet_size = getMaxPacketSize(aiob->fd);
+
+ if (posix_madvise(const_cast<void*>(data), len, POSIX_MADV_SEQUENTIAL | POSIX_MADV_WILLNEED) <
+ 0) {
+ D("[ Failed to madvise: %d ]", errno);
+ }
+
+ for (int i = 0; i < num_bufs; i++) {
+ int buf_len = std::min(len, USB_FFS_BULK_SIZE);
+ io_prep(&aiob->iocb[i], aiob->fd, cur_data, buf_len, 0, read);
+
+ len -= buf_len;
+ cur_data += buf_len;
+
+ if (len == 0 && buf_len % packet_size == 0 && read) {
+ // adb does not expect the device to send a zero packet after data transfer,
+ // but the host *does* send a zero packet for the device to read.
+ zero_packet = true;
+ }
+ }
+ if (zero_packet) {
+ io_prep(&aiob->iocb[num_bufs], aiob->fd, reinterpret_cast<const void*>(cur_data),
+ packet_size, 0, read);
+ num_bufs += 1;
+ }
+
+ if (io_submit(aiob->ctx, num_bufs, aiob->iocbs.data()) < num_bufs) {
+ D("[ aio: got error submitting %s (%d) ]", read ? "read" : "write", errno);
+ return -1;
+ }
+ if (TEMP_FAILURE_RETRY(
+ io_getevents(aiob->ctx, num_bufs, num_bufs, aiob->events.data(), nullptr)) < num_bufs) {
+ D("[ aio: got error waiting %s (%d) ]", read ? "read" : "write", errno);
+ return -1;
+ }
+ for (int i = 0; i < num_bufs; i++) {
+ if (aiob->events[i].res < 0) {
+ errno = aiob->events[i].res;
+ D("[ aio: got error event on %s (%d) ]", read ? "read" : "write", errno);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int usb_ffs_aio_read(usb_handle* h, void* data, int len) {
+ return usb_ffs_do_aio(h, data, len, true);
+}
+
+static int usb_ffs_aio_write(usb_handle* h, const void* data, int len) {
+ return usb_ffs_do_aio(h, data, len, false);
+}
+
static void usb_ffs_kick(usb_handle* h) {
int err;
@@ -438,6 +521,9 @@
h->kicked = false;
adb_close(h->bulk_out);
adb_close(h->bulk_in);
+ io_destroy(h->read_aiob.ctx);
+ io_destroy(h->write_aiob.ctx);
+
// Notify usb_adb_open_thread to open a new connection.
h->lock.lock();
h->open_new_connection = true;
@@ -450,8 +536,17 @@
usb_handle* h = new usb_handle();
- h->write = usb_ffs_write;
- h->read = usb_ffs_read;
+ if (android::base::GetBoolProperty("sys.usb.ffs.aio_compat", false)) {
+ // Devices on older kernels (< 3.18) will not have aio support for ffs
+ // unless backported. Fall back on the non-aio functions instead.
+ h->write = usb_ffs_write;
+ h->read = usb_ffs_read;
+ } else {
+ h->write = usb_ffs_aio_write;
+ h->read = usb_ffs_aio_read;
+ aio_block_init(&h->read_aiob);
+ aio_block_init(&h->write_aiob);
+ }
h->kick = usb_ffs_kick;
h->close = usb_ffs_close;
diff --git a/adb/daemon/usb.h b/adb/daemon/usb.h
index 55b5995..db1a6d6 100644
--- a/adb/daemon/usb.h
+++ b/adb/daemon/usb.h
@@ -20,6 +20,17 @@
#include <condition_variable>
#include <mutex>
+#include <asyncio/AsyncIO.h>
+
+struct aio_block {
+ std::vector<struct iocb> iocb;
+ std::vector<struct iocb*> iocbs;
+ std::vector<struct io_event> events;
+ aio_context_t ctx;
+ int num_submitted;
+ int fd;
+};
+
struct usb_handle {
usb_handle() : kicked(false) {
}
@@ -39,7 +50,11 @@
int bulk_out = -1; /* "out" from the host's perspective => source for adbd */
int bulk_in = -1; /* "in" from the host's perspective => sink for adbd */
+ // Access to these blocks is very not thread safe. Have one block for both the
+ // read and write threads.
+ struct aio_block read_aiob;
+ struct aio_block write_aiob;
+
int max_rw;
};
-bool init_functionfs(struct usb_handle* h);
diff --git a/adb/file_sync_client.cpp b/adb/file_sync_client.cpp
index 2576fb1..26f8d83 100644
--- a/adb/file_sync_client.cpp
+++ b/adb/file_sync_client.cpp
@@ -441,7 +441,7 @@
syncsendbuf sbuf;
sbuf.id = ID_DATA;
while (true) {
- int bytes_read = adb_read(lfd, sbuf.data, max);
+ int bytes_read = adb_read(lfd, sbuf.data, max - sizeof(SyncRequest));
if (bytes_read == -1) {
Error("reading '%s' locally failed: %s", lpath, strerror(errno));
adb_close(lfd);
diff --git a/adb/file_sync_service.cpp b/adb/file_sync_service.cpp
index 3448ee0..c6f3e66 100644
--- a/adb/file_sync_service.cpp
+++ b/adb/file_sync_service.cpp
@@ -206,6 +206,12 @@
__android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path);
int fd = adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode);
+
+ if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED) <
+ 0) {
+ D("[ Failed to fadvise: %d ]", errno);
+ }
+
if (fd < 0 && errno == ENOENT) {
if (!secure_mkdirs(android::base::Dirname(path))) {
SendSyncFailErrno(s, "secure_mkdirs failed");
@@ -413,10 +419,14 @@
return false;
}
+ if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE) < 0) {
+ D("[ Failed to fadvise: %d ]", errno);
+ }
+
syncmsg msg;
msg.data.id = ID_DATA;
while (true) {
- int r = adb_read(fd, &buffer[0], buffer.size());
+ int r = adb_read(fd, &buffer[0], buffer.size() - sizeof(msg.data));
if (r <= 0) {
if (r == 0) break;
SendSyncFailErrno(s, "read failed");
diff --git a/libasyncio/Android.bp b/libasyncio/Android.bp
new file mode 100644
index 0000000..9a637ac
--- /dev/null
+++ b/libasyncio/Android.bp
@@ -0,0 +1,44 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+libasyncio_cppflags = [
+ "-Wall",
+ "-Wextra",
+ "-Werror",
+]
+
+cc_library {
+ name: "libasyncio",
+ vendor_available: true,
+ host_supported: true,
+ srcs: [
+ "AsyncIO.cpp",
+ ],
+ cppflags: libasyncio_cppflags,
+
+ export_include_dirs: ["include"],
+ target: {
+ darwin: {
+ enabled: false,
+ },
+ linux_bionic: {
+ enabled: true,
+ },
+ windows: {
+ enabled: false,
+ },
+ },
+}
diff --git a/libasyncio/AsyncIO.cpp b/libasyncio/AsyncIO.cpp
new file mode 100644
index 0000000..7430bc8
--- /dev/null
+++ b/libasyncio/AsyncIO.cpp
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <asyncio/AsyncIO.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+int io_setup(unsigned nr, aio_context_t* ctxp) {
+ memset(ctxp, 0, sizeof(*ctxp));
+ return syscall(__NR_io_setup, nr, ctxp);
+}
+
+int io_destroy(aio_context_t ctx) {
+ return syscall(__NR_io_destroy, ctx);
+}
+
+int io_submit(aio_context_t ctx, long nr, iocb** iocbpp) {
+ return syscall(__NR_io_submit, ctx, nr, iocbpp);
+}
+
+int io_getevents(aio_context_t ctx, long min_nr, long max_nr, io_event* events, timespec* timeout) {
+ return syscall(__NR_io_getevents, ctx, min_nr, max_nr, events, timeout);
+}
+
+int io_cancel(aio_context_t ctx, iocb* iocbp, io_event* result) {
+ return syscall(__NR_io_cancel, ctx, iocbp, result);
+}
+
+void io_prep(iocb* iocb, int fd, const void* buf, uint64_t count, int64_t offset, bool read) {
+ memset(iocb, 0, sizeof(*iocb));
+ iocb->aio_fildes = fd;
+ iocb->aio_lio_opcode = read ? IOCB_CMD_PREAD : IOCB_CMD_PWRITE;
+ iocb->aio_reqprio = 0;
+ iocb->aio_buf = reinterpret_cast<uint64_t>(buf);
+ iocb->aio_nbytes = count;
+ iocb->aio_offset = offset;
+}
diff --git a/libasyncio/include/asyncio/AsyncIO.h b/libasyncio/include/asyncio/AsyncIO.h
new file mode 100644
index 0000000..e3fb93a
--- /dev/null
+++ b/libasyncio/include/asyncio/AsyncIO.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _ASYNCIO_H
+#define _ASYNCIO_H
+
+#include <cstring>
+#include <cstdint>
+#include <linux/aio_abi.h>
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Provides kernel aio operations.
+ */
+
+int io_setup(unsigned nr, aio_context_t* ctxp);
+int io_destroy(aio_context_t ctx);
+int io_submit(aio_context_t ctx, long nr, iocb** iocbpp);
+int io_getevents(aio_context_t ctx, long min_nr, long max_nr, io_event* events, timespec* timeout);
+int io_cancel(aio_context_t ctx, iocb*, io_event* result);
+void io_prep(iocb* iocb, int fd, const void* buf, uint64_t count, int64_t offset, bool read);
+
+#ifdef __cplusplus
+};
+#endif
+
+#endif // ASYNCIO_H