adb: create unix_isatty() function.
Our Windows code has several different ways it checks whether an FD is
a console or not, some of which aren't exactly correct as they may
treat character devices (e.g. NUL) as consoles.
This CL disallows using the builtin isatty() function and provides
unix_isatty() instead which correctly checks these corner cases.
Change-Id: I6d551c745dae691c7eb3446b585265d62c1e62fa
diff --git a/adb/adb_trace.cpp b/adb/adb_trace.cpp
index 04b82f6..9586f7c 100644
--- a/adb/adb_trace.cpp
+++ b/adb/adb_trace.cpp
@@ -157,7 +157,7 @@
// Don't open log file if no tracing, since this will block
// the crypto unmount of /data
if (!get_trace_setting().empty()) {
- if (isatty(STDOUT_FILENO) == 0) {
+ if (unix_isatty(STDOUT_FILENO) == 0) {
start_device_log();
}
}
diff --git a/adb/commandline.cpp b/adb/commandline.cpp
index 37bd777..c912936 100644
--- a/adb/commandline.cpp
+++ b/adb/commandline.cpp
@@ -1388,12 +1388,12 @@
// things like `adb shell < my_script.sh` work as expected.
// Otherwise leave |shell_type_arg| blank which uses PTY for
// interactive shells and raw for non-interactive.
- if (!isatty(STDIN_FILENO)) {
+ if (!unix_isatty(STDIN_FILENO)) {
shell_type_arg = kShellServiceArgRaw;
}
} else if (t_arg_count == 1) {
// A single -t arg isn't enough to override implicit -T.
- if (!isatty(STDIN_FILENO)) {
+ if (!unix_isatty(STDIN_FILENO)) {
fprintf(stderr,
"Remote PTY will not be allocated because stdin is not a terminal.\n"
"Use multiple -t options to force remote PTY allocation.\n");
diff --git a/adb/line_printer.cpp b/adb/line_printer.cpp
index 81b3f0a..aa332f7 100644
--- a/adb/line_printer.cpp
+++ b/adb/line_printer.cpp
@@ -46,7 +46,7 @@
LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) {
#ifndef _WIN32
const char* term = getenv("TERM");
- smart_terminal_ = isatty(1) && term && string(term) != "dumb";
+ smart_terminal_ = unix_isatty(1) && term && string(term) != "dumb";
#else
// Disable output buffer. It'd be nice to use line buffering but
// MSDN says: "For some systems, [_IOLBF] provides line
diff --git a/adb/sysdeps.h b/adb/sysdeps.h
index 51d09a6..1735627 100644
--- a/adb/sysdeps.h
+++ b/adb/sysdeps.h
@@ -181,6 +181,17 @@
extern int unix_open(const char* path, int options, ...);
#define open ___xxx_unix_open
+// Checks if |fd| corresponds to a console.
+// Standard Windows isatty() returns 1 for both console FDs and character
+// devices like NUL. unix_isatty() performs some extra checking to only match
+// console FDs.
+// |fd| must be a real file descriptor, meaning STDxx_FILENO or unix_open() FDs
+// will work but adb_open() FDs will not. Additionally the OS handle associated
+// with |fd| must have GENERIC_READ access (which console FDs have by default).
+// Returns 1 if |fd| is a console FD, 0 otherwise. The value of errno after
+// calling this function is unreliable and should not be used.
+int unix_isatty(int fd);
+#define isatty ___xxx_isatty
/* normally provided by <cutils/misc.h> */
extern void* load_file(const char* pathname, unsigned* psize);
@@ -551,6 +562,11 @@
#undef creat
#define creat ___xxx_creat
+static __inline__ int unix_isatty(int fd) {
+ return isatty(fd);
+}
+#define isatty ___xxx_isatty
+
// Helper for network_* functions.
inline int _fd_set_error_str(int fd, std::string* error) {
if (fd == -1) {
diff --git a/adb/sysdeps_win32.cpp b/adb/sysdeps_win32.cpp
index 14d1375..d2e6cdb 100644
--- a/adb/sysdeps_win32.cpp
+++ b/adb/sysdeps_win32.cpp
@@ -2499,10 +2499,52 @@
//
// Code organization:
//
+// * _get_console_handle() and unix_isatty() provide console information.
// * stdin_raw_init() and stdin_raw_restore() reconfigure the console.
// * unix_read() detects console windows (as opposed to pipes, files, etc.).
// * _console_read() is the main code of the emulation.
+// Returns a console HANDLE if |fd| is a console, otherwise returns nullptr.
+// If a valid HANDLE is returned and |mode| is not null, |mode| is also filled
+// with the console mode. Requires GENERIC_READ access to the underlying HANDLE.
+static HANDLE _get_console_handle(int fd, DWORD* mode=nullptr) {
+ // First check isatty(); this is very fast and eliminates most non-console
+ // FDs, but returns 1 for both consoles and character devices like NUL.
+#pragma push_macro("isatty")
+#undef isatty
+ if (!isatty(fd)) {
+ return nullptr;
+ }
+#pragma pop_macro("isatty")
+
+ // To differentiate between character devices and consoles we need to get
+ // the underlying HANDLE and use GetConsoleMode(), which is what requires
+ // GENERIC_READ permissions.
+ const intptr_t intptr_handle = _get_osfhandle(fd);
+ if (intptr_handle == -1) {
+ return nullptr;
+ }
+ const HANDLE handle = reinterpret_cast<const HANDLE>(intptr_handle);
+ DWORD temp_mode = 0;
+ if (!GetConsoleMode(handle, mode ? mode : &temp_mode)) {
+ return nullptr;
+ }
+
+ return handle;
+}
+
+// Returns a console handle if |stream| is a console, otherwise returns nullptr.
+static HANDLE _get_console_handle(FILE* const stream) {
+ const int fd = fileno(stream);
+ if (fd < 0) {
+ return nullptr;
+ }
+ return _get_console_handle(fd);
+}
+
+int unix_isatty(int fd) {
+ return _get_console_handle(fd) ? 1 : 0;
+}
// Read an input record from the console; one that should be processed.
static bool _get_interesting_input_record_uncached(const HANDLE console,
@@ -3302,20 +3344,7 @@
void stdin_raw_init(const int fd) {
if (STDIN_FILENO == fd) {
- const HANDLE in = GetStdHandle(STD_INPUT_HANDLE);
- if ((in == INVALID_HANDLE_VALUE) || (in == NULL)) {
- return;
- }
-
- if (GetFileType(in) != FILE_TYPE_CHAR) {
- // stdin might be a file or pipe.
- return;
- }
-
- if (!GetConsoleMode(in, &_old_console_mode)) {
- // If GetConsoleMode() fails, stdin is probably is not a console.
- return;
- }
+ const HANDLE in = _get_console_handle(fd, &_old_console_mode);
// Disable ENABLE_PROCESSED_INPUT so that Ctrl-C is read instead of
// calling the process Ctrl-C routine (configured by
@@ -3366,11 +3395,8 @@
} else {
// On older versions of Windows (definitely 7, definitely not 10),
// ReadConsole() with a size >= 31367 fails, so if |fd| is a console
- // we need to limit the read size. This may also catch devices like NUL,
- // but that is OK as we just want to avoid capping pipes and files which
- // don't need size limiting. This isatty() test is very simple and quick
- // and doesn't call the OS.
- if (isatty(fd) && len > 4096) {
+ // we need to limit the read size.
+ if (len > 4096 && unix_isatty(fd)) {
len = 4096;
}
// Just call into C Runtime which can read from pipes/files and which
@@ -3725,40 +3751,6 @@
return _wchmod(widen(path).c_str(), mode);
}
-// Internal function to get a Win32 console HANDLE from a C Runtime FILE*.
-static HANDLE _get_console_handle(FILE* const stream) {
- // Get a C Runtime file descriptor number from the FILE* structure.
- const int fd = fileno(stream);
- if (fd < 0) {
- return NULL;
- }
-
- // If it is not a "character device", it is probably a file and not a
- // console. Do this check early because it is probably cheap. Still do more
- // checks after this since there are devices that pass this test, but are
- // not a console, such as NUL, the Windows /dev/null equivalent (I think).
- if (!isatty(fd)) {
- return NULL;
- }
-
- // Given a C Runtime file descriptor number, get the underlying OS
- // file handle.
- const intptr_t osfh = _get_osfhandle(fd);
- if (osfh == -1) {
- return NULL;
- }
-
- const HANDLE h = reinterpret_cast<const HANDLE>(osfh);
-
- DWORD old_mode = 0;
- if (!GetConsoleMode(h, &old_mode)) {
- return NULL;
- }
-
- // If GetConsoleMode() was successful, assume this is a console.
- return h;
-}
-
// Internal helper function to write UTF-8 bytes to a console. Returns -1
// on error.
static int _console_write_utf8(const char* buf, size_t size, FILE* stream,
diff --git a/adb/sysdeps_win32_test.cpp b/adb/sysdeps_win32_test.cpp
index 66d1ba8..55b5eb4 100755
--- a/adb/sysdeps_win32_test.cpp
+++ b/adb/sysdeps_win32_test.cpp
@@ -18,6 +18,8 @@
#include "sysdeps.h"
+#include "base/test_utils.h"
+
TEST(sysdeps_win32, adb_getenv) {
// Insert all test env vars before first call to adb_getenv() which will
// read the env var block only once.
@@ -93,3 +95,45 @@
// adb_strerror() returns.
TestAdbStrError(ECONNRESET, "Connection reset by peer");
}
+
+TEST(sysdeps_win32, unix_isatty) {
+ // stdin and stdout should be consoles. Use CONIN$ and CONOUT$ special files
+ // so that we can test this even if stdin/stdout have been redirected. Read
+ // permissions are required for unix_isatty().
+ int conin_fd = unix_open("CONIN$", O_RDONLY);
+ int conout_fd = unix_open("CONOUT$", O_RDWR);
+ for (const int fd : {conin_fd, conout_fd}) {
+ EXPECT_TRUE(fd >= 0);
+ EXPECT_EQ(1, unix_isatty(fd));
+ EXPECT_EQ(0, unix_close(fd));
+ }
+
+ // nul returns 1 from isatty(), make sure unix_isatty() corrects that.
+ for (auto flags : {O_RDONLY, O_RDWR}) {
+ int nul_fd = unix_open("nul", flags);
+ EXPECT_TRUE(nul_fd >= 0);
+ EXPECT_EQ(0, unix_isatty(nul_fd));
+ EXPECT_EQ(0, unix_close(nul_fd));
+ }
+
+ // Check a real file, both read-write and read-only.
+ TemporaryFile temp_file;
+ EXPECT_TRUE(temp_file.fd >= 0);
+ EXPECT_EQ(0, unix_isatty(temp_file.fd));
+
+ int temp_file_ro_fd = unix_open(temp_file.path, O_RDONLY);
+ EXPECT_TRUE(temp_file_ro_fd >= 0);
+ EXPECT_EQ(0, unix_isatty(temp_file_ro_fd));
+ EXPECT_EQ(0, unix_close(temp_file_ro_fd));
+
+ // Check a real OS pipe.
+ int pipe_fds[2];
+ EXPECT_EQ(0, _pipe(pipe_fds, 64, _O_BINARY));
+ EXPECT_EQ(0, unix_isatty(pipe_fds[0]));
+ EXPECT_EQ(0, unix_isatty(pipe_fds[1]));
+ EXPECT_EQ(0, _close(pipe_fds[0]));
+ EXPECT_EQ(0, _close(pipe_fds[1]));
+
+ // Make sure an invalid FD is handled correctly.
+ EXPECT_EQ(0, unix_isatty(-1));
+}