blob: a222139982305cdea72ff04e2c0edc547862d9a0 [file] [log] [blame]
Alex Deymoaea4c1c2015-08-19 20:24:43 -07001//
2// Copyright (C) 2009 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
rspangler@google.com49fdf182009-10-10 00:57:34 +000016
Alex Deymo39910dc2015-11-09 17:04:30 -080017#include "update_engine/common/libcurl_http_fetcher.h"
Andrew de los Reyesd57d1472010-10-21 13:34:08 -070018
adlr@google.comc98a7ed2009-12-04 18:54:03 +000019#include <algorithm>
Andrew de los Reyesd57d1472010-10-21 13:34:08 -070020#include <string>
21
Alex Vakulenko4906c1c2014-08-21 13:17:44 -070022#include <base/bind.h>
Alex Deymoc00c98a2015-03-17 17:38:00 -070023#include <base/format_macros.h>
Alex Deymo60ca1a72015-06-18 18:19:15 -070024#include <base/location.h>
Andrew de los Reyesd57d1472010-10-21 13:34:08 -070025#include <base/logging.h>
Alex Vakulenko75039d72014-03-25 12:36:28 -070026#include <base/strings/string_util.h>
27#include <base/strings/stringprintf.h>
Andrew de los Reyesd57d1472010-10-21 13:34:08 -070028
Alex Deymo39910dc2015-11-09 17:04:30 -080029#include "update_engine/common/certificate_checker.h"
30#include "update_engine/common/hardware_interface.h"
31#include "update_engine/common/platform_constants.h"
adlr@google.comc98a7ed2009-12-04 18:54:03 +000032
Alex Deymo60ca1a72015-06-18 18:19:15 -070033using base::TimeDelta;
Alex Vakulenko3f39d5c2015-10-13 09:27:13 -070034using brillo::MessageLoop;
Alex Deymoc4acdf42014-05-28 21:07:10 -070035using std::max;
Andrew de los Reyesd57d1472010-10-21 13:34:08 -070036using std::string;
rspangler@google.com49fdf182009-10-10 00:57:34 +000037
38// This is a concrete implementation of HttpFetcher that uses libcurl to do the
39// http work.
40
41namespace chromeos_update_engine {
42
Andrew de los Reyes9bbd1872010-07-16 14:52:29 -070043namespace {
Andrew de los Reyes5d0783d2010-11-29 18:14:16 -080044const int kNoNetworkRetrySeconds = 10;
Alex Vakulenkod2779df2014-06-16 13:19:00 -070045} // namespace
Andrew de los Reyes9bbd1872010-07-16 14:52:29 -070046
Alex Deymo33e91e72015-12-01 18:26:08 -030047LibcurlHttpFetcher::LibcurlHttpFetcher(ProxyResolver* proxy_resolver,
48 HardwareInterface* hardware)
49 : HttpFetcher(proxy_resolver), hardware_(hardware) {
Alex Deymoc1c17b42015-11-23 03:53:15 -030050 // Dev users want a longer timeout (180 seconds) because they may
51 // be waiting on the dev server to build an image.
52 if (!hardware_->IsOfficialBuild())
53 low_speed_time_seconds_ = kDownloadDevModeLowSpeedTimeSeconds;
Alex Deymo46a9aae2016-05-04 20:20:11 -070054 if (hardware_->IsOOBEEnabled() && !hardware_->IsOOBEComplete(nullptr))
Alex Deymoc1c17b42015-11-23 03:53:15 -030055 max_retry_count_ = kDownloadMaxRetryCountOobeNotComplete;
56}
57
rspangler@google.com49fdf182009-10-10 00:57:34 +000058LibcurlHttpFetcher::~LibcurlHttpFetcher() {
Darin Petkov9ce452b2010-11-17 14:33:28 -080059 LOG_IF(ERROR, transfer_in_progress_)
60 << "Destroying the fetcher while a transfer is in progress.";
rspangler@google.com49fdf182009-10-10 00:57:34 +000061 CleanUp();
62}
63
Alex Deymof329b932014-10-30 01:37:48 -070064bool LibcurlHttpFetcher::GetProxyType(const string& proxy,
Gilad Arnold59d9e012013-07-23 16:41:43 -070065 curl_proxytype* out_type) {
Alex Deymo56ccb072016-02-05 00:50:48 -080066 if (base::StartsWith(
67 proxy, "socks5://", base::CompareCase::INSENSITIVE_ASCII) ||
68 base::StartsWith(
69 proxy, "socks://", base::CompareCase::INSENSITIVE_ASCII)) {
Gilad Arnold59d9e012013-07-23 16:41:43 -070070 *out_type = CURLPROXY_SOCKS5_HOSTNAME;
71 return true;
72 }
Alex Deymo56ccb072016-02-05 00:50:48 -080073 if (base::StartsWith(
74 proxy, "socks4://", base::CompareCase::INSENSITIVE_ASCII)) {
Gilad Arnold59d9e012013-07-23 16:41:43 -070075 *out_type = CURLPROXY_SOCKS4A;
76 return true;
77 }
Alex Deymo56ccb072016-02-05 00:50:48 -080078 if (base::StartsWith(
79 proxy, "http://", base::CompareCase::INSENSITIVE_ASCII) ||
80 base::StartsWith(
81 proxy, "https://", base::CompareCase::INSENSITIVE_ASCII)) {
Gilad Arnold59d9e012013-07-23 16:41:43 -070082 *out_type = CURLPROXY_HTTP;
83 return true;
84 }
Alex Deymo56ccb072016-02-05 00:50:48 -080085 if (base::StartsWith(proxy, kNoProxy, base::CompareCase::INSENSITIVE_ASCII)) {
Gilad Arnold59d9e012013-07-23 16:41:43 -070086 // known failure case. don't log.
87 return false;
88 }
89 LOG(INFO) << "Unknown proxy type: " << proxy;
90 return false;
91}
92
Alex Deymof329b932014-10-30 01:37:48 -070093void LibcurlHttpFetcher::ResumeTransfer(const string& url) {
Andrew de los Reyes3270f742010-07-15 22:28:14 -070094 LOG(INFO) << "Starting/Resuming transfer";
rspangler@google.com49fdf182009-10-10 00:57:34 +000095 CHECK(!transfer_in_progress_);
96 url_ = url;
97 curl_multi_handle_ = curl_multi_init();
98 CHECK(curl_multi_handle_);
99
100 curl_handle_ = curl_easy_init();
101 CHECK(curl_handle_);
Alex Deymof2858572016-02-25 11:20:13 -0800102 ignore_failure_ = false;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000103
Andrew de los Reyes45168102010-11-22 11:13:50 -0800104 CHECK(HasProxy());
Gilad Arnoldfbaee242012-04-04 15:59:43 -0700105 bool is_direct = (GetCurrentProxy() == kNoProxy);
106 LOG(INFO) << "Using proxy: " << (is_direct ? "no" : "yes");
107 if (is_direct) {
Andrew de los Reyes45168102010-11-22 11:13:50 -0800108 CHECK_EQ(curl_easy_setopt(curl_handle_,
109 CURLOPT_PROXY,
110 ""), CURLE_OK);
111 } else {
112 CHECK_EQ(curl_easy_setopt(curl_handle_,
113 CURLOPT_PROXY,
114 GetCurrentProxy().c_str()), CURLE_OK);
115 // Curl seems to require us to set the protocol
116 curl_proxytype type;
Gilad Arnold59d9e012013-07-23 16:41:43 -0700117 if (GetProxyType(GetCurrentProxy(), &type)) {
Andrew de los Reyes45168102010-11-22 11:13:50 -0800118 CHECK_EQ(curl_easy_setopt(curl_handle_,
119 CURLOPT_PROXYTYPE,
120 type), CURLE_OK);
121 }
122 }
123
rspangler@google.com49fdf182009-10-10 00:57:34 +0000124 if (post_data_set_) {
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000125 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POST, 1), CURLE_OK);
126 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDS,
Alex Vakulenkof68bbbc2015-02-09 12:53:18 -0800127 post_data_.data()),
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000128 CURLE_OK);
129 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDSIZE,
130 post_data_.size()),
131 CURLE_OK);
Alex Deymofdd6dec2016-03-03 22:35:43 -0800132 }
Gilad Arnold9dd1e7c2012-02-16 12:13:36 -0800133
Alex Deymofdd6dec2016-03-03 22:35:43 -0800134 // Setup extra HTTP headers.
135 if (curl_http_headers_) {
136 curl_slist_free_all(curl_http_headers_);
137 curl_http_headers_ = nullptr;
138 }
139 for (const auto& header : extra_headers_) {
140 // curl_slist_append() copies the string.
141 curl_http_headers_ =
142 curl_slist_append(curl_http_headers_, header.second.c_str());
143 }
144 if (post_data_set_) {
Gilad Arnold9dd1e7c2012-02-16 12:13:36 -0800145 // Set the Content-Type HTTP header, if one was specifically set.
Gilad Arnold9dd1e7c2012-02-16 12:13:36 -0800146 if (post_content_type_ != kHttpContentTypeUnspecified) {
Alex Deymofdd6dec2016-03-03 22:35:43 -0800147 const string content_type_attr = base::StringPrintf(
148 "Content-Type: %s", GetHttpContentTypeString(post_content_type_));
149 curl_http_headers_ =
150 curl_slist_append(curl_http_headers_, content_type_attr.c_str());
Gilad Arnold9dd1e7c2012-02-16 12:13:36 -0800151 } else {
152 LOG(WARNING) << "no content type set, using libcurl default";
153 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000154 }
Alex Deymofdd6dec2016-03-03 22:35:43 -0800155 CHECK_EQ(
156 curl_easy_setopt(curl_handle_, CURLOPT_HTTPHEADER, curl_http_headers_),
157 CURLE_OK);
rspangler@google.com49fdf182009-10-10 00:57:34 +0000158
Gilad Arnolde4ad2502011-12-29 17:08:54 -0800159 if (bytes_downloaded_ > 0 || download_length_) {
160 // Resume from where we left off.
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000161 resume_offset_ = bytes_downloaded_;
Gilad Arnolde4ad2502011-12-29 17:08:54 -0800162 CHECK_GE(resume_offset_, 0);
163
164 // Compute end offset, if one is specified. As per HTTP specification, this
165 // is an inclusive boundary. Make sure it doesn't overflow.
166 size_t end_offset = 0;
167 if (download_length_) {
168 end_offset = static_cast<size_t>(resume_offset_) + download_length_ - 1;
169 CHECK_LE((size_t) resume_offset_, end_offset);
170 }
171
172 // Create a string representation of the desired range.
Alex Deymoc00c98a2015-03-17 17:38:00 -0700173 string range_str = base::StringPrintf(
174 "%" PRIu64 "-", static_cast<uint64_t>(resume_offset_));
175 if (end_offset)
176 range_str += std::to_string(end_offset);
Gilad Arnolde4ad2502011-12-29 17:08:54 -0800177 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_RANGE, range_str.c_str()),
178 CURLE_OK);
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000179 }
180
181 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEDATA, this), CURLE_OK);
182 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEFUNCTION,
183 StaticLibcurlWrite), CURLE_OK);
Chris Sosa77f79e82014-06-02 18:16:24 -0700184 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_URL, url_.c_str()),
Andrew de los Reyesd57d1472010-10-21 13:34:08 -0700185 CURLE_OK);
Andrew de los Reyes3270f742010-07-15 22:28:14 -0700186
David Zeuthen34135a92013-08-06 11:16:16 -0700187 // If the connection drops under |low_speed_limit_bps_| (10
188 // bytes/sec by default) for |low_speed_time_seconds_| (90 seconds,
189 // 180 on non-official builds), reconnect.
190 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_LIMIT,
191 low_speed_limit_bps_),
Andrew de los Reyes3270f742010-07-15 22:28:14 -0700192 CURLE_OK);
David Zeuthen34135a92013-08-06 11:16:16 -0700193 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_TIME,
194 low_speed_time_seconds_),
Andrew de los Reyes3270f742010-07-15 22:28:14 -0700195 CURLE_OK);
David Zeuthen34135a92013-08-06 11:16:16 -0700196 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CONNECTTIMEOUT,
197 connect_timeout_seconds_),
Andrew de los Reyese72f9c02011-04-20 10:47:40 -0700198 CURLE_OK);
Andrew de los Reyes3270f742010-07-15 22:28:14 -0700199
Darin Petkov41c2fcf2010-08-25 13:14:48 -0700200 // By default, libcurl doesn't follow redirections. Allow up to
David Zeuthen34135a92013-08-06 11:16:16 -0700201 // |kDownloadMaxRedirects| redirections.
Darin Petkov3a4016a2010-09-28 13:54:17 -0700202 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_FOLLOWLOCATION, 1), CURLE_OK);
David Zeuthen34135a92013-08-06 11:16:16 -0700203 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_MAXREDIRS,
204 kDownloadMaxRedirects),
Darin Petkov41c2fcf2010-08-25 13:14:48 -0700205 CURLE_OK);
206
Nam T. Nguyen7d623eb2014-05-13 16:06:28 -0700207 // Lock down the appropriate curl options for HTTP or HTTPS depending on
208 // the url.
Alex Deymoc1c17b42015-11-23 03:53:15 -0300209 if (hardware_->IsOfficialBuild()) {
Alex Deymo56ccb072016-02-05 00:50:48 -0800210 if (base::StartsWith(
211 url_, "http://", base::CompareCase::INSENSITIVE_ASCII)) {
Jay Srinivasanb3f55402012-12-03 18:12:04 -0800212 SetCurlOptionsForHttp();
Alex Deymo56ccb072016-02-05 00:50:48 -0800213 } else if (base::StartsWith(
214 url_, "https://", base::CompareCase::INSENSITIVE_ASCII)) {
Jay Srinivasanb3f55402012-12-03 18:12:04 -0800215 SetCurlOptionsForHttps();
Alex Deymo56ccb072016-02-05 00:50:48 -0800216#if !defined(__CHROMEOS__) && !defined(__BRILLO__)
217 } else if (base::StartsWith(
218 url_, "file://", base::CompareCase::INSENSITIVE_ASCII)) {
219 SetCurlOptionsForFile();
220#endif
221 } else {
222 LOG(ERROR) << "Received invalid URI: " << url_;
223 // Lock down to no protocol supported for the transfer.
224 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, 0), CURLE_OK);
225 }
Jay Srinivasanb3f55402012-12-03 18:12:04 -0800226 } else {
Nam T. Nguyen7d623eb2014-05-13 16:06:28 -0700227 LOG(INFO) << "Not setting http(s) curl options because we are "
228 << "running a dev/test image";
Darin Petkovfc7a0ce2010-10-25 10:38:37 -0700229 }
230
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000231 CHECK_EQ(curl_multi_add_handle(curl_multi_handle_, curl_handle_), CURLM_OK);
rspangler@google.com49fdf182009-10-10 00:57:34 +0000232 transfer_in_progress_ = true;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000233}
234
Jay Srinivasanb3f55402012-12-03 18:12:04 -0800235// Lock down only the protocol in case of HTTP.
236void LibcurlHttpFetcher::SetCurlOptionsForHttp() {
237 LOG(INFO) << "Setting up curl options for HTTP";
238 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTP),
239 CURLE_OK);
240 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS,
241 CURLPROTO_HTTP),
242 CURLE_OK);
243}
244
245// Security lock-down in official builds: makes sure that peer certificate
246// verification is enabled, restricts the set of trusted certificates,
247// restricts protocols to HTTPS, restricts ciphers to HIGH.
248void LibcurlHttpFetcher::SetCurlOptionsForHttps() {
249 LOG(INFO) << "Setting up curl options for HTTPS";
250 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_VERIFYPEER, 1),
251 CURLE_OK);
Alex Deymo8fd98d82016-06-23 18:22:08 -0700252 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_VERIFYHOST, 2),
253 CURLE_OK);
Alex Deymo35b35842015-10-20 11:21:56 -0700254 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CAPATH,
255 constants::kCACertificatesPath),
Jay Srinivasanb3f55402012-12-03 18:12:04 -0800256 CURLE_OK);
257 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS),
258 CURLE_OK);
259 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS,
260 CURLPROTO_HTTPS),
261 CURLE_OK);
262 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CIPHER_LIST, "HIGH:!ADH"),
263 CURLE_OK);
Alex Deymo33e91e72015-12-01 18:26:08 -0300264 if (server_to_check_ != ServerToCheck::kNone) {
265 CHECK_EQ(
266 curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_DATA, &server_to_check_),
267 CURLE_OK);
Jay Srinivasanb3f55402012-12-03 18:12:04 -0800268 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_FUNCTION,
269 CertificateChecker::ProcessSSLContext),
270 CURLE_OK);
271 }
272}
273
Alex Deymo56ccb072016-02-05 00:50:48 -0800274// Lock down only the protocol in case of a local file.
275void LibcurlHttpFetcher::SetCurlOptionsForFile() {
276 LOG(INFO) << "Setting up curl options for FILE";
277 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_FILE),
278 CURLE_OK);
279 CHECK_EQ(
280 curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_FILE),
281 CURLE_OK);
282}
Jay Srinivasanb3f55402012-12-03 18:12:04 -0800283
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000284// Begins the transfer, which must not have already been started.
Alex Deymof329b932014-10-30 01:37:48 -0700285void LibcurlHttpFetcher::BeginTransfer(const string& url) {
Andrew de los Reyesf3ed8e72011-02-16 10:35:46 -0800286 CHECK(!transfer_in_progress_);
287 url_ = url;
Alex Vakulenko4906c1c2014-08-21 13:17:44 -0700288 auto closure = base::Bind(&LibcurlHttpFetcher::ProxiesResolved,
289 base::Unretained(this));
Alex Deymo60ca1a72015-06-18 18:19:15 -0700290 if (!ResolveProxiesForUrl(url_, closure)) {
Andrew de los Reyesf3ed8e72011-02-16 10:35:46 -0800291 LOG(ERROR) << "Couldn't resolve proxies";
292 if (delegate_)
293 delegate_->TransferComplete(this, false);
294 }
295}
296
297void LibcurlHttpFetcher::ProxiesResolved() {
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000298 transfer_size_ = -1;
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000299 resume_offset_ = 0;
Andrew de los Reyes9bbd1872010-07-16 14:52:29 -0700300 retry_count_ = 0;
Darin Petkova0929552010-11-29 14:19:06 -0800301 no_network_retry_count_ = 0;
Darin Petkovcb466212010-08-26 09:40:11 -0700302 http_response_code_ = 0;
Andrew de los Reyes819fef22010-12-17 11:33:58 -0800303 terminate_requested_ = false;
Gilad Arnolda2dee1d2012-04-12 11:50:37 -0700304 sent_byte_ = false;
Alex Deymof2858572016-02-25 11:20:13 -0800305
306 // If we are paused, we delay these two operations until Unpause is called.
307 if (transfer_paused_) {
308 restart_transfer_on_unpause_ = true;
309 return;
310 }
Andrew de los Reyesf3ed8e72011-02-16 10:35:46 -0800311 ResumeTransfer(url_);
Andrew de los Reyes9bbd1872010-07-16 14:52:29 -0700312 CurlPerformOnce();
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000313}
314
Darin Petkov9ce452b2010-11-17 14:33:28 -0800315void LibcurlHttpFetcher::ForceTransferTermination() {
316 CleanUp();
317 if (delegate_) {
318 // Note that after the callback returns this object may be destroyed.
319 delegate_->TransferTerminated(this);
320 }
321}
322
rspangler@google.com49fdf182009-10-10 00:57:34 +0000323void LibcurlHttpFetcher::TerminateTransfer() {
Darin Petkov9ce452b2010-11-17 14:33:28 -0800324 if (in_write_callback_) {
Andrew de los Reyes3fd5d302010-10-07 20:07:18 -0700325 terminate_requested_ = true;
Darin Petkov9ce452b2010-11-17 14:33:28 -0800326 } else {
327 ForceTransferTermination();
328 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000329}
330
Alex Deymofdd6dec2016-03-03 22:35:43 -0800331void LibcurlHttpFetcher::SetHeader(const string& header_name,
332 const string& header_value) {
333 string header_line = header_name + ": " + header_value;
334 // Avoid the space if no data on the right side of the semicolon.
335 if (header_value.empty())
336 header_line = header_name + ":";
337 TEST_AND_RETURN(header_line.find('\n') == string::npos);
338 TEST_AND_RETURN(header_name.find(':') == string::npos);
339 extra_headers_[base::ToLowerASCII(header_name)] = header_line;
340}
341
Andrew de los Reyescb319332010-07-19 10:55:01 -0700342void LibcurlHttpFetcher::CurlPerformOnce() {
rspangler@google.com49fdf182009-10-10 00:57:34 +0000343 CHECK(transfer_in_progress_);
344 int running_handles = 0;
345 CURLMcode retcode = CURLM_CALL_MULTI_PERFORM;
346
347 // libcurl may request that we immediately call curl_multi_perform after it
348 // returns, so we do. libcurl promises that curl_multi_perform will not block.
349 while (CURLM_CALL_MULTI_PERFORM == retcode) {
350 retcode = curl_multi_perform(curl_multi_handle_, &running_handles);
Andrew de los Reyes3fd5d302010-10-07 20:07:18 -0700351 if (terminate_requested_) {
Darin Petkov9ce452b2010-11-17 14:33:28 -0800352 ForceTransferTermination();
Andrew de los Reyes3fd5d302010-10-07 20:07:18 -0700353 return;
354 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000355 }
Alex Deymof2858572016-02-25 11:20:13 -0800356
357 // If the transfer completes while paused, we should ignore the failure once
358 // the fetcher is unpaused.
359 if (running_handles == 0 && transfer_paused_ && !ignore_failure_) {
360 LOG(INFO) << "Connection closed while paused, ignoring failure.";
361 ignore_failure_ = true;
362 }
363
364 if (running_handles != 0 || transfer_paused_) {
365 // There's either more work to do or we are paused, so we just keep the
366 // file descriptors to watch up to date and exit, until we are done with the
367 // work and we are not paused.
368 SetupMessageLoopSources();
369 return;
370 }
371
372 // At this point, the transfer was completed in some way (error, connection
373 // closed or download finished).
374
375 GetHttpResponseCode();
376 if (http_response_code_) {
377 LOG(INFO) << "HTTP response code: " << http_response_code_;
378 no_network_retry_count_ = 0;
379 } else {
380 LOG(ERROR) << "Unable to get http response code.";
381 }
382
383 // we're done!
384 CleanUp();
385
386 // TODO(petkov): This temporary code tries to deal with the case where the
387 // update engine performs an update check while the network is not ready
388 // (e.g., right after resume). Longer term, we should check if the network
389 // is online/offline and return an appropriate error code.
390 if (!sent_byte_ &&
391 http_response_code_ == 0 &&
392 no_network_retry_count_ < no_network_max_retries_) {
393 no_network_retry_count_++;
394 MessageLoop::current()->PostDelayedTask(
395 FROM_HERE,
396 base::Bind(&LibcurlHttpFetcher::RetryTimeoutCallback,
397 base::Unretained(this)),
398 TimeDelta::FromSeconds(kNoNetworkRetrySeconds));
399 LOG(INFO) << "No HTTP response, retry " << no_network_retry_count_;
400 } else if ((!sent_byte_ && !IsHttpResponseSuccess()) ||
401 IsHttpResponseError()) {
402 // The transfer completed w/ error and we didn't get any bytes.
403 // If we have another proxy to try, try that.
404 //
405 // TODO(garnold) in fact there are two separate cases here: one case is an
406 // other-than-success return code (including no return code) and no
407 // received bytes, which is necessary due to the way callbacks are
408 // currently processing error conditions; the second is an explicit HTTP
409 // error code, where some data may have been received (as in the case of a
410 // semi-successful multi-chunk fetch). This is a confusing behavior and
411 // should be unified into a complete, coherent interface.
412 LOG(INFO) << "Transfer resulted in an error (" << http_response_code_
413 << "), " << bytes_downloaded_ << " bytes downloaded";
414
415 PopProxy(); // Delete the proxy we just gave up on.
416
417 if (HasProxy()) {
418 // We have another proxy. Retry immediately.
419 LOG(INFO) << "Retrying with next proxy setting";
420 MessageLoop::current()->PostTask(
421 FROM_HERE,
422 base::Bind(&LibcurlHttpFetcher::RetryTimeoutCallback,
423 base::Unretained(this)));
Andrew de los Reyes9bbd1872010-07-16 14:52:29 -0700424 } else {
Alex Deymof2858572016-02-25 11:20:13 -0800425 // Out of proxies. Give up.
426 LOG(INFO) << "No further proxies, indicating transfer complete";
427 if (delegate_)
428 delegate_->TransferComplete(this, false); // signal fail
Alex Deymo021a45e2016-03-15 13:12:05 -0700429 return;
Andrew de los Reyes9bbd1872010-07-16 14:52:29 -0700430 }
Alex Deymof2858572016-02-25 11:20:13 -0800431 } else if ((transfer_size_ >= 0) && (bytes_downloaded_ < transfer_size_)) {
432 if (!ignore_failure_)
433 retry_count_++;
434 LOG(INFO) << "Transfer interrupted after downloading "
435 << bytes_downloaded_ << " of " << transfer_size_ << " bytes. "
436 << transfer_size_ - bytes_downloaded_ << " bytes remaining "
437 << "after " << retry_count_ << " attempt(s)";
Darin Petkov192ced42010-07-23 16:20:24 -0700438
Alex Deymof2858572016-02-25 11:20:13 -0800439 if (retry_count_ > max_retry_count_) {
440 LOG(INFO) << "Reached max attempts (" << retry_count_ << ")";
441 if (delegate_)
442 delegate_->TransferComplete(this, false); // signal fail
Alex Deymo021a45e2016-03-15 13:12:05 -0700443 return;
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000444 }
Alex Deymo021a45e2016-03-15 13:12:05 -0700445 // Need to restart transfer
446 LOG(INFO) << "Restarting transfer to download the remaining bytes";
447 MessageLoop::current()->PostDelayedTask(
448 FROM_HERE,
449 base::Bind(&LibcurlHttpFetcher::RetryTimeoutCallback,
450 base::Unretained(this)),
451 TimeDelta::FromSeconds(retry_seconds_));
rspangler@google.com49fdf182009-10-10 00:57:34 +0000452 } else {
Alex Deymof2858572016-02-25 11:20:13 -0800453 LOG(INFO) << "Transfer completed (" << http_response_code_
454 << "), " << bytes_downloaded_ << " bytes downloaded";
455 if (delegate_) {
456 bool success = IsHttpResponseSuccess();
457 delegate_->TransferComplete(this, success);
458 }
Alex Deymo021a45e2016-03-15 13:12:05 -0700459 return;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000460 }
Alex Deymo021a45e2016-03-15 13:12:05 -0700461 // If we reach this point is because TransferComplete() was not called in any
462 // of the previous branches. The delegate is allowed to destroy the object
463 // once TransferComplete is called so this would be illegal.
Alex Deymof2858572016-02-25 11:20:13 -0800464 ignore_failure_ = false;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000465}
466
467size_t LibcurlHttpFetcher::LibcurlWrite(void *ptr, size_t size, size_t nmemb) {
Gilad Arnold48085ba2011-11-16 09:36:08 -0800468 // Update HTTP response first.
Andrew de los Reyes3fd5d302010-10-07 20:07:18 -0700469 GetHttpResponseCode();
Gilad Arnold48085ba2011-11-16 09:36:08 -0800470 const size_t payload_size = size * nmemb;
471
472 // Do nothing if no payload or HTTP response is an error.
Gilad Arnold9bedeb52011-11-17 16:19:57 -0800473 if (payload_size == 0 || !IsHttpResponseSuccess()) {
Gilad Arnold48085ba2011-11-16 09:36:08 -0800474 LOG(INFO) << "HTTP response unsuccessful (" << http_response_code_
475 << ") or no payload (" << payload_size << "), nothing to do";
476 return 0;
477 }
478
479 sent_byte_ = true;
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000480 {
481 double transfer_size_double;
482 CHECK_EQ(curl_easy_getinfo(curl_handle_,
483 CURLINFO_CONTENT_LENGTH_DOWNLOAD,
484 &transfer_size_double), CURLE_OK);
485 off_t new_transfer_size = static_cast<off_t>(transfer_size_double);
486 if (new_transfer_size > 0) {
487 transfer_size_ = resume_offset_ + new_transfer_size;
488 }
489 }
Gilad Arnold48085ba2011-11-16 09:36:08 -0800490 bytes_downloaded_ += payload_size;
Andrew de los Reyes3fd5d302010-10-07 20:07:18 -0700491 in_write_callback_ = true;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000492 if (delegate_)
Alex Vakulenkof68bbbc2015-02-09 12:53:18 -0800493 delegate_->ReceivedBytes(this, ptr, payload_size);
Andrew de los Reyes3fd5d302010-10-07 20:07:18 -0700494 in_write_callback_ = false;
Gilad Arnold48085ba2011-11-16 09:36:08 -0800495 return payload_size;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000496}
497
498void LibcurlHttpFetcher::Pause() {
Alex Deymof2858572016-02-25 11:20:13 -0800499 if (transfer_paused_) {
500 LOG(ERROR) << "Fetcher already paused.";
501 return;
502 }
503 transfer_paused_ = true;
504 if (!transfer_in_progress_) {
505 // If pause before we started a connection, we don't need to notify curl
506 // about that, we will simply not start the connection later.
507 return;
508 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000509 CHECK(curl_handle_);
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000510 CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_ALL), CURLE_OK);
rspangler@google.com49fdf182009-10-10 00:57:34 +0000511}
512
513void LibcurlHttpFetcher::Unpause() {
Alex Deymof2858572016-02-25 11:20:13 -0800514 if (!transfer_paused_) {
515 LOG(ERROR) << "Resume attempted when fetcher not paused.";
516 return;
517 }
518 transfer_paused_ = false;
519 if (restart_transfer_on_unpause_) {
520 restart_transfer_on_unpause_ = false;
521 ResumeTransfer(url_);
522 CurlPerformOnce();
523 return;
524 }
525 if (!transfer_in_progress_) {
526 // If resumed before starting the connection, there's no need to notify
527 // anybody. We will simply start the connection once it is time.
528 return;
529 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000530 CHECK(curl_handle_);
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000531 CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_CONT), CURLE_OK);
Alex Deymof2858572016-02-25 11:20:13 -0800532 // Since the transfer is in progress, we need to dispatch a CurlPerformOnce()
533 // now to let the connection continue, otherwise it would be called by the
534 // TimeoutCallback but with a delay.
535 CurlPerformOnce();
rspangler@google.com49fdf182009-10-10 00:57:34 +0000536}
537
Alex Deymo29b81532015-07-09 11:51:49 -0700538// This method sets up callbacks with the MessageLoop.
539void LibcurlHttpFetcher::SetupMessageLoopSources() {
rspangler@google.com49fdf182009-10-10 00:57:34 +0000540 fd_set fd_read;
541 fd_set fd_write;
Darin Petkov60e14152010-10-27 16:57:04 -0700542 fd_set fd_exc;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000543
544 FD_ZERO(&fd_read);
545 FD_ZERO(&fd_write);
Darin Petkov60e14152010-10-27 16:57:04 -0700546 FD_ZERO(&fd_exc);
rspangler@google.com49fdf182009-10-10 00:57:34 +0000547
548 int fd_max = 0;
549
550 // Ask libcurl for the set of file descriptors we should track on its
551 // behalf.
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000552 CHECK_EQ(curl_multi_fdset(curl_multi_handle_, &fd_read, &fd_write,
Darin Petkov60e14152010-10-27 16:57:04 -0700553 &fd_exc, &fd_max), CURLM_OK);
rspangler@google.com49fdf182009-10-10 00:57:34 +0000554
555 // We should iterate through all file descriptors up to libcurl's fd_max or
Darin Petkov60e14152010-10-27 16:57:04 -0700556 // the highest one we're tracking, whichever is larger.
Alex Deymo29b81532015-07-09 11:51:49 -0700557 for (size_t t = 0; t < arraysize(fd_task_maps_); ++t) {
558 if (!fd_task_maps_[t].empty())
559 fd_max = max(fd_max, fd_task_maps_[t].rbegin()->first);
Darin Petkov60e14152010-10-27 16:57:04 -0700560 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000561
Darin Petkov60e14152010-10-27 16:57:04 -0700562 // For each fd, if we're not tracking it, track it. If we are tracking it, but
563 // libcurl doesn't care about it anymore, stop tracking it. After this loop,
Alex Deymo29b81532015-07-09 11:51:49 -0700564 // there should be exactly as many tasks scheduled in fd_task_maps_[0|1] as
Darin Petkov60e14152010-10-27 16:57:04 -0700565 // there are read/write fds that we're tracking.
566 for (int fd = 0; fd <= fd_max; ++fd) {
567 // Note that fd_exc is unused in the current version of libcurl so is_exc
568 // should always be false.
569 bool is_exc = FD_ISSET(fd, &fd_exc) != 0;
570 bool must_track[2] = {
571 is_exc || (FD_ISSET(fd, &fd_read) != 0), // track 0 -- read
572 is_exc || (FD_ISSET(fd, &fd_write) != 0) // track 1 -- write
573 };
Alex Deymo29b81532015-07-09 11:51:49 -0700574 MessageLoop::WatchMode watch_modes[2] = {
575 MessageLoop::WatchMode::kWatchRead,
576 MessageLoop::WatchMode::kWatchWrite,
577 };
Darin Petkov60e14152010-10-27 16:57:04 -0700578
Alex Deymo29b81532015-07-09 11:51:49 -0700579 for (size_t t = 0; t < arraysize(fd_task_maps_); ++t) {
580 auto fd_task_it = fd_task_maps_[t].find(fd);
581 bool tracked = fd_task_it != fd_task_maps_[t].end();
Darin Petkov60e14152010-10-27 16:57:04 -0700582
583 if (!must_track[t]) {
584 // If we have an outstanding io_channel, remove it.
585 if (tracked) {
Alex Deymo29b81532015-07-09 11:51:49 -0700586 MessageLoop::current()->CancelTask(fd_task_it->second);
587 fd_task_maps_[t].erase(fd_task_it);
Darin Petkov60e14152010-10-27 16:57:04 -0700588 }
589 continue;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000590 }
Darin Petkov60e14152010-10-27 16:57:04 -0700591
592 // If we are already tracking this fd, continue -- nothing to do.
593 if (tracked)
594 continue;
595
Darin Petkov60e14152010-10-27 16:57:04 -0700596 // Track a new fd.
Alex Deymo29b81532015-07-09 11:51:49 -0700597 fd_task_maps_[t][fd] = MessageLoop::current()->WatchFileDescriptor(
598 FROM_HERE,
599 fd,
600 watch_modes[t],
601 true, // persistent
602 base::Bind(&LibcurlHttpFetcher::CurlPerformOnce,
603 base::Unretained(this)));
Darin Petkov60e14152010-10-27 16:57:04 -0700604
Darin Petkov60e14152010-10-27 16:57:04 -0700605 static int io_counter = 0;
606 io_counter++;
607 if (io_counter % 50 == 0) {
608 LOG(INFO) << "io_counter = " << io_counter;
609 }
Andrew de los Reyes3270f742010-07-15 22:28:14 -0700610 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000611 }
612
Darin Petkovb83371f2010-08-17 09:34:49 -0700613 // Set up a timeout callback for libcurl.
Alex Deymo60ca1a72015-06-18 18:19:15 -0700614 if (timeout_id_ == MessageLoop::kTaskIdNull) {
Alex Deymof2858572016-02-25 11:20:13 -0800615 VLOG(1) << "Setting up timeout source: " << idle_seconds_ << " seconds.";
Alex Deymo60ca1a72015-06-18 18:19:15 -0700616 timeout_id_ = MessageLoop::current()->PostDelayedTask(
617 FROM_HERE,
618 base::Bind(&LibcurlHttpFetcher::TimeoutCallback,
619 base::Unretained(this)),
620 TimeDelta::FromSeconds(idle_seconds_));
rspangler@google.com49fdf182009-10-10 00:57:34 +0000621 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000622}
623
Alex Deymo60ca1a72015-06-18 18:19:15 -0700624void LibcurlHttpFetcher::RetryTimeoutCallback() {
Alex Deymof2858572016-02-25 11:20:13 -0800625 if (transfer_paused_) {
626 restart_transfer_on_unpause_ = true;
627 return;
628 }
Andrew de los Reyes9bbd1872010-07-16 14:52:29 -0700629 ResumeTransfer(url_);
630 CurlPerformOnce();
Andrew de los Reyes9bbd1872010-07-16 14:52:29 -0700631}
632
Alex Deymo60ca1a72015-06-18 18:19:15 -0700633void LibcurlHttpFetcher::TimeoutCallback() {
Alex Deymo60ca1a72015-06-18 18:19:15 -0700634 // We always re-schedule the callback, even if we don't want to be called
635 // anymore. We will remove the event source separately if we don't want to
Andrew de los Reyescb319332010-07-19 10:55:01 -0700636 // be called back.
Alex Deymo60ca1a72015-06-18 18:19:15 -0700637 timeout_id_ = MessageLoop::current()->PostDelayedTask(
638 FROM_HERE,
639 base::Bind(&LibcurlHttpFetcher::TimeoutCallback, base::Unretained(this)),
640 TimeDelta::FromSeconds(idle_seconds_));
Alex Deymof123ae22015-09-24 14:59:43 -0700641
642 // CurlPerformOnce() may call CleanUp(), so we need to schedule our callback
643 // first, since it could be canceled by this call.
644 if (transfer_in_progress_)
645 CurlPerformOnce();
rspangler@google.com49fdf182009-10-10 00:57:34 +0000646}
647
648void LibcurlHttpFetcher::CleanUp() {
Alex Deymo60ca1a72015-06-18 18:19:15 -0700649 MessageLoop::current()->CancelTask(timeout_id_);
650 timeout_id_ = MessageLoop::kTaskIdNull;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000651
Alex Deymo29b81532015-07-09 11:51:49 -0700652 for (size_t t = 0; t < arraysize(fd_task_maps_); ++t) {
653 for (const auto& fd_taks_pair : fd_task_maps_[t]) {
654 if (!MessageLoop::current()->CancelTask(fd_taks_pair.second)) {
655 LOG(WARNING) << "Error canceling the watch task "
656 << fd_taks_pair.second << " for "
657 << (t ? "writing" : "reading") << " the fd "
658 << fd_taks_pair.first;
659 }
Darin Petkov60e14152010-10-27 16:57:04 -0700660 }
Alex Deymo29b81532015-07-09 11:51:49 -0700661 fd_task_maps_[t].clear();
rspangler@google.com49fdf182009-10-10 00:57:34 +0000662 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000663
Gilad Arnold9dd1e7c2012-02-16 12:13:36 -0800664 if (curl_http_headers_) {
665 curl_slist_free_all(curl_http_headers_);
Alex Vakulenko88b591f2014-08-28 16:48:57 -0700666 curl_http_headers_ = nullptr;
Gilad Arnold9dd1e7c2012-02-16 12:13:36 -0800667 }
rspangler@google.com49fdf182009-10-10 00:57:34 +0000668 if (curl_handle_) {
669 if (curl_multi_handle_) {
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000670 CHECK_EQ(curl_multi_remove_handle(curl_multi_handle_, curl_handle_),
671 CURLM_OK);
rspangler@google.com49fdf182009-10-10 00:57:34 +0000672 }
673 curl_easy_cleanup(curl_handle_);
Alex Vakulenko88b591f2014-08-28 16:48:57 -0700674 curl_handle_ = nullptr;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000675 }
676 if (curl_multi_handle_) {
adlr@google.comc98a7ed2009-12-04 18:54:03 +0000677 CHECK_EQ(curl_multi_cleanup(curl_multi_handle_), CURLM_OK);
Alex Vakulenko88b591f2014-08-28 16:48:57 -0700678 curl_multi_handle_ = nullptr;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000679 }
680 transfer_in_progress_ = false;
Alex Deymof2858572016-02-25 11:20:13 -0800681 transfer_paused_ = false;
682 restart_transfer_on_unpause_ = false;
rspangler@google.com49fdf182009-10-10 00:57:34 +0000683}
684
Andrew de los Reyes3fd5d302010-10-07 20:07:18 -0700685void LibcurlHttpFetcher::GetHttpResponseCode() {
Alex Vakulenkod2779df2014-06-16 13:19:00 -0700686 long http_response_code = 0; // NOLINT(runtime/int) - curl needs long.
Alex Deymo56ccb072016-02-05 00:50:48 -0800687 if (base::StartsWith(url_, "file://", base::CompareCase::INSENSITIVE_ASCII)) {
688 // Fake out a valid response code for file:// URLs.
689 http_response_code_ = 299;
690 } else if (curl_easy_getinfo(curl_handle_,
691 CURLINFO_RESPONSE_CODE,
692 &http_response_code) == CURLE_OK) {
Andrew de los Reyes3fd5d302010-10-07 20:07:18 -0700693 http_response_code_ = static_cast<int>(http_response_code);
694 }
695}
696
rspangler@google.com49fdf182009-10-10 00:57:34 +0000697} // namespace chromeos_update_engine