tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge changes from topic "adbd_lz4"

Browse source 
* changes:
  adb: add dry-run option to push/sync.
  adb: implement LZ4 compression.
  adb: fix use of wrong union variant.
  adb: fix front_size, front_data.
  adb: add interfaces for Encoder/Decoder.
This commit is contained in:
Josh Gao 2020-04-07 21:30:30 +00:00 committed by Gerrit Code Review
commit 89cce05891
16 changed files with 1264 additions and 762 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
adb/Android.bp
View file

@ -470,6 +470,7 @@ cc_library {
"libadbd_core",
"libbrotli",
"libdiagnose_usb",
"liblz4",
],
shared_libs: [
@ -571,6 +572,7 @@ cc_library {
"libbrotli",
"libcutils_sockets",
"libdiagnose_usb",
"liblz4",
"libmdnssd",
],
@ -605,6 +607,7 @@ cc_binary {
"libadbd_services",
"libasyncio",
"libcap",
"liblz4",
"libminijail",
"libssl",
],

2
adb/client/adb_install.cpp
View file

@ -290,7 +290,7 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy)
}
}
if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) {
if (do_sync_push(apk_file, apk_dest.c_str(), false, CompressionType::Any, false)) {
result = pm_command(argc, argv);
delete_device_file(apk_dest);
}

2
adb/client/bugreport.cpp
View file

@ -282,5 +282,5 @@ int Bugreport::SendShellCommand(const std::string& command, bool disable_shell_p
bool Bugreport::DoSyncPull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
const char* name) {
return do_sync_pull(srcs, dst, copy_attrs, false, name);
return do_sync_pull(srcs, dst, copy_attrs, CompressionType::None, name);
}

98
adb/client/commandline.cpp
View file

@ -129,20 +129,22 @@ static void help() {
" reverse --remove-all remove all reverse socket connections from device\n"
"\n"
"file transfer:\n"
" push [--sync] [-zZ] LOCAL... REMOTE\n"
" push [--sync] [-z ALGORITHM] [-Z] LOCAL... REMOTE\n"
" copy local files/directories to device\n"
" --sync: only push files that are newer on the host than the device\n"
" -z: enable compression\n"
" -n: dry run: push files to device without storing to the filesystem\n"
" -z: enable compression with a specified algorithm (any, none, brotli)\n"
" -Z: disable compression\n"
" pull [-azZ] REMOTE... LOCAL\n"
" pull [-a] [-z ALGORITHM] [-Z] REMOTE... LOCAL\n"
" copy files/dirs from device\n"
" -a: preserve file timestamp and mode\n"
" -z: enable compression\n"
" -z: enable compression with a specified algorithm (any, none, brotli)\n"
" -Z: disable compression\n"
" sync [-lzZ] [all|data|odm|oem|product|system|system_ext|vendor]\n"
" sync [-l] [-z ALGORITHM] [-Z] [all|data|odm|oem|product|system|system_ext|vendor]\n"
" sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n"
" -n: dry run: push files to device without storing to the filesystem\n"
" -l: list files that would be copied, but don't copy them\n"
" -z: enable compression\n"
" -z: enable compression with a specified algorithm (any, none, brotli)\n"
" -Z: disable compression\n"
"\n"
"shell:\n"
@ -1314,12 +1316,36 @@ static int restore(int argc, const char** argv) {
return 0;
}
static CompressionType parse_compression_type(const std::string& str, bool allow_numbers) {
if (allow_numbers) {
if (str == "0") {
return CompressionType::None;
} else if (str == "1") {
return CompressionType::Any;
}
}
if (str == "any") {
return CompressionType::Any;
} else if (str == "none") {
return CompressionType::None;
}
if (str == "brotli") {
return CompressionType::Brotli;
} else if (str == "lz4") {
return CompressionType::LZ4;
}
error_exit("unexpected compression type %s", str.c_str());
}
static void parse_push_pull_args(const char** arg, int narg, std::vector<const char*>* srcs,
const char** dst, bool* copy_attrs, bool* sync, bool* compressed) {
const char** dst, bool* copy_attrs, bool* sync,
CompressionType* compression, bool* dry_run) {
*copy_attrs = false;
const char* adb_compression = getenv("ADB_COMPRESSION");
if (adb_compression && strcmp(adb_compression, "0") == 0) {
*compressed = false;
if (const char* adb_compression = getenv("ADB_COMPRESSION")) {
*compression = parse_compression_type(adb_compression, true);
}
srcs->clear();
@ -1333,13 +1359,15 @@ static void parse_push_pull_args(const char** arg, int narg, std::vector<const c
} else if (!strcmp(*arg, "-a")) {
*copy_attrs = true;
} else if (!strcmp(*arg, "-z")) {
if (compressed != nullptr) {
*compressed = true;
if (narg < 2) {
error_exit("-z requires an argument");
}
*compression = parse_compression_type(*++arg, false);
--narg;
} else if (!strcmp(*arg, "-Z")) {
if (compressed != nullptr) {
*compressed = false;
}
*compression = CompressionType::None;
} else if (dry_run && !strcmp(*arg, "-n")) {
*dry_run = true;
} else if (!strcmp(*arg, "--sync")) {
if (sync != nullptr) {
*sync = true;
@ -1894,22 +1922,25 @@ int adb_commandline(int argc, const char** argv) {
} else if (!strcmp(argv[0], "push")) {
bool copy_attrs = false;
bool sync = false;
bool compressed = true;
bool dry_run = false;
CompressionType compression = CompressionType::Any;
std::vector<const char*> srcs;
const char* dst = nullptr;
parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, &sync, &compressed);
parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, &sync, &compression,
&dry_run);
if (srcs.empty() || !dst) error_exit("push requires an argument");
return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1;
return do_sync_push(srcs, dst, sync, compression, dry_run) ? 0 : 1;
} else if (!strcmp(argv[0], "pull")) {
bool copy_attrs = false;
bool compressed = true;
CompressionType compression = CompressionType::Any;
std::vector<const char*> srcs;
const char* dst = ".";
parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, nullptr, &compressed);
parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, &copy_attrs, nullptr, &compression,
nullptr);
if (srcs.empty()) error_exit("pull requires an argument");
return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1;
return do_sync_pull(srcs, dst, copy_attrs, compression) ? 0 : 1;
} else if (!strcmp(argv[0], "install")) {
if (argc < 2) error_exit("install requires an argument");
return install_app(argc, argv);
@ -1925,27 +1956,30 @@ int adb_commandline(int argc, const char** argv) {
} else if (!strcmp(argv[0], "sync")) {
std::string src;
bool list_only = false;
bool compressed = true;
bool dry_run = false;
CompressionType compression = CompressionType::Any;
const char* adb_compression = getenv("ADB_COMPRESSION");
if (adb_compression && strcmp(adb_compression, "0") == 0) {
compressed = false;
if (const char* adb_compression = getenv("ADB_COMPRESSION"); adb_compression) {
compression = parse_compression_type(adb_compression, true);
}
int opt;
while ((opt = getopt(argc, const_cast<char**>(argv), "lzZ")) != -1) {
while ((opt = getopt(argc, const_cast<char**>(argv), "lnz:Z")) != -1) {
switch (opt) {
case 'l':
list_only = true;
break;
case 'n':
dry_run = true;
break;
case 'z':
compressed = true;
compression = parse_compression_type(optarg, false);
break;
case 'Z':
compressed = false;
compression = CompressionType::None;
break;
default:
error_exit("usage: adb sync [-lzZ] [PARTITION]");
error_exit("usage: adb sync [-l] [-n] [-z ALGORITHM] [-Z] [PARTITION]");
}
}
@ -1954,7 +1988,7 @@ int adb_commandline(int argc, const char** argv) {
} else if (optind + 1 == argc) {
src = argv[optind];
} else {
error_exit("usage: adb sync [-lzZ] [PARTITION]");
error_exit("usage: adb sync [-l] [-n] [-z ALGORITHM] [-Z] [PARTITION]");
}
std::vector<std::string> partitions{"data", "odm", "oem", "product",
@ -1965,7 +1999,9 @@ int adb_commandline(int argc, const char** argv) {
std::string src_dir{product_file(partition)};
if (!directory_exists(src_dir)) continue;
found = true;
if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1;
if (!do_sync_sync(src_dir, "/" + partition, list_only, compression, dry_run)) {
return 1;
}
}
}
if (!found) error_exit("don't know how to sync %s partition", src.c_str());

2
adb/client/fastdeploy.cpp
View file

@ -112,7 +112,7 @@ static void push_to_device(const void* data, size_t byte_count, const char* dst,
// but can't be removed until after the push.
unix_close(tf.release());
if (!do_sync_push(srcs, dst, sync, true)) {
if (!do_sync_push(srcs, dst, sync, CompressionType::Any, false)) {
error_exit("Failed to push fastdeploy agent to device.");
}
}

255
adb/client/file_sync_client.cpp
View file

@ -34,6 +34,7 @@
#include <memory>
#include <sstream>
#include <string>
#include <variant>
#include <vector>
#include "sysdeps.h"
@ -236,6 +237,8 @@ class SyncConnection {
have_ls_v2_ = CanUseFeature(features_, kFeatureLs2);
have_sendrecv_v2_ = CanUseFeature(features_, kFeatureSendRecv2);
have_sendrecv_v2_brotli_ = CanUseFeature(features_, kFeatureSendRecv2Brotli);
have_sendrecv_v2_lz4_ = CanUseFeature(features_, kFeatureSendRecv2LZ4);
have_sendrecv_v2_dry_run_send_ = CanUseFeature(features_, kFeatureSendRecv2DryRunSend);
fd.reset(adb_connect("sync:", &error));
if (fd < 0) {
Error("connect failed: %s", error.c_str());
@ -261,6 +264,22 @@ class SyncConnection {
bool HaveSendRecv2() const { return have_sendrecv_v2_; }
bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; }
bool HaveSendRecv2LZ4() const { return have_sendrecv_v2_lz4_; }
bool HaveSendRecv2DryRunSend() const { return have_sendrecv_v2_dry_run_send_; }
// Resolve a compression type which might be CompressionType::Any to a specific compression
// algorithm.
CompressionType ResolveCompressionType(CompressionType compression) const {
if (compression == CompressionType::Any) {
if (HaveSendRecv2LZ4()) {
return CompressionType::LZ4;
} else if (HaveSendRecv2Brotli()) {
return CompressionType::Brotli;
}
return CompressionType::None;
}
return compression;
}
const FeatureSet& Features() const { return features_; }
@ -323,7 +342,7 @@ class SyncConnection {
return WriteFdExactly(fd, buf.data(), buf.size());
}
bool SendSend2(std::string_view path, mode_t mode, bool compressed) {
bool SendSend2(std::string_view path, mode_t mode, CompressionType compression, bool dry_run) {
if (path.length() > 1024) {
Error("SendRequest failed: path too long: %zu", path.length());
errno = ENAMETOOLONG;
@ -339,7 +358,26 @@ class SyncConnection {
syncmsg msg;
msg.send_v2_setup.id = ID_SEND_V2;
msg.send_v2_setup.mode = mode;
msg.send_v2_setup.flags = compressed ? kSyncFlagBrotli : kSyncFlagNone;
msg.send_v2_setup.flags = 0;
switch (compression) {
case CompressionType::None:
break;
case CompressionType::Brotli:
msg.send_v2_setup.flags = kSyncFlagBrotli;
break;
case CompressionType::LZ4:
msg.send_v2_setup.flags = kSyncFlagLZ4;
break;
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
if (dry_run) {
msg.send_v2_setup.flags |= kSyncFlagDryRun;
}
buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup));
@ -352,7 +390,7 @@ class SyncConnection {
return WriteFdExactly(fd, buf.data(), buf.size());
}
bool SendRecv2(const std::string& path) {
bool SendRecv2(const std::string& path, CompressionType compression) {
if (path.length() > 1024) {
Error("SendRequest failed: path too long: %zu", path.length());
errno = ENAMETOOLONG;
@ -367,7 +405,22 @@ class SyncConnection {
syncmsg msg;
msg.recv_v2_setup.id = ID_RECV_V2;
msg.recv_v2_setup.flags = kSyncFlagBrotli;
msg.recv_v2_setup.flags = 0;
switch (compression) {
case CompressionType::None:
break;
case CompressionType::Brotli:
msg.recv_v2_setup.flags |= kSyncFlagBrotli;
break;
case CompressionType::LZ4:
msg.recv_v2_setup.flags |= kSyncFlagLZ4;
break;
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.recv_v2_setup));
@ -494,7 +547,12 @@ class SyncConnection {
// difference to "adb sync" performance.
bool SendSmallFile(const std::string& path, mode_t mode, const std::string& lpath,
const std::string& rpath, unsigned mtime, const char* data,
size_t data_length) {
size_t data_length, bool dry_run) {
if (dry_run) {
// We need to use send v2 for dry run.
return SendLargeFile(path, mode, lpath, rpath, mtime, CompressionType::None, dry_run);
}
std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode);
if (path_and_mode.length() > 1024) {
Error("SendSmallFile failed: path too long: %zu", path_and_mode.length());
@ -533,9 +591,21 @@ class SyncConnection {
return true;
}
bool SendLargeFileCompressed(const std::string& path, mode_t mode, const std::string& lpath,
const std::string& rpath, unsigned mtime) {
if (!SendSend2(path, mode, true)) {
bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath,
const std::string& rpath, unsigned mtime, CompressionType compression,
bool dry_run) {
if (dry_run && !HaveSendRecv2DryRunSend()) {
Error("dry-run not supported by the device");
return false;
}
if (!HaveSendRecv2()) {
return SendLargeFileLegacy(path, mode, lpath, rpath, mtime);
}
compression = ResolveCompressionType(compression);
if (!SendSend2(path, mode, compression, dry_run)) {
Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno));
return false;
}
@ -558,7 +628,25 @@ class SyncConnection {
syncsendbuf sbuf;
sbuf.id = ID_DATA;
BrotliEncoder<SYNC_DATA_MAX> encoder;
std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder> encoder_storage;
Encoder* encoder = nullptr;
switch (compression) {
case CompressionType::None:
encoder = &encoder_storage.emplace<NullEncoder>(SYNC_DATA_MAX);
break;
case CompressionType::Brotli:
encoder = &encoder_storage.emplace<BrotliEncoder>(SYNC_DATA_MAX);
break;
case CompressionType::LZ4:
encoder = &encoder_storage.emplace<LZ4Encoder>(SYNC_DATA_MAX);
break;
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
bool sending = true;
while (sending) {
Block input(SYNC_DATA_MAX);
@ -569,10 +657,10 @@ class SyncConnection {
}
if (r == 0) {
encoder.Finish();
encoder->Finish();
} else {
input.resize(r);
encoder.Append(std::move(input));
encoder->Append(std::move(input));
RecordBytesTransferred(r);
bytes_copied += r;
ReportProgress(rpath, bytes_copied, total_size);
@ -580,7 +668,7 @@ class SyncConnection {
while (true) {
Block output;
EncodeResult result = encoder.Encode(&output);
EncodeResult result = encoder->Encode(&output);
if (result == EncodeResult::Error) {
Error("compressing '%s' locally failed", lpath.c_str());
return false;
@ -610,12 +698,8 @@ class SyncConnection {
return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data));
}
bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath,
const std::string& rpath, unsigned mtime, bool compressed) {
if (compressed && HaveSendRecv2Brotli()) {
return SendLargeFileCompressed(path, mode, lpath, rpath, mtime);
}
bool SendLargeFileLegacy(const std::string& path, mode_t mode, const std::string& lpath,
const std::string& rpath, unsigned mtime) {
std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode);
if (!SendRequest(ID_SEND_V1, path_and_mode)) {
Error("failed to send ID_SEND_V1 message '%s': %s", path_and_mode.c_str(),
@ -840,6 +924,8 @@ class SyncConnection {
bool have_ls_v2_;
bool have_sendrecv_v2_;
bool have_sendrecv_v2_brotli_;
bool have_sendrecv_v2_lz4_;
bool have_sendrecv_v2_dry_run_send_;
TransferLedger global_ledger_;
TransferLedger current_ledger_;
@ -921,7 +1007,8 @@ static bool sync_stat_fallback(SyncConnection& sc, const std::string& path, stru
}
static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath,
unsigned mtime, mode_t mode, bool sync, bool compressed) {
unsigned mtime, mode_t mode, bool sync, CompressionType compression,
bool dry_run) {
if (sync) {
struct stat st;
if (sync_lstat(sc, rpath, &st)) {
@ -942,7 +1029,7 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s
}
buf[data_length++] = '\0';
if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length)) {
if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length, dry_run)) {
return false;
}
return sc.ReadAcknowledgements();
@ -960,11 +1047,12 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s
sc.Error("failed to read all of '%s': %s", lpath.c_str(), strerror(errno));
return false;
}
if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size())) {
if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size(),
dry_run)) {
return false;
}
} else {
if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compressed)) {
if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compression, dry_run)) {
return false;
}
}
@ -1027,8 +1115,10 @@ static bool sync_recv_v1(SyncConnection& sc, const char* rpath, const char* lpat
}
static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpath, const char* name,
uint64_t expected_size) {
if (!sc.SendRecv2(rpath)) return false;
uint64_t expected_size, CompressionType compression) {
compression = sc.ResolveCompressionType(compression);
if (!sc.SendRecv2(rpath, compression)) return false;
adb_unlink(lpath);
unique_fd lfd(adb_creat(lpath, 0644));
@ -1040,9 +1130,28 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat
uint64_t bytes_copied = 0;
Block buffer(SYNC_DATA_MAX);
BrotliDecoder decoder(std::span(buffer.data(), buffer.size()));
bool reading = true;
while (reading) {
std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder> decoder_storage;
Decoder* decoder = nullptr;
std::span buffer_span(buffer.data(), buffer.size());
switch (compression) {
case CompressionType::None:
decoder = &decoder_storage.emplace<NullDecoder>(buffer_span);
break;
case CompressionType::Brotli:
decoder = &decoder_storage.emplace<BrotliDecoder>(buffer_span);
break;
case CompressionType::LZ4:
decoder = &decoder_storage.emplace<LZ4Decoder>(buffer_span);
break;
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
while (true) {
syncmsg msg;
if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) {
adb_unlink(lpath);
@ -1050,33 +1159,32 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat
}
if (msg.data.id == ID_DONE) {
adb_unlink(lpath);
sc.Error("unexpected ID_DONE");
return false;
}
if (msg.data.id != ID_DATA) {
if (!decoder->Finish()) {
sc.Error("unexpected ID_DONE");
return false;
}
} else if (msg.data.id != ID_DATA) {
adb_unlink(lpath);
sc.ReportCopyFailure(rpath, lpath, msg);
return false;
}
} else {
if (msg.data.size > sc.max) {
sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max);
adb_unlink(lpath);
return false;
}
if (msg.data.size > sc.max) {
sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max);
adb_unlink(lpath);
return false;
Block block(msg.data.size);
if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) {
adb_unlink(lpath);
return false;
}
decoder->Append(std::move(block));
}
Block block(msg.data.size);
if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) {
adb_unlink(lpath);
return false;
}
decoder.Append(std::move(block));
while (true) {
std::span<char> output;
DecodeResult result = decoder.Decode(&output);
DecodeResult result = decoder->Decode(&output);
if (result == DecodeResult::Error) {
sc.Error("decompress failed");
@ -1093,8 +1201,7 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat
}
bytes_copied += output.size();
sc.RecordBytesTransferred(msg.data.size);
sc.RecordBytesTransferred(output.size());
sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size);
if (result == DecodeResult::NeedInput) {
@ -1102,33 +1209,19 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat
} else if (result == DecodeResult::MoreOutput) {
continue;
} else if (result == DecodeResult::Done) {
reading = false;
break;
sc.RecordFilesTransferred(1);
return true;
} else {
LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result);
}
}
}
syncmsg msg;
if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) {
sc.Error("failed to read ID_DONE");
return false;
}
if (msg.data.id != ID_DONE) {
sc.Error("unexpected message after transfer: id = %d (expected ID_DONE)", msg.data.id);
return false;
}
sc.RecordFilesTransferred(1);
return true;
}
static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, const char* name,
uint64_t expected_size, bool compressed) {
if (sc.HaveSendRecv2() && compressed) {
return sync_recv_v2(sc, rpath, lpath, name, expected_size);
uint64_t expected_size, CompressionType compression) {
if (sc.HaveSendRecv2()) {
return sync_recv_v2(sc, rpath, lpath, name, expected_size, compression);
} else {
return sync_recv_v1(sc, rpath, lpath, name, expected_size);
}
@ -1210,7 +1303,8 @@ static bool is_root_dir(std::string_view path) {
}
static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath,
bool check_timestamps, bool list_only, bool compressed) {
bool check_timestamps, bool list_only,
CompressionType compression, bool dry_run) {
sc.NewTransfer();
// Make sure that both directory paths end in a slash.
@ -1292,7 +1386,8 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st
if (list_only) {
sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str());
} else {
if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compressed)) {
if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compression,
dry_run)) {
return false;
}
}
@ -1308,7 +1403,7 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st
}
bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,
bool compressed) {
CompressionType compression, bool dry_run) {
SyncConnection sc;
if (!sc.IsValid()) return false;
@ -1373,7 +1468,8 @@ bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sy
dst_dir.append(android::base::Basename(src_path));
}
success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compressed);
success &=
copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compression, dry_run);
continue;
} else if (!should_push_file(st.st_mode)) {
sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode);
@ -1394,7 +1490,8 @@ bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sy
sc.NewTransfer();
sc.SetExpectedTotalBytes(st.st_size);
success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compressed);
success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compression,
dry_run);
sc.ReportTransferRate(src_path, TransferDirection::push);
}
@ -1480,7 +1577,7 @@ static int set_time_and_mode(const std::string& lpath, time_t time,
}
static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::string lpath,
bool copy_attrs, bool compressed) {
bool copy_attrs, CompressionType compression) {
sc.NewTransfer();
// Make sure that both directory paths end in a slash.
@ -1510,7 +1607,7 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::st
continue;
}
if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compressed)) {
if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compression)) {
return false;
}
@ -1528,7 +1625,7 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::st
}
bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
bool compressed, const char* name) {
CompressionType compression, const char* name) {
SyncConnection sc;
if (!sc.IsValid()) return false;
@ -1602,7 +1699,7 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool co
dst_dir.append(android::base::Basename(src_path));
}
success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compressed);
success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compression);
continue;
} else if (!should_pull_file(src_st.st_mode)) {
sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode);
@ -1621,7 +1718,7 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool co
sc.NewTransfer();
sc.SetExpectedTotalBytes(src_st.st_size);
if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compressed)) {
if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compression)) {
success = false;
continue;
}
@ -1638,11 +1735,11 @@ bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool co
}
bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,
bool compressed) {
CompressionType compression, bool dry_run) {
SyncConnection sc;
if (!sc.IsValid()) return false;
bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compressed);
bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compression, dry_run);
if (!list_only) {
sc.ReportOverallTransferRate(TransferDirection::push);
}

8
adb/client/file_sync_client.h
View file

@ -19,11 +19,13 @@
#include <string>
#include <vector>
#include "file_sync_protocol.h"
bool do_sync_ls(const char* path);
bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,
bool compressed);
CompressionType compression, bool dry_run);
bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
bool compressed, const char* name = nullptr);
CompressionType compression, const char* name = nullptr);
bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,
bool compressed);
CompressionType compression, bool dry_run);

287
adb/compression_utils.h
View file

@ -16,10 +16,15 @@
#pragma once
#include <algorithm>
#include <memory>
#include <span>
#include <android-base/logging.h>
#include <brotli/decode.h>
#include <brotli/encode.h>
#include <lz4frame.h>
#include "types.h"
@ -37,15 +42,103 @@ enum class EncodeResult {
MoreOutput,
};
struct BrotliDecoder {
struct Decoder {
void Append(Block&& block) { input_buffer_.append(std::move(block)); }
bool Finish() {
bool old = std::exchange(finished_, true);
if (old) {
LOG(FATAL) << "Decoder::Finish called while already finished?";
return false;
}
return true;
}
virtual DecodeResult Decode(std::span<char>* output) = 0;
protected:
Decoder(std::span<char> output_buffer) : output_buffer_(output_buffer) {}
~Decoder() = default;
bool finished_ = false;
IOVector input_buffer_;
std::span<char> output_buffer_;
};
struct Encoder {
void Append(Block input) { input_buffer_.append(std::move(input)); }
bool Finish() {
bool old = std::exchange(finished_, true);
if (old) {
LOG(FATAL) << "Decoder::Finish called while already finished?";
return false;
}
return true;
}
virtual EncodeResult Encode(Block* output) = 0;
protected:
explicit Encoder(size_t output_block_size) : output_block_size_(output_block_size) {}
~Encoder() = default;
const size_t output_block_size_;
bool finished_ = false;
IOVector input_buffer_;
};
struct NullDecoder final : public Decoder {
explicit NullDecoder(std::span<char> output_buffer) : Decoder(output_buffer) {}
DecodeResult Decode(std::span<char>* output) final {
size_t available_out = output_buffer_.size();
void* p = output_buffer_.data();
while (available_out > 0 && !input_buffer_.empty()) {
size_t len = std::min(available_out, input_buffer_.front_size());
p = mempcpy(p, input_buffer_.front_data(), len);
available_out -= len;
input_buffer_.drop_front(len);
}
*output = std::span(output_buffer_.data(), static_cast<char*>(p));
if (input_buffer_.empty()) {
return finished_ ? DecodeResult::Done : DecodeResult::NeedInput;
}
return DecodeResult::MoreOutput;
}
};
struct NullEncoder final : public Encoder {
explicit NullEncoder(size_t output_block_size) : Encoder(output_block_size) {}
EncodeResult Encode(Block* output) final {
output->clear();
output->resize(output_block_size_);
size_t available_out = output->size();
void* p = output->data();
while (available_out > 0 && !input_buffer_.empty()) {
size_t len = std::min(available_out, input_buffer_.front_size());
p = mempcpy(p, input_buffer_.front_data(), len);
available_out -= len;
input_buffer_.drop_front(len);
}
output->resize(output->size() - available_out);
if (input_buffer_.empty()) {
return finished_ ? EncodeResult::Done : EncodeResult::NeedInput;
}
return EncodeResult::MoreOutput;
}
};
struct BrotliDecoder final : public Decoder {
explicit BrotliDecoder(std::span<char> output_buffer)
: output_buffer_(output_buffer),
: Decoder(output_buffer),
decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr),
BrotliDecoderDestroyInstance) {}
void Append(Block&& block) { input_buffer_.append(std::move(block)); }
DecodeResult Decode(std::span<char>* output) {
DecodeResult Decode(std::span<char>* output) final {
size_t available_in = input_buffer_.front_size();
const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());
@ -63,7 +156,8 @@ struct BrotliDecoder {
switch (r) {
case BROTLI_DECODER_RESULT_SUCCESS:
return DecodeResult::Done;
// We need to wait for ID_DONE from the other end.
return finished_ ? DecodeResult::Done : DecodeResult::NeedInput;
case BROTLI_DECODER_RESULT_ERROR:
return DecodeResult::Error;
case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
@ -77,33 +171,29 @@ struct BrotliDecoder {
}
private:
IOVector input_buffer_;
std::span<char> output_buffer_;
std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState*)> decoder_;
};
template <size_t OutputBlockSize>
struct BrotliEncoder {
explicit BrotliEncoder()
: output_block_(OutputBlockSize),
output_bytes_left_(OutputBlockSize),
struct BrotliEncoder final : public Encoder {
explicit BrotliEncoder(size_t output_block_size)
: Encoder(output_block_size),
output_block_(output_block_size_),
output_bytes_left_(output_block_size_),
encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr),
BrotliEncoderDestroyInstance) {
BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1);
}
void Append(Block input) { input_buffer_.append(std::move(input)); }
void Finish() { finished_ = true; }
EncodeResult Encode(Block* output) {
EncodeResult Encode(Block* output) final {
output->clear();
while (true) {
size_t available_in = input_buffer_.front_size();
const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());
size_t available_out = output_bytes_left_;
uint8_t* next_out = reinterpret_cast<uint8_t*>(output_block_.data() +
(OutputBlockSize - output_bytes_left_));
uint8_t* next_out = reinterpret_cast<uint8_t*>(
output_block_.data() + (output_block_size_ - output_bytes_left_));
BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS;
if (finished_) {
@ -121,13 +211,13 @@ struct BrotliEncoder {
output_bytes_left_ = available_out;
if (BrotliEncoderIsFinished(encoder_.get())) {
output_block_.resize(OutputBlockSize - output_bytes_left_);
output_block_.resize(output_block_size_ - output_bytes_left_);
*output = std::move(output_block_);
return EncodeResult::Done;
} else if (output_bytes_left_ == 0) {
*output = std::move(output_block_);
output_block_.resize(OutputBlockSize);
output_bytes_left_ = OutputBlockSize;
output_block_.resize(output_block_size_);
output_bytes_left_ = output_block_size_;
return EncodeResult::MoreOutput;
} else if (input_buffer_.empty()) {
return EncodeResult::NeedInput;
@ -136,9 +226,158 @@ struct BrotliEncoder {
}
private:
bool finished_ = false;
IOVector input_buffer_;
Block output_block_;
size_t output_bytes_left_;
std::unique_ptr<BrotliEncoderState, void (*)(BrotliEncoderState*)> encoder_;
};
struct LZ4Decoder final : public Decoder {
explicit LZ4Decoder(std::span<char> output_buffer)
: Decoder(output_buffer), decoder_(nullptr, nullptr) {
LZ4F_dctx* dctx;
if (LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) != 0) {
LOG(FATAL) << "failed to initialize LZ4 decompression context";
}
decoder_ = std::unique_ptr<LZ4F_dctx, decltype(&LZ4F_freeDecompressionContext)>(
dctx, LZ4F_freeDecompressionContext);
}
DecodeResult Decode(std::span<char>* output) final {
size_t available_in = input_buffer_.front_size();
const char* next_in = input_buffer_.front_data();
size_t available_out = output_buffer_.size();
char* next_out = output_buffer_.data();
size_t rc = LZ4F_decompress(decoder_.get(), next_out, &available_out, next_in,
&available_in, nullptr);
if (LZ4F_isError(rc)) {
LOG(ERROR) << "LZ4F_decompress failed: " << LZ4F_getErrorName(rc);
return DecodeResult::Error;
}
input_buffer_.drop_front(available_in);
if (rc == 0) {
if (!input_buffer_.empty()) {
LOG(ERROR) << "LZ4 stream hit end before reading all data";
return DecodeResult::Error;
}
lz4_done_ = true;
}
*output = std::span<char>(output_buffer_.data(), available_out);
if (finished_) {
return input_buffer_.empty() && lz4_done_ ? DecodeResult::Done
: DecodeResult::MoreOutput;
}
return DecodeResult::NeedInput;
}
private:
bool lz4_done_ = false;
std::unique_ptr<LZ4F_dctx, LZ4F_errorCode_t (*)(LZ4F_dctx*)> decoder_;
};
struct LZ4Encoder final : public Encoder {
explicit LZ4Encoder(size_t output_block_size)
: Encoder(output_block_size), encoder_(nullptr, nullptr) {
LZ4F_cctx* cctx;
if (LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) != 0) {
LOG(FATAL) << "failed to initialize LZ4 compression context";
}
encoder_ = std::unique_ptr<LZ4F_cctx, decltype(&LZ4F_freeCompressionContext)>(
cctx, LZ4F_freeCompressionContext);
Block header(LZ4F_HEADER_SIZE_MAX);
size_t rc = LZ4F_compressBegin(encoder_.get(), header.data(), header.size(), nullptr);
if (LZ4F_isError(rc)) {
LOG(FATAL) << "LZ4F_compressBegin failed: %s", LZ4F_getErrorName(rc);
}
header.resize(rc);
output_buffer_.append(std::move(header));
}
// As an optimization, only emit a block if we have an entire output block ready, or we're done.
bool OutputReady() const {
return output_buffer_.size() >= output_block_size_ || lz4_finalized_;
}
// TODO: Switch the output type to IOVector to remove a copy?
EncodeResult Encode(Block* output) final {
size_t available_in = input_buffer_.front_size();
const char* next_in = input_buffer_.front_data();
// LZ4 makes no guarantees about being able to recover from trying to compress with an
// insufficiently large output buffer. LZ4F_compressBound tells us how much buffer we
// need to compress a given number of bytes, but the smallest value seems to be bigger
// than SYNC_DATA_MAX, so we need to buffer ourselves.
// Input size chosen to be a local maximum for LZ4F_compressBound (i.e. the block size).
constexpr size_t max_input_size = 65536;
const size_t encode_block_size = LZ4F_compressBound(max_input_size, nullptr);
if (available_in != 0) {
if (lz4_finalized_) {
LOG(ERROR) << "LZ4Encoder received data after Finish?";
return EncodeResult::Error;
}
available_in = std::min(available_in, max_input_size);
Block encode_block(encode_block_size);
size_t available_out = encode_block.capacity();
char* next_out = encode_block.data();
size_t rc = LZ4F_compressUpdate(encoder_.get(), next_out, available_out, next_in,
available_in, nullptr);
if (LZ4F_isError(rc)) {
LOG(ERROR) << "LZ4F_compressUpdate failed: " << LZ4F_getErrorName(rc);
return EncodeResult::Error;
}
input_buffer_.drop_front(available_in);
available_out -= rc;
next_out += rc;
encode_block.resize(encode_block_size - available_out);
output_buffer_.append(std::move(encode_block));
}
if (finished_ && !lz4_finalized_) {
lz4_finalized_ = true;
Block final_block(encode_block_size + 4);
size_t rc = LZ4F_compressEnd(encoder_.get(), final_block.data(), final_block.size(),
nullptr);
if (LZ4F_isError(rc)) {
LOG(ERROR) << "LZ4F_compressEnd failed: " << LZ4F_getErrorName(rc);
return EncodeResult::Error;
}
final_block.resize(rc);
output_buffer_.append(std::move(final_block));
}
if (OutputReady()) {
size_t len = std::min(output_block_size_, output_buffer_.size());
*output = output_buffer_.take_front(len).coalesce();
} else {
output->clear();
}
if (lz4_finalized_ && output_buffer_.empty()) {
return EncodeResult::Done;
} else if (OutputReady()) {
return EncodeResult::MoreOutput;
}
return EncodeResult::NeedInput;
}
private:
bool lz4_finalized_ = false;
std::unique_ptr<LZ4F_cctx, LZ4F_errorCode_t (*)(LZ4F_cctx*)> encoder_;
IOVector output_buffer_;
};

366
adb/daemon/file_sync_service.cpp
View file

@ -35,6 +35,7 @@
#include <optional>
#include <span>
#include <string>
#include <variant>
#include <vector>
#include <android-base/file.h>
@ -266,37 +267,60 @@ static bool SendSyncFailErrno(borrowed_fd fd, const std::string& reason) {
return SendSyncFail(fd, StringPrintf("%s: %s", reason.c_str(), strerror(errno)));
}
static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp) {
static bool handle_send_file_data(borrowed_fd s, unique_fd fd, uint32_t* timestamp,
CompressionType compression) {
syncmsg msg;
Block decode_buffer(SYNC_DATA_MAX);
BrotliDecoder decoder(std::span(decode_buffer.data(), decode_buffer.size()));
Block buffer(SYNC_DATA_MAX);
std::span<char> buffer_span(buffer.data(), buffer.size());
std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder> decoder_storage;
Decoder* decoder = nullptr;
switch (compression) {
case CompressionType::None:
decoder = &decoder_storage.emplace<NullDecoder>(buffer_span);
break;
case CompressionType::Brotli:
decoder = &decoder_storage.emplace<BrotliDecoder>(buffer_span);
break;
case CompressionType::LZ4:
decoder = &decoder_storage.emplace<LZ4Decoder>(buffer_span);
break;
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
while (true) {
if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false;
if (msg.data.id != ID_DATA) {
if (msg.data.id == ID_DONE) {
*timestamp = msg.data.size;
return true;
}
if (msg.data.id == ID_DONE) {
*timestamp = msg.data.size;
decoder->Finish();
} else if (msg.data.id == ID_DATA) {
Block block(msg.data.size);
if (!ReadFdExactly(s, block.data(), msg.data.size)) return false;
decoder->Append(std::move(block));
} else {
SendSyncFail(s, "invalid data message");
return false;
}
Block block(msg.data.size);
if (!ReadFdExactly(s, block.data(), msg.data.size)) return false;
decoder.Append(std::move(block));
while (true) {
std::span<char> output;
DecodeResult result = decoder.Decode(&output);
DecodeResult result = decoder->Decode(&output);
if (result == DecodeResult::Error) {
SendSyncFailErrno(s, "decompress failed");
return false;
}
if (!WriteFdExactly(fd, output.data(), output.size())) {
SendSyncFailErrno(s, "write failed");
return false;
// fd is -1 if the client is pushing with --dry-run.
if (fd != -1) {
if (!WriteFdExactly(fd, output.data(), output.size())) {
SendSyncFailErrno(s, "write failed");
return false;
}
}
if (result == DecodeResult::NeedInput) {
@ -304,7 +328,7 @@ static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* t
} else if (result == DecodeResult::MoreOutput) {
continue;
} else if (result == DecodeResult::Done) {
break;
return true;
} else {
LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result);
}
@ -314,102 +338,67 @@ static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* t
__builtin_unreachable();
}
static bool handle_send_file_uncompressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp,
std::vector<char>& buffer) {
syncmsg msg;
while (true) {
if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false;
if (msg.data.id != ID_DATA) {
if (msg.data.id == ID_DONE) {
*timestamp = msg.data.size;
return true;
}
SendSyncFail(s, "invalid data message");
return false;
}
if (msg.data.size > buffer.size()) { // TODO: resize buffer?
SendSyncFail(s, "oversize data message");
return false;
}
if (!ReadFdExactly(s, &buffer[0], msg.data.size)) return false;
if (!WriteFdExactly(fd, &buffer[0], msg.data.size)) {
SendSyncFailErrno(s, "write failed");
return false;
}
}
}
static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestamp, uid_t uid,
gid_t gid, uint64_t capabilities, mode_t mode, bool compressed,
std::vector<char>& buffer, bool do_unlink) {
int rc;
gid_t gid, uint64_t capabilities, mode_t mode,
CompressionType compression, bool dry_run, std::vector<char>& buffer,
bool do_unlink) {
syncmsg msg;
unique_fd fd;
__android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path);
unique_fd fd(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode));
if (fd < 0 && errno == ENOENT) {
if (!secure_mkdirs(Dirname(path))) {
SendSyncFailErrno(s, "secure_mkdirs failed");
goto fail;
}
if (!dry_run) {
__android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path);
fd.reset(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode));
}
if (fd < 0 && errno == EEXIST) {
fd.reset(adb_open_mode(path, O_WRONLY | O_CLOEXEC, mode));
}
if (fd < 0) {
SendSyncFailErrno(s, "couldn't create file");
goto fail;
} else {
if (fchown(fd.get(), uid, gid) == -1) {
SendSyncFailErrno(s, "fchown failed");
goto fail;
if (fd < 0 && errno == ENOENT) {
if (!secure_mkdirs(Dirname(path))) {
SendSyncFailErrno(s, "secure_mkdirs failed");
goto fail;
}
fd.reset(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode));
}
if (fd < 0 && errno == EEXIST) {
fd.reset(adb_open_mode(path, O_WRONLY | O_CLOEXEC, mode));
}
if (fd < 0) {
SendSyncFailErrno(s, "couldn't create file");
goto fail;
} else {
if (fchown(fd.get(), uid, gid) == -1) {
SendSyncFailErrno(s, "fchown failed");
goto fail;
}
#if defined(__ANDROID__)
// Not all filesystems support setting SELinux labels. http://b/23530370.
selinux_android_restorecon(path, 0);
// Not all filesystems support setting SELinux labels. http://b/23530370.
selinux_android_restorecon(path, 0);
#endif
// fchown clears the setuid bit - restore it if present.
// Ignore the result of calling fchmod. It's not supported
// by all filesystems, so we don't check for success. b/12441485
fchmod(fd.get(), mode);
}
// fchown clears the setuid bit - restore it if present.
// Ignore the result of calling fchmod. It's not supported
// by all filesystems, so we don't check for success. b/12441485
fchmod(fd.get(), mode);
}
{
rc = posix_fadvise(fd.get(), 0, 0,
POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED);
int rc = posix_fadvise(fd.get(), 0, 0,
POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED);
if (rc != 0) {
D("[ Failed to fadvise: %s ]", strerror(rc));
}
bool result;
if (compressed) {
result = handle_send_file_compressed(s, std::move(fd), timestamp);
} else {
result = handle_send_file_uncompressed(s, std::move(fd), timestamp, buffer);
}
if (!result) {
goto fail;
}
if (!update_capabilities(path, capabilities)) {
SendSyncFailErrno(s, "update_capabilities failed");
goto fail;
}
msg.status.id = ID_OKAY;
msg.status.msglen = 0;
return WriteFdExactly(s, &msg.status, sizeof(msg.status));
}
if (!handle_send_file_data(s, std::move(fd), timestamp, compression)) {
goto fail;
}
if (!update_capabilities(path, capabilities)) {
SendSyncFailErrno(s, "update_capabilities failed");
goto fail;
}
msg.status.id = ID_OKAY;
msg.status.msglen = 0;
return WriteFdExactly(s, &msg.status, sizeof(msg.status));
fail:
// If there's a problem on the device, we'll send an ID_FAIL message and
// close the socket. Unfortunately the kernel will sometimes throw that
@ -448,7 +437,7 @@ extern bool handle_send_link(int s, const std::string& path,
uint32_t* timestamp, std::vector<char>& buffer)
__attribute__((error("no symlinks on Windows")));
#else
static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp,
static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp, bool dry_run,
std::vector<char>& buffer) {
syncmsg msg;
@ -467,19 +456,21 @@ static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp
if (!ReadFdExactly(s, &buffer[0], len)) return false;
std::string buf_link;
if (!android::base::Readlink(path, &buf_link) || (buf_link != &buffer[0])) {
adb_unlink(path.c_str());
auto ret = symlink(&buffer[0], path.c_str());
if (ret && errno == ENOENT) {
if (!secure_mkdirs(Dirname(path))) {
SendSyncFailErrno(s, "secure_mkdirs failed");
if (!dry_run) {
if (!android::base::Readlink(path, &buf_link) || (buf_link != &buffer[0])) {
adb_unlink(path.c_str());
auto ret = symlink(&buffer[0], path.c_str());
if (ret && errno == ENOENT) {
if (!secure_mkdirs(Dirname(path))) {
SendSyncFailErrno(s, "secure_mkdirs failed");
return false;
}
ret = symlink(&buffer[0], path.c_str());
}
if (ret) {
SendSyncFailErrno(s, "symlink failed");
return false;
}
ret = symlink(&buffer[0], path.c_str());
}
if (ret) {
SendSyncFailErrno(s, "symlink failed");
return false;
}
}
@ -499,12 +490,15 @@ static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp
}
#endif
static bool send_impl(int s, const std::string& path, mode_t mode, bool compressed,
std::vector<char>& buffer) {
static bool send_impl(int s, const std::string& path, mode_t mode, CompressionType compression,
bool dry_run, std::vector<char>& buffer) {
// Don't delete files before copying if they are not "regular" or symlinks.
struct stat st;
bool do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) ||
(S_ISLNK(st.st_mode) && !S_ISLNK(mode));
bool do_unlink = false;
if (!dry_run) {
do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) ||
(S_ISLNK(st.st_mode) && !S_ISLNK(mode));
}
if (do_unlink) {
adb_unlink(path.c_str());
}
@ -512,7 +506,7 @@ static bool send_impl(int s, const std::string& path, mode_t mode, bool compress
bool result;
uint32_t timestamp;
if (S_ISLNK(mode)) {
result = handle_send_link(s, path, &timestamp, buffer);
result = handle_send_link(s, path, &timestamp, dry_run, buffer);
} else {
// Copy user permission bits to "group" and "other" permissions.
mode &= 0777;
@ -522,12 +516,12 @@ static bool send_impl(int s, const std::string& path, mode_t mode, bool compress
uid_t uid = -1;
gid_t gid = -1;
uint64_t capabilities = 0;
if (should_use_fs_config(path)) {
if (should_use_fs_config(path) && !dry_run) {
adbd_fs_config(path.c_str(), 0, nullptr, &uid, &gid, &mode, &capabilities);
}
result = handle_send_file(s, path.c_str(), &timestamp, uid, gid, capabilities, mode,
compressed, buffer, do_unlink);
compression, dry_run, buffer, do_unlink);
}
if (!result) {
@ -560,7 +554,7 @@ static bool do_send_v1(int s, const std::string& spec, std::vector<char>& buffer
return false;
}
return send_impl(s, path, mode, false, buffer);
return send_impl(s, path, mode, CompressionType::None, false, buffer);
}
static bool do_send_v2(int s, const std::string& path, std::vector<char>& buffer) {
@ -574,45 +568,80 @@ static bool do_send_v2(int s, const std::string& path, std::vector<char>& buffer
PLOG(ERROR) << "failed to read send_v2 setup packet";
}
bool compressed = false;
bool dry_run = false;
std::optional<CompressionType> compression;
uint32_t orig_flags = msg.send_v2_setup.flags;
if (msg.send_v2_setup.flags & kSyncFlagBrotli) {
msg.send_v2_setup.flags &= ~kSyncFlagBrotli;
compressed = true;
if (compression) {
SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
orig_flags));
return false;
}
compression = CompressionType::Brotli;
}
if (msg.send_v2_setup.flags & kSyncFlagLZ4) {
msg.send_v2_setup.flags &= ~kSyncFlagLZ4;
if (compression) {
SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
orig_flags));
return false;
}
compression = CompressionType::LZ4;
}
if (msg.send_v2_setup.flags & kSyncFlagDryRun) {
msg.send_v2_setup.flags &= ~kSyncFlagDryRun;
dry_run = true;
}
if (msg.send_v2_setup.flags) {
SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.send_v2_setup.flags));
return false;
}
errno = 0;
return send_impl(s, path, msg.send_v2_setup.mode, compressed, buffer);
return send_impl(s, path, msg.send_v2_setup.mode, compression.value_or(CompressionType::None),
dry_run, buffer);
}
static bool recv_uncompressed(borrowed_fd s, unique_fd fd, std::vector<char>& buffer) {
syncmsg msg;
msg.data.id = ID_DATA;
while (true) {
int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data));
if (r <= 0) {
if (r == 0) break;
SendSyncFailErrno(s, "read failed");
return false;
}
msg.data.size = r;
static bool recv_impl(borrowed_fd s, const char* path, CompressionType compression,
std::vector<char>& buffer) {
__android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path);
if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || !WriteFdExactly(s, &buffer[0], r)) {
return false;
}
unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC));
if (fd < 0) {
SendSyncFailErrno(s, "open failed");
return false;
}
return true;
}
int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE);
if (rc != 0) {
D("[ Failed to fadvise: %s ]", strerror(rc));
}
static bool recv_compressed(borrowed_fd s, unique_fd fd) {
syncmsg msg;
msg.data.id = ID_DATA;
BrotliEncoder<SYNC_DATA_MAX> encoder;
std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder> encoder_storage;
Encoder* encoder;
switch (compression) {
case CompressionType::None:
encoder = &encoder_storage.emplace<NullEncoder>(SYNC_DATA_MAX);
break;
case CompressionType::Brotli:
encoder = &encoder_storage.emplace<BrotliEncoder>(SYNC_DATA_MAX);
break;
case CompressionType::LZ4:
encoder = &encoder_storage.emplace<LZ4Encoder>(SYNC_DATA_MAX);
break;
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
bool sending = true;
while (sending) {
@ -624,15 +653,15 @@ static bool recv_compressed(borrowed_fd s, unique_fd fd) {
}
if (r == 0) {
encoder.Finish();
encoder->Finish();
} else {
input.resize(r);
encoder.Append(std::move(input));
encoder->Append(std::move(input));
}
while (true) {
Block output;
EncodeResult result = encoder.Encode(&output);
EncodeResult result = encoder->Encode(&output);
if (result == EncodeResult::Error) {
SendSyncFailErrno(s, "compress failed");
return false;
@ -657,42 +686,13 @@ static bool recv_compressed(borrowed_fd s, unique_fd fd) {
}
}
return true;
}
static bool recv_impl(borrowed_fd s, const char* path, bool compressed, std::vector<char>& buffer) {
__android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path);
unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC));
if (fd < 0) {
SendSyncFailErrno(s, "open failed");
return false;
}
int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE);
if (rc != 0) {
D("[ Failed to fadvise: %s ]", strerror(rc));
}
bool result;
if (compressed) {
result = recv_compressed(s, std::move(fd));
} else {
result = recv_uncompressed(s, std::move(fd), buffer);
}
if (!result) {
return false;
}
syncmsg msg;
msg.data.id = ID_DONE;
msg.data.size = 0;
return WriteFdExactly(s, &msg.data, sizeof(msg.data));
}
static bool do_recv_v1(borrowed_fd s, const char* path, std::vector<char>& buffer) {
return recv_impl(s, path, false, buffer);
return recv_impl(s, path, CompressionType::None, buffer);
}
static bool do_recv_v2(borrowed_fd s, const char* path, std::vector<char>& buffer) {
@ -706,17 +706,33 @@ static bool do_recv_v2(borrowed_fd s, const char* path, std::vector<char>& buffe
PLOG(ERROR) << "failed to read recv_v2 setup packet";
}
bool compressed = false;
std::optional<CompressionType> compression;
uint32_t orig_flags = msg.recv_v2_setup.flags;
if (msg.recv_v2_setup.flags & kSyncFlagBrotli) {
msg.recv_v2_setup.flags &= ~kSyncFlagBrotli;
compressed = true;
if (compression) {
SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
orig_flags));
return false;
}
compression = CompressionType::Brotli;
}
if (msg.recv_v2_setup.flags & kSyncFlagLZ4) {
msg.recv_v2_setup.flags &= ~kSyncFlagLZ4;
if (compression) {
SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
orig_flags));
return false;
}
compression = CompressionType::LZ4;
}
if (msg.recv_v2_setup.flags) {
SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.recv_v2_setup.flags));
return false;
}
return recv_impl(s, path, compressed, buffer);
return recv_impl(s, path, compression.value_or(CompressionType::None), buffer);
}
static const char* sync_id_to_name(uint32_t id) {

9
adb/file_sync_protocol.h
View file

@ -92,6 +92,15 @@ struct __attribute__((packed)) sync_dent_v2 {
enum SyncFlag : uint32_t {
kSyncFlagNone = 0,
kSyncFlagBrotli = 1,
kSyncFlagLZ4 = 2,
kSyncFlagDryRun = 0x8000'0000U,
};
enum class CompressionType {
None,
Any,
Brotli,
LZ4,
};
// send_v1 sent the path in a buffer, followed by a comma and the mode as a string.

963
adb/test_device.py
View file

File diff suppressed because it is too large Load diff

4
adb/transport.cpp
View file

@ -84,6 +84,8 @@ const char* const kFeatureRemountShell = "remount_shell";
const char* const kFeatureTrackApp = "track_app";
const char* const kFeatureSendRecv2 = "sendrecv_v2";
const char* const kFeatureSendRecv2Brotli = "sendrecv_v2_brotli";
const char* const kFeatureSendRecv2LZ4 = "sendrecv_v2_lz4";
const char* const kFeatureSendRecv2DryRunSend = "sendrecv_v2_dry_run_send";
namespace {
@ -1183,6 +1185,8 @@ const FeatureSet& supported_features() {
kFeatureTrackApp,
kFeatureSendRecv2,
kFeatureSendRecv2Brotli,
kFeatureSendRecv2LZ4,
kFeatureSendRecv2DryRunSend,
// Increment ADB_SERVER_VERSION when adding a feature that adbd needs
// to know about. Otherwise, the client can be stuck running an old
// version of the server even after upgrading their copy of adb.

4
adb/transport.h
View file

@ -88,6 +88,10 @@ extern const char* const kFeatureTrackApp;
extern const char* const kFeatureSendRecv2;
// adbd supports brotli for send/recv v2.
extern const char* const kFeatureSendRecv2Brotli;
// adbd supports LZ4 for send/recv v2.
extern const char* const kFeatureSendRecv2LZ4;
// adbd supports dry-run send for send/recv v2.
extern const char* const kFeatureSendRecv2DryRunSend;
TransportId NextTransportId();

2
adb/types.cpp
View file

@ -51,7 +51,7 @@ void IOVector::drop_front(IOVector::size_type len) {
auto dropped = 0u;
while (dropped < len) {
const auto next = chain_[start_index_].size() - begin_offset_;
if (dropped + next < len) {
if (dropped + next <= len) {
pop_front_block();
dropped += next;
} else {

4
adb/types.h
View file

@ -155,7 +155,7 @@ struct IOVector {
return nullptr;
}
return chain_.front().data() + begin_offset_;
return chain_[start_index_].data() + begin_offset_;
}
size_type front_size() const {
@ -163,7 +163,7 @@ struct IOVector {
return 0;
}
return chain_.front().size() - begin_offset_;
return chain_[start_index_].size() - begin_offset_;
}
size_type size() const { return chain_length_ - begin_offset_; }

17
adb/types_test.cpp
View file

@ -117,3 +117,20 @@ TEST(IOVector, misaligned_split) {
ASSERT_EQ(1ULL, bc.size());
ASSERT_EQ(create_block("x"), bc.coalesce());
}
TEST(IOVector, drop_front) {
IOVector vec;
vec.append(create_block('x', 2));
vec.append(create_block('y', 1000));
ASSERT_EQ(2U, vec.front_size());
ASSERT_EQ(1002U, vec.size());
vec.drop_front(1);
ASSERT_EQ(1U, vec.front_size());
ASSERT_EQ(1001U, vec.size());
vec.drop_front(1);
ASSERT_EQ(1000U, vec.front_size());
ASSERT_EQ(1000U, vec.size());
}