diff --git a/adb/Android.bp b/adb/Android.bp index 47dafff26..06cfcbfa4 100644 --- a/adb/Android.bp +++ b/adb/Android.bp @@ -27,6 +27,7 @@ cc_defaults { "-DADB_HOST=1", // overridden by adbd_defaults "-DALLOW_ADBD_ROOT=0", // overridden by adbd_defaults "-DANDROID_BASE_UNIQUE_FD_DISABLE_IMPLICIT_CONVERSION=1", + "-DENABLE_FASTDEPLOY=1", // enable fast deploy ], cpp_std: "experimental", @@ -270,22 +271,33 @@ cc_binary_host { "client/console.cpp", "client/adb_install.cpp", "client/line_printer.cpp", + "client/fastdeploy.cpp", + "client/fastdeploycallbacks.cpp", "shell_service_protocol.cpp", ], + generated_headers: [ + "bin2c_fastdeployagent", + "bin2c_fastdeployagentscript" + ], + static_libs: [ "libadb_host", + "libandroidfw", "libbase", "libcutils", "libcrypto_utils", "libcrypto", + "libfastdeploy_host", "libdiagnose_usb", "liblog", "libmdnssd", + "libprotobuf-cpp-lite", "libusb", "libutils", "liblog", - "libcutils", + "libziparchive", + "libz", ], stl: "libc++_static", @@ -295,10 +307,6 @@ cc_binary_host { // will violate ODR shared_libs: [], - required: [ - "deploypatchgenerator", - ], - // Archive adb, adb.exe. dist: { targets: [ @@ -658,3 +666,90 @@ python_test_host { }, }, } + +// Note: using pipe for xxd to control the variable name generated +// the default name used by xxd is the path to the input file. +java_genrule { + name: "bin2c_fastdeployagent", + out: ["deployagent.inc"], + srcs: [":deployagent"], + cmd: "(echo 'unsigned char kDeployAgent[] = {' && xxd -i <$(in) && echo '};') > $(out)", +} + +genrule { + name: "bin2c_fastdeployagentscript", + out: ["deployagentscript.inc"], + srcs: ["fastdeploy/deployagent/deployagent.sh"], + cmd: "(echo 'unsigned char kDeployAgentScript[] = {' && xxd -i <$(in) && echo '};') > $(out)", +} + +cc_library_host_static { + name: "libfastdeploy_host", + defaults: ["adb_defaults"], + srcs: [ + "fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp", + "fastdeploy/deploypatchgenerator/patch_utils.cpp", + "fastdeploy/proto/ApkEntry.proto", + ], + static_libs: [ + "libadb_host", + "libandroidfw", + "libbase", + "libcutils", + "libcrypto_utils", + "libcrypto", + "libdiagnose_usb", + "liblog", + "libmdnssd", + "libusb", + "libutils", + "libziparchive", + "libz", + ], + stl: "libc++_static", + proto: { + type: "lite", + export_proto_headers: true, + }, + target: { + windows: { + enabled: true, + shared_libs: ["AdbWinApi"], + }, + }, +} + +cc_test_host { + name: "fastdeploy_test", + defaults: ["adb_defaults"], + srcs: [ + "fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp", + "fastdeploy/deploypatchgenerator/patch_utils_test.cpp", + ], + static_libs: [ + "libadb_host", + "libandroidfw", + "libbase", + "libcutils", + "libcrypto_utils", + "libcrypto", + "libdiagnose_usb", + "libfastdeploy_host", + "liblog", + "libmdnssd", + "libprotobuf-cpp-lite", + "libusb", + "libutils", + "libziparchive", + "libz", + ], + target: { + windows: { + enabled: true, + shared_libs: ["AdbWinApi"], + }, + }, + data: [ + "fastdeploy/testdata/rotating_cube-release.apk", + ], +} diff --git a/adb/client/fastdeploy.cpp b/adb/client/fastdeploy.cpp index f4e8664a8..fbae2190d 100644 --- a/adb/client/fastdeploy.cpp +++ b/adb/client/fastdeploy.cpp @@ -27,6 +27,9 @@ #include "androidfw/ZipFileRO.h" #include "client/file_sync_client.h" #include "commandline.h" +#include "deployagent.inc" // Generated include via build rule. +#include "deployagentscript.inc" // Generated include via build rule. +#include "fastdeploy/deploypatchgenerator/deploy_patch_generator.h" #include "fastdeploycallbacks.h" #include "sysdeps.h" @@ -35,6 +38,8 @@ static constexpr long kRequiredAgentVersion = 0x00000002; static constexpr const char* kDeviceAgentPath = "/data/local/tmp/"; +static constexpr const char* kDeviceAgentFile = "/data/local/tmp/deployagent.jar"; +static constexpr const char* kDeviceAgentScript = "/data/local/tmp/deployagent"; static bool g_use_localagent = false; @@ -71,46 +76,32 @@ void fastdeploy_set_local_agent(bool use_localagent) { g_use_localagent = use_localagent; } -// local_path - must start with a '/' and be relative to $ANDROID_PRODUCT_OUT -static std::string get_agent_component_host_path(const char* local_path, const char* sdk_path) { - std::string adb_dir = android::base::GetExecutableDirectory(); - if (adb_dir.empty()) { - error_exit("Could not determine location of adb!"); - } - - if (g_use_localagent) { - const char* product_out = getenv("ANDROID_PRODUCT_OUT"); - if (product_out == nullptr) { - error_exit("Could not locate %s because $ANDROID_PRODUCT_OUT is not defined", - local_path); - } - return android::base::StringPrintf("%s%s", product_out, local_path); - } else { - return adb_dir + sdk_path; - } -} - static bool deploy_agent(bool checkTimeStamps) { std::vector srcs; - std::string jar_path = - get_agent_component_host_path("/system/framework/deployagent.jar", "/deployagent.jar"); - std::string script_path = - get_agent_component_host_path("/system/bin/deployagent", "/deployagent"); - srcs.push_back(jar_path.c_str()); - srcs.push_back(script_path.c_str()); - - if (do_sync_push(srcs, kDeviceAgentPath, checkTimeStamps)) { - // on windows the shell script might have lost execute permission - // so need to set this explicitly - const char* kChmodCommandPattern = "chmod 777 %sdeployagent"; - std::string chmodCommand = - android::base::StringPrintf(kChmodCommandPattern, kDeviceAgentPath); - int ret = send_shell_command(chmodCommand); - if (ret != 0) { - error_exit("Error executing %s returncode: %d", chmodCommand.c_str(), ret); - } - } else { - error_exit("Error pushing agent files to device"); + // TODO: Deploy agent from bin2c directly instead of writing to disk first. + TemporaryFile tempAgent; + android::base::WriteFully(tempAgent.fd, kDeployAgent, sizeof(kDeployAgent)); + srcs.push_back(tempAgent.path); + if (!do_sync_push(srcs, kDeviceAgentFile, checkTimeStamps)) { + error_exit("Failed to push fastdeploy agent to device."); + } + srcs.clear(); + // TODO: Deploy agent from bin2c directly instead of writing to disk first. + TemporaryFile tempAgentScript; + android::base::WriteFully(tempAgentScript.fd, kDeployAgentScript, sizeof(kDeployAgentScript)); + srcs.push_back(tempAgentScript.path); + if (!do_sync_push(srcs, kDeviceAgentScript, checkTimeStamps)) { + error_exit("Failed to push fastdeploy agent script to device."); + } + srcs.clear(); + // on windows the shell script might have lost execute permission + // so need to set this explicitly + const char* kChmodCommandPattern = "chmod 777 %s"; + std::string chmodCommand = + android::base::StringPrintf(kChmodCommandPattern, kDeviceAgentScript); + int ret = send_shell_command(chmodCommand); + if (ret != 0) { + error_exit("Error executing %s returncode: %d", chmodCommand.c_str(), ret); } return true; @@ -238,34 +229,15 @@ void extract_metadata(const char* apkPath, FILE* outputFp) { } } -static std::string get_patch_generator_command() { - if (g_use_localagent) { - // This should never happen on a Windows machine - const char* host_out = getenv("ANDROID_HOST_OUT"); - if (host_out == nullptr) { - error_exit( - "Could not locate deploypatchgenerator.jar because $ANDROID_HOST_OUT " - "is not defined"); - } - return android::base::StringPrintf("java -jar %s/framework/deploypatchgenerator.jar", - host_out); - } - - std::string adb_dir = android::base::GetExecutableDirectory(); - if (adb_dir.empty()) { - error_exit("Could not locate deploypatchgenerator.jar"); - } - return android::base::StringPrintf(R"(java -jar "%s/deploypatchgenerator.jar")", - adb_dir.c_str()); -} - void create_patch(const char* apkPath, const char* metadataPath, const char* patchPath) { - std::string generatePatchCommand = android::base::StringPrintf( - R"(%s "%s" "%s" > "%s")", get_patch_generator_command().c_str(), apkPath, metadataPath, - patchPath); - int returnCode = system(generatePatchCommand.c_str()); - if (returnCode != 0) { - error_exit("Executing %s returned %d", generatePatchCommand.c_str(), returnCode); + DeployPatchGenerator generator(false); + unique_fd patchFd(adb_open(patchPath, O_WRONLY | O_CREAT | O_CLOEXEC)); + if (patchFd < 0) { + perror_exit("adb: failed to create %s", patchPath); + } + bool success = generator.CreatePatch(apkPath, metadataPath, patchFd); + if (!success) { + error_exit("Failed to create patch for %s", apkPath); } } diff --git a/adb/fastdeploy/Android.bp b/adb/fastdeploy/Android.bp index 1ba0de0a9..95e1a28d5 100644 --- a/adb/fastdeploy/Android.bp +++ b/adb/fastdeploy/Android.bp @@ -13,27 +13,15 @@ // See the License for the specific language governing permissions and // limitations under the License. // - java_binary { name: "deployagent", sdk_version: "24", srcs: ["deployagent/src/**/*.java", "deploylib/src/**/*.java", "proto/**/*.proto"], static_libs: ["apkzlib_zip"], - wrapper: "deployagent/deployagent.sh", proto: { type: "lite", }, dex_preopt: { enabled: false, } -} - -java_binary_host { - name: "deploypatchgenerator", - srcs: ["deploypatchgenerator/src/**/*.java", "deploylib/src/**/*.java", "proto/**/*.proto"], - static_libs: ["apkzlib"], - manifest: "deploypatchgenerator/manifest.txt", - proto: { - type: "full", - } -} +} \ No newline at end of file diff --git a/adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java index 2d3b135d0..a8103c4d2 100644 --- a/adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java +++ b/adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java @@ -181,7 +181,7 @@ public final class DeployAgent { private static void extractMetaData(String packageName) throws IOException { File apkFile = getFileFromPackageName(packageName); APKMetaData apkMetaData = PatchUtils.getAPKMetaData(apkFile); - apkMetaData.writeDelimitedTo(System.out); + apkMetaData.writeTo(System.out); } private static int createInstallSession(String[] args) throws IOException { @@ -223,7 +223,6 @@ public final class DeployAgent { } int writeExitCode = writePatchedDataToSession(new RandomAccessFile(deviceFile, "r"), deltaStream, sessionId); - if (writeExitCode == 0) { return commitInstallSession(sessionId); } else { @@ -285,7 +284,6 @@ public final class DeployAgent { if (oldDataLen > 0) { PatchUtils.pipe(oldData, outputStream, buffer, (int) oldDataLen); } - newDataBytesWritten += copyLen + oldDataLen; } diff --git a/adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchUtils.java b/adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchUtils.java index f0f00e16c..c60f9a613 100644 --- a/adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchUtils.java +++ b/adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchUtils.java @@ -39,7 +39,7 @@ import com.android.fastdeploy.APKEntry; class PatchUtils { private static final long NEGATIVE_MASK = 1L << 63; private static final long NEGATIVE_LONG_SIGN_MASK = 1L << 63; - public static final String SIGNATURE = "HAMADI/IHD"; + public static final String SIGNATURE = "FASTDEPLOY"; private static long getOffsetFromEntry(StoredEntry entry) { return entry.getCentralDirectoryHeader().getOffset() + entry.getLocalHeaderSize(); diff --git a/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp new file mode 100644 index 000000000..22c924322 --- /dev/null +++ b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "deploy_patch_generator.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "adb_unique_fd.h" +#include "android-base/file.h" +#include "patch_utils.h" +#include "sysdeps.h" + +using namespace com::android::fastdeploy; + +void DeployPatchGenerator::Log(const char* fmt, ...) { + if (!is_verbose_) { + return; + } + va_list ap; + va_start(ap, fmt); + vprintf(fmt, ap); + printf("\n"); + va_end(ap); +} + +void DeployPatchGenerator::APKEntryToLog(const APKEntry& entry) { + Log("Filename: %s", entry.filename().c_str()); + Log("CRC32: 0x%08llX", entry.crc32()); + Log("Data Offset: %lld", entry.dataoffset()); + Log("Compressed Size: %lld", entry.compressedsize()); + Log("Uncompressed Size: %lld", entry.uncompressedsize()); +} + +void DeployPatchGenerator::APKMetaDataToLog(const char* file, const APKMetaData& metadata) { + if (!is_verbose_) { + return; + } + Log("APK Metadata: %s", file); + for (int i = 0; i < metadata.entries_size(); i++) { + const APKEntry& entry = metadata.entries(i); + APKEntryToLog(entry); + } +} + +void DeployPatchGenerator::ReportSavings(const std::vector& identicalEntries, + uint64_t totalSize) { + long totalEqualBytes = 0; + int totalEqualFiles = 0; + for (size_t i = 0; i < identicalEntries.size(); i++) { + if (identicalEntries[i].deviceEntry != nullptr) { + totalEqualBytes += identicalEntries[i].localEntry->compressedsize(); + totalEqualFiles++; + } + } + float savingPercent = (totalEqualBytes * 100.0f) / totalSize; + fprintf(stderr, "Detected %d equal APK entries\n", totalEqualFiles); + fprintf(stderr, "%ld bytes are equal out of %" PRIu64 " (%.2f%%)\n", totalEqualBytes, totalSize, + savingPercent); +} + +void DeployPatchGenerator::GeneratePatch(const std::vector& entriesToUseOnDevice, + const char* localApkPath, borrowed_fd output) { + unique_fd input(adb_open(localApkPath, O_RDONLY | O_CLOEXEC)); + size_t newApkSize = adb_lseek(input, 0L, SEEK_END); + adb_lseek(input, 0L, SEEK_SET); + + PatchUtils::WriteSignature(output); + PatchUtils::WriteLong(newApkSize, output); + size_t currentSizeOut = 0; + // Write data from the host upto the first entry we have that matches a device entry. Then write + // the metadata about the device entry and repeat for all entries that match on device. Finally + // write out any data left. If the device and host APKs are exactly the same this ends up + // writing out zip metadata from the local APK followed by offsets to the data to use from the + // device APK. + for (auto&& entry : entriesToUseOnDevice) { + int64_t deviceDataOffset = entry.deviceEntry->dataoffset(); + int64_t hostDataOffset = entry.localEntry->dataoffset(); + int64_t deviceDataLength = entry.deviceEntry->compressedsize(); + int64_t deltaFromDeviceDataStart = hostDataOffset - currentSizeOut; + PatchUtils::WriteLong(deltaFromDeviceDataStart, output); + if (deltaFromDeviceDataStart > 0) { + PatchUtils::Pipe(input, output, deltaFromDeviceDataStart); + } + PatchUtils::WriteLong(deviceDataOffset, output); + PatchUtils::WriteLong(deviceDataLength, output); + adb_lseek(input, deviceDataLength, SEEK_CUR); + currentSizeOut += deltaFromDeviceDataStart + deviceDataLength; + } + if (currentSizeOut != newApkSize) { + PatchUtils::WriteLong(newApkSize - currentSizeOut, output); + PatchUtils::Pipe(input, output, newApkSize - currentSizeOut); + PatchUtils::WriteLong(0, output); + PatchUtils::WriteLong(0, output); + } +} + +bool DeployPatchGenerator::CreatePatch(const char* localApkPath, const char* deviceApkMetadataPath, + borrowed_fd output) { + std::string content; + APKMetaData deviceApkMetadata; + if (android::base::ReadFileToString(deviceApkMetadataPath, &content)) { + deviceApkMetadata.ParsePartialFromString(content); + } else { + // TODO: What do we want to do if we don't find any metadata. + // The current fallback behavior is to build a patch with the contents of |localApkPath|. + } + + APKMetaData localApkMetadata = PatchUtils::GetAPKMetaData(localApkPath); + // Log gathered metadata info. + APKMetaDataToLog(deviceApkMetadataPath, deviceApkMetadata); + APKMetaDataToLog(localApkPath, localApkMetadata); + + std::vector identicalEntries; + uint64_t totalSize = + BuildIdenticalEntries(identicalEntries, localApkMetadata, deviceApkMetadata); + ReportSavings(identicalEntries, totalSize); + GeneratePatch(identicalEntries, localApkPath, output); + return true; +} + +uint64_t DeployPatchGenerator::BuildIdenticalEntries(std::vector& outIdenticalEntries, + const APKMetaData& localApkMetadata, + const APKMetaData& deviceApkMetadata) { + uint64_t totalSize = 0; + for (int i = 0; i < localApkMetadata.entries_size(); i++) { + const APKEntry& localEntry = localApkMetadata.entries(i); + totalSize += localEntry.compressedsize(); + for (int j = 0; j < deviceApkMetadata.entries_size(); j++) { + const APKEntry& deviceEntry = deviceApkMetadata.entries(j); + if (deviceEntry.crc32() == localEntry.crc32() && + deviceEntry.filename().compare(localEntry.filename()) == 0) { + SimpleEntry simpleEntry; + simpleEntry.localEntry = const_cast(&localEntry); + simpleEntry.deviceEntry = const_cast(&deviceEntry); + APKEntryToLog(localEntry); + outIdenticalEntries.push_back(simpleEntry); + break; + } + } + } + std::sort(outIdenticalEntries.begin(), outIdenticalEntries.end(), + [](const SimpleEntry& lhs, const SimpleEntry& rhs) { + return lhs.localEntry->dataoffset() < rhs.localEntry->dataoffset(); + }); + return totalSize; +} \ No newline at end of file diff --git a/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.h b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.h new file mode 100644 index 000000000..30e41a542 --- /dev/null +++ b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.h @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "adb_unique_fd.h" +#include "fastdeploy/proto/ApkEntry.pb.h" + +/** + * This class is responsible for creating a patch that can be accepted by the deployagent. The + * patch format is documented in GeneratePatch. + */ +class DeployPatchGenerator { + public: + /** + * Simple struct to hold mapping between local metadata and device metadata. + */ + struct SimpleEntry { + com::android::fastdeploy::APKEntry* localEntry; + com::android::fastdeploy::APKEntry* deviceEntry; + }; + + /** + * If |is_verbose| is true ApkEntries that are similar between device and host are written to + * the console. + */ + explicit DeployPatchGenerator(bool is_verbose) : is_verbose_(is_verbose) {} + /** + * Given a |localApkPath|, and the |deviceApkMetadataPath| from an installed APK this function + * writes a patch to the given |output|. + */ + bool CreatePatch(const char* localApkPath, const char* deviceApkMetadataPath, + android::base::borrowed_fd output); + + private: + bool is_verbose_; + + /** + * Log function only logs data to stdout when |is_verbose_| is true. + */ + void Log(const char* fmt, ...) __attribute__((__format__(__printf__, 2, 3))); + + /** + * Helper function to log the APKMetaData structure. If |is_verbose_| is false this function + * early outs. |file| is the path to the file represented by |metadata|. This function is used + * for debugging / information. + */ + void APKMetaDataToLog(const char* file, const com::android::fastdeploy::APKMetaData& metadata); + /** + * Helper function to log APKEntry. + */ + void APKEntryToLog(const com::android::fastdeploy::APKEntry& entry); + + /** + * Helper function to report savings by fastdeploy. This function prints out savings even with + * |is_verbose_| set to false. |totalSize| is used to show a percentage of savings. Note: + * |totalSize| is the size of the ZipEntries. Not the size of the entire file. The metadata of + * the zip data needs to be sent across with every iteration. + * [Patch format] + * |Fixed String| Signature + * |long| New Size of Apk + * |Packets[]| Array of Packets + * + * [Packet Format] + * |long| Size of data to use from patch + * |byte[]| Patch data + * |long| Offset of data to use already on device + * |long| Length of data to read from device APK + * TODO(b/138306784): Move the patch format to a proto. + */ + void ReportSavings(const std::vector& identicalEntries, uint64_t totalSize); + + /** + * This enumerates each entry in |entriesToUseOnDevice| and builds a patch file copying data + * from |localApkPath| where we are unable to use entries already on the device. The new patch + * is written to |output|. The entries are expected to be sorted by data offset from lowest to + * highest. + */ + void GeneratePatch(const std::vector& entriesToUseOnDevice, + const char* localApkPath, android::base::borrowed_fd output); + + protected: + uint64_t BuildIdenticalEntries( + std::vector& outIdenticalEntries, + const com::android::fastdeploy::APKMetaData& localApkMetadata, + const com::android::fastdeploy::APKMetaData& deviceApkMetadataPath); +}; \ No newline at end of file diff --git a/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp new file mode 100644 index 000000000..9cdc44ef1 --- /dev/null +++ b/adb/fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "deploy_patch_generator.h" +#include "patch_utils.h" + +#include +#include +#include +#include +#include + +#include "sysdeps.h" + +using namespace com::android::fastdeploy; + +static std::string GetTestFile(const std::string& name) { + return "fastdeploy/testdata/" + name; +} + +class TestPatchGenerator : DeployPatchGenerator { + public: + TestPatchGenerator() : DeployPatchGenerator(false) {} + void GatherIdenticalEntries(std::vector& outIdenticalEntries, + const APKMetaData& metadataA, const APKMetaData& metadataB) { + BuildIdenticalEntries(outIdenticalEntries, metadataA, metadataB); + } +}; + +TEST(DeployPatchGeneratorTest, IdenticalFileEntries) { + std::string apkPath = GetTestFile("rotating_cube-release.apk"); + APKMetaData metadataA = PatchUtils::GetAPKMetaData(apkPath.c_str()); + TestPatchGenerator generator; + std::vector entries; + generator.GatherIdenticalEntries(entries, metadataA, metadataA); + // Expect the entry count to match the number of entries in the metadata. + const uint32_t identicalCount = entries.size(); + const uint32_t entriesCount = metadataA.entries_size(); + EXPECT_EQ(identicalCount, entriesCount); +} + +TEST(DeployPatchGeneratorTest, NoDeviceMetadata) { + std::string apkPath = GetTestFile("rotating_cube-release.apk"); + // Get size of our test apk. + long apkSize = 0; + { + unique_fd apkFile(adb_open(apkPath.c_str(), O_RDWR)); + apkSize = adb_lseek(apkFile, 0L, SEEK_END); + } + + // Create a patch that is 100% different. + TemporaryFile output; + DeployPatchGenerator generator(true); + generator.CreatePatch(apkPath.c_str(), "", output.fd); + + // Expect a patch file that has a size at least the size of our initial APK. + long patchSize = adb_lseek(output.fd, 0L, SEEK_END); + EXPECT_GT(patchSize, apkSize); +} \ No newline at end of file diff --git a/adb/fastdeploy/deploypatchgenerator/manifest.txt b/adb/fastdeploy/deploypatchgenerator/manifest.txt deleted file mode 100644 index 5c0050597..000000000 --- a/adb/fastdeploy/deploypatchgenerator/manifest.txt +++ /dev/null @@ -1 +0,0 @@ -Main-Class: com.android.fastdeploy.DeployPatchGenerator diff --git a/adb/fastdeploy/deploypatchgenerator/patch_utils.cpp b/adb/fastdeploy/deploypatchgenerator/patch_utils.cpp new file mode 100644 index 000000000..f11ddd11f --- /dev/null +++ b/adb/fastdeploy/deploypatchgenerator/patch_utils.cpp @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "patch_utils.h" + +#include +#include + +#include "adb_io.h" +#include "android-base/endian.h" +#include "sysdeps.h" + +using namespace com::android; +using namespace com::android::fastdeploy; +using namespace android::base; + +static constexpr char kSignature[] = "FASTDEPLOY"; + +APKMetaData PatchUtils::GetAPKMetaData(const char* apkPath) { + APKMetaData apkMetaData; +#undef open + std::unique_ptr zipFile(android::ZipFileRO::open(apkPath)); +#define open ___xxx_unix_open + if (zipFile == nullptr) { + printf("Could not open %s", apkPath); + exit(1); + } + void* cookie; + if (zipFile->startIteration(&cookie)) { + android::ZipEntryRO entry; + while ((entry = zipFile->nextEntry(cookie)) != NULL) { + char fileName[256]; + // Make sure we have a file name. + // TODO: Handle filenames longer than 256. + if (zipFile->getEntryFileName(entry, fileName, sizeof(fileName))) { + continue; + } + + uint32_t uncompressedSize, compressedSize, crc32; + int64_t dataOffset; + zipFile->getEntryInfo(entry, nullptr, &uncompressedSize, &compressedSize, &dataOffset, + nullptr, &crc32); + APKEntry* apkEntry = apkMetaData.add_entries(); + apkEntry->set_crc32(crc32); + apkEntry->set_filename(fileName); + apkEntry->set_compressedsize(compressedSize); + apkEntry->set_uncompressedsize(uncompressedSize); + apkEntry->set_dataoffset(dataOffset); + } + } + return apkMetaData; +} + +void PatchUtils::WriteSignature(borrowed_fd output) { + WriteFdExactly(output, kSignature, sizeof(kSignature) - 1); +} + +void PatchUtils::WriteLong(int64_t value, borrowed_fd output) { + int64_t toLittleEndian = htole64(value); + WriteFdExactly(output, &toLittleEndian, sizeof(int64_t)); +} + +void PatchUtils::Pipe(borrowed_fd input, borrowed_fd output, size_t amount) { + constexpr static int BUFFER_SIZE = 128 * 1024; + char buffer[BUFFER_SIZE]; + size_t transferAmount = 0; + while (transferAmount != amount) { + long chunkAmount = + amount - transferAmount > BUFFER_SIZE ? BUFFER_SIZE : amount - transferAmount; + long readAmount = adb_read(input, buffer, chunkAmount); + WriteFdExactly(output, buffer, readAmount); + transferAmount += readAmount; + } +} \ No newline at end of file diff --git a/adb/fastdeploy/deploypatchgenerator/patch_utils.h b/adb/fastdeploy/deploypatchgenerator/patch_utils.h new file mode 100644 index 000000000..0ebfe8fe3 --- /dev/null +++ b/adb/fastdeploy/deploypatchgenerator/patch_utils.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "adb_unique_fd.h" +#include "fastdeploy/proto/ApkEntry.pb.h" + +/** + * Helper class that mirrors the PatchUtils from deploy agent. + */ +class PatchUtils { + public: + /** + * This function takes a local APK file and builds the APKMetaData required by the patching + * algorithm. The if this function has an error a string is printed to the terminal and exit(1) + * is called. + */ + static com::android::fastdeploy::APKMetaData GetAPKMetaData(const char* file); + /** + * Writes a fixed signature string to the header of the patch. + */ + static void WriteSignature(android::base::borrowed_fd output); + /** + * Writes an int64 to the |output| reversing the bytes. + */ + static void WriteLong(int64_t value, android::base::borrowed_fd output); + /** + * Copy |amount| of data from |input| to |output|. + */ + static void Pipe(android::base::borrowed_fd input, android::base::borrowed_fd output, + size_t amount); +}; \ No newline at end of file diff --git a/adb/fastdeploy/deploypatchgenerator/patch_utils_test.cpp b/adb/fastdeploy/deploypatchgenerator/patch_utils_test.cpp new file mode 100644 index 000000000..a7eeebfc7 --- /dev/null +++ b/adb/fastdeploy/deploypatchgenerator/patch_utils_test.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "patch_utils.h" + +#include +#include +#include +#include +#include +#include + +#include "adb_io.h" +#include "sysdeps.h" + +using namespace com::android::fastdeploy; + +static std::string GetTestFile(const std::string& name) { + return "fastdeploy/testdata/" + name; +} + +bool FileMatchesContent(android::base::borrowed_fd input, const char* contents, + ssize_t contentsSize) { + adb_lseek(input, 0, SEEK_SET); + // Use a temp buffer larger than any test contents. + constexpr int BUFFER_SIZE = 2048; + char buffer[BUFFER_SIZE]; + bool result = true; + // Validate size of files is equal. + ssize_t readAmount = adb_read(input, buffer, BUFFER_SIZE); + EXPECT_EQ(readAmount, contentsSize); + result = memcmp(buffer, contents, readAmount) == 0; + for (int i = 0; i < readAmount; i++) { + printf("%x", buffer[i]); + } + printf(" == "); + for (int i = 0; i < contentsSize; i++) { + printf("%x", contents[i]); + } + printf("\n"); + + return result; +} + +TEST(PatchUtilsTest, SwapLongWrites) { + TemporaryFile output; + PatchUtils::WriteLong(0x0011223344556677, output.fd); + adb_lseek(output.fd, 0, SEEK_SET); + const char expected[] = {0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x00}; + EXPECT_TRUE(FileMatchesContent(output.fd, expected, 8)); +} + +TEST(PatchUtilsTest, PipeWritesAmountToOutput) { + std::string expected("Some Data"); + TemporaryFile input; + TemporaryFile output; + // Populate input file. + WriteFdExactly(input.fd, expected); + adb_lseek(input.fd, 0, SEEK_SET); + // Open input file for read, and output file for write. + PatchUtils::Pipe(input.fd, output.fd, expected.size()); + // Validate pipe worked + EXPECT_TRUE(FileMatchesContent(output.fd, expected.c_str(), expected.size())); +} + +TEST(PatchUtilsTest, SignatureConstMatches) { + std::string apkFile = GetTestFile("rotating_cube-release.apk"); + TemporaryFile output; + PatchUtils::WriteSignature(output.fd); + std::string contents("FASTDEPLOY"); + EXPECT_TRUE(FileMatchesContent(output.fd, contents.c_str(), contents.size())); +} + +TEST(PatchUtilsTest, GatherMetadata) { + std::string apkFile = GetTestFile("rotating_cube-release.apk"); + APKMetaData metadata = PatchUtils::GetAPKMetaData(apkFile.c_str()); + std::string expectedMetadata; + android::base::ReadFileToString(GetTestFile("rotating_cube-metadata-release.data"), + &expectedMetadata); + std::string actualMetadata; + metadata.SerializeToString(&actualMetadata); + EXPECT_EQ(expectedMetadata, actualMetadata); +} \ No newline at end of file diff --git a/adb/fastdeploy/deploypatchgenerator/src/com/android/fastdeploy/DeployPatchGenerator.java b/adb/fastdeploy/deploypatchgenerator/src/com/android/fastdeploy/DeployPatchGenerator.java deleted file mode 100644 index 24b2eab04..000000000 --- a/adb/fastdeploy/deploypatchgenerator/src/com/android/fastdeploy/DeployPatchGenerator.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.fastdeploy; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.StringBuilder; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.ArrayList; - -import java.nio.charset.StandardCharsets; -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.AbstractMap.SimpleEntry; - -import com.android.fastdeploy.APKMetaData; -import com.android.fastdeploy.APKEntry; - -public final class DeployPatchGenerator { - private static final int BUFFER_SIZE = 128 * 1024; - - public static void main(String[] args) { - try { - if (args.length < 2) { - showUsage(0); - } - - boolean verbose = false; - if (args.length > 2) { - String verboseFlag = args[2]; - if (verboseFlag.compareTo("--verbose") == 0) { - verbose = true; - } - } - - StringBuilder sb = null; - String apkPath = args[0]; - String deviceMetadataPath = args[1]; - File hostFile = new File(apkPath); - - List deviceZipEntries = getMetadataFromFile(deviceMetadataPath); - System.err.println("Device Entries (" + deviceZipEntries.size() + ")"); - if (verbose) { - sb = new StringBuilder(); - for (APKEntry entry : deviceZipEntries) { - APKEntryToString(entry, sb); - } - System.err.println(sb.toString()); - } - - List hostFileEntries = PatchUtils.getAPKMetaData(hostFile).getEntriesList(); - System.err.println("Host Entries (" + hostFileEntries.size() + ")"); - if (verbose) { - sb = new StringBuilder(); - for (APKEntry entry : hostFileEntries) { - APKEntryToString(entry, sb); - } - System.err.println(sb.toString()); - } - - List> identicalContentsEntrySet = - getIdenticalContents(deviceZipEntries, hostFileEntries); - reportIdenticalContents(identicalContentsEntrySet, hostFile); - - if (verbose) { - sb = new StringBuilder(); - for (SimpleEntry identicalEntry : identicalContentsEntrySet) { - APKEntry entry = identicalEntry.getValue(); - APKEntryToString(entry, sb); - } - System.err.println("Identical Entries (" + identicalContentsEntrySet.size() + ")"); - System.err.println(sb.toString()); - } - - createPatch(identicalContentsEntrySet, hostFile, System.out); - } catch (Exception e) { - System.err.println("Error: " + e); - e.printStackTrace(); - System.exit(2); - } - System.exit(0); - } - - private static void showUsage(int exitCode) { - System.err.println("usage: deploypatchgenerator [--verbose]"); - System.err.println(""); - System.exit(exitCode); - } - - private static void APKEntryToString(APKEntry entry, StringBuilder outputString) { - outputString.append(String.format("Filename: %s\n", entry.getFileName())); - outputString.append(String.format("CRC32: 0x%08X\n", entry.getCrc32())); - outputString.append(String.format("Data Offset: %d\n", entry.getDataOffset())); - outputString.append(String.format("Compressed Size: %d\n", entry.getCompressedSize())); - outputString.append(String.format("Uncompressed Size: %d\n", entry.getUncompressedSize())); - } - - private static List getMetadataFromFile(String deviceMetadataPath) throws IOException { - InputStream is = new FileInputStream(new File(deviceMetadataPath)); - APKMetaData apkMetaData = APKMetaData.parseDelimitedFrom(is); - return apkMetaData.getEntriesList(); - } - - private static List> getIdenticalContents( - List deviceZipEntries, List hostZipEntries) throws IOException { - List> identicalContents = - new ArrayList>(); - - for (APKEntry deviceZipEntry : deviceZipEntries) { - for (APKEntry hostZipEntry : hostZipEntries) { - if (deviceZipEntry.getCrc32() == hostZipEntry.getCrc32() && - deviceZipEntry.getFileName().equals(hostZipEntry.getFileName())) { - identicalContents.add(new SimpleEntry(deviceZipEntry, hostZipEntry)); - } - } - } - - Collections.sort(identicalContents, new Comparator>() { - @Override - public int compare( - SimpleEntry p1, SimpleEntry p2) { - return Long.compare(p1.getValue().getDataOffset(), p2.getValue().getDataOffset()); - } - }); - - return identicalContents; - } - - private static void reportIdenticalContents( - List> identicalContentsEntrySet, File hostFile) - throws IOException { - long totalEqualBytes = 0; - int totalEqualFiles = 0; - for (SimpleEntry entries : identicalContentsEntrySet) { - APKEntry hostAPKEntry = entries.getValue(); - totalEqualBytes += hostAPKEntry.getCompressedSize(); - totalEqualFiles++; - } - - float savingPercent = (float) (totalEqualBytes * 100) / hostFile.length(); - - System.err.println("Detected " + totalEqualFiles + " equal APK entries"); - System.err.println(totalEqualBytes + " bytes are equal out of " + hostFile.length() + " (" - + savingPercent + "%)"); - } - - static void createPatch(List> zipEntrySimpleEntrys, - File hostFile, OutputStream patchStream) throws IOException, PatchFormatException { - FileInputStream hostFileInputStream = new FileInputStream(hostFile); - - patchStream.write(PatchUtils.SIGNATURE.getBytes(StandardCharsets.US_ASCII)); - PatchUtils.writeFormattedLong(hostFile.length(), patchStream); - - byte[] buffer = new byte[BUFFER_SIZE]; - long totalBytesWritten = 0; - Iterator> entrySimpleEntryIterator = - zipEntrySimpleEntrys.iterator(); - while (entrySimpleEntryIterator.hasNext()) { - SimpleEntry entrySimpleEntry = entrySimpleEntryIterator.next(); - APKEntry deviceAPKEntry = entrySimpleEntry.getKey(); - APKEntry hostAPKEntry = entrySimpleEntry.getValue(); - - long newDataLen = hostAPKEntry.getDataOffset() - totalBytesWritten; - long oldDataOffset = deviceAPKEntry.getDataOffset(); - long oldDataLen = deviceAPKEntry.getCompressedSize(); - - PatchUtils.writeFormattedLong(newDataLen, patchStream); - PatchUtils.pipe(hostFileInputStream, patchStream, buffer, newDataLen); - PatchUtils.writeFormattedLong(oldDataOffset, patchStream); - PatchUtils.writeFormattedLong(oldDataLen, patchStream); - - long skip = hostFileInputStream.skip(oldDataLen); - if (skip != oldDataLen) { - throw new PatchFormatException("skip error: attempted to skip " + oldDataLen - + " bytes but return code was " + skip); - } - totalBytesWritten += oldDataLen + newDataLen; - } - long remainderLen = hostFile.length() - totalBytesWritten; - PatchUtils.writeFormattedLong(remainderLen, patchStream); - PatchUtils.pipe(hostFileInputStream, patchStream, buffer, remainderLen); - PatchUtils.writeFormattedLong(0, patchStream); - PatchUtils.writeFormattedLong(0, patchStream); - patchStream.flush(); - } -} diff --git a/adb/fastdeploy/testdata/rotating_cube-metadata-release.data b/adb/fastdeploy/testdata/rotating_cube-metadata-release.data new file mode 100644 index 000000000..0671bf352 --- /dev/null +++ b/adb/fastdeploy/testdata/rotating_cube-metadata-release.data @@ -0,0 +1,6 @@ + +#ǂϫ META-INF/MANIFEST.MF Q(W +#AndroidManifest.xml1 ( +6>#lib/armeabi-v7a/libvulkan_sample.so Q( + 㗑resources.arscQ ( + classes.dex ( diff --git a/adb/fastdeploy/testdata/rotating_cube-release.apk b/adb/fastdeploy/testdata/rotating_cube-release.apk new file mode 100644 index 000000000..d47e0ea07 Binary files /dev/null and b/adb/fastdeploy/testdata/rotating_cube-release.apk differ