tequilaOS/platform_system_core
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Fastdeploy converted to c++ and bin2c on the jar."

Browse source 
am: 60fa8b8ad2

Change-Id: Ied5f860f08599c5ab236ecca084effbb90f93285
This commit is contained in:
Joshua Gilpatrick 2019-07-30 15:46:02 -07:00 committed by android-build-merger
commit a84882427e
15 changed files with 716 additions and 296 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

105
adb/Android.bp
View file

@ -27,6 +27,7 @@ cc_defaults {
"-DADB_HOST=1", // overridden by adbd_defaults
"-DALLOW_ADBD_ROOT=0", // overridden by adbd_defaults
"-DANDROID_BASE_UNIQUE_FD_DISABLE_IMPLICIT_CONVERSION=1",
"-DENABLE_FASTDEPLOY=1", // enable fast deploy
],
cpp_std: "experimental",
@ -270,22 +271,33 @@ cc_binary_host {
"client/console.cpp",
"client/adb_install.cpp",
"client/line_printer.cpp",
"client/fastdeploy.cpp",
"client/fastdeploycallbacks.cpp",
"shell_service_protocol.cpp",
],
generated_headers: [
"bin2c_fastdeployagent",
"bin2c_fastdeployagentscript"
],
static_libs: [
"libadb_host",
"libandroidfw",
"libbase",
"libcutils",
"libcrypto_utils",
"libcrypto",
"libfastdeploy_host",
"libdiagnose_usb",
"liblog",
"libmdnssd",
"libprotobuf-cpp-lite",
"libusb",
"libutils",
"liblog",
"libcutils",
"libziparchive",
"libz",
],
stl: "libc++_static",
@ -295,10 +307,6 @@ cc_binary_host {
// will violate ODR
shared_libs: [],
required: [
"deploypatchgenerator",
],
// Archive adb, adb.exe.
dist: {
targets: [
@ -658,3 +666,90 @@ python_test_host {
},
},
}
// Note: using pipe for xxd to control the variable name generated
// the default name used by xxd is the path to the input file.
java_genrule {
name: "bin2c_fastdeployagent",
out: ["deployagent.inc"],
srcs: [":deployagent"],
cmd: "(echo 'unsigned char kDeployAgent[] = {' && xxd -i <$(in) && echo '};') > $(out)",
}
genrule {
name: "bin2c_fastdeployagentscript",
out: ["deployagentscript.inc"],
srcs: ["fastdeploy/deployagent/deployagent.sh"],
cmd: "(echo 'unsigned char kDeployAgentScript[] = {' && xxd -i <$(in) && echo '};') > $(out)",
}
cc_library_host_static {
name: "libfastdeploy_host",
defaults: ["adb_defaults"],
srcs: [
"fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp",
"fastdeploy/deploypatchgenerator/patch_utils.cpp",
"fastdeploy/proto/ApkEntry.proto",
],
static_libs: [
"libadb_host",
"libandroidfw",
"libbase",
"libcutils",
"libcrypto_utils",
"libcrypto",
"libdiagnose_usb",
"liblog",
"libmdnssd",
"libusb",
"libutils",
"libziparchive",
"libz",
],
stl: "libc++_static",
proto: {
type: "lite",
export_proto_headers: true,
},
target: {
windows: {
enabled: true,
shared_libs: ["AdbWinApi"],
},
},
}
cc_test_host {
name: "fastdeploy_test",
defaults: ["adb_defaults"],
srcs: [
"fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp",
"fastdeploy/deploypatchgenerator/patch_utils_test.cpp",
],
static_libs: [
"libadb_host",
"libandroidfw",
"libbase",
"libcutils",
"libcrypto_utils",
"libcrypto",
"libdiagnose_usb",
"libfastdeploy_host",
"liblog",
"libmdnssd",
"libprotobuf-cpp-lite",
"libusb",
"libutils",
"libziparchive",
"libz",
],
target: {
windows: {
enabled: true,
shared_libs: ["AdbWinApi"],
},
},
data: [
"fastdeploy/testdata/rotating_cube-release.apk",
],
}

102
adb/client/fastdeploy.cpp
View file

@ -27,6 +27,9 @@
#include "androidfw/ZipFileRO.h"
#include "client/file_sync_client.h"
#include "commandline.h"
#include "deployagent.inc" // Generated include via build rule.
#include "deployagentscript.inc" // Generated include via build rule.
#include "fastdeploy/deploypatchgenerator/deploy_patch_generator.h"
#include "fastdeploycallbacks.h"
#include "sysdeps.h"
@ -35,6 +38,8 @@
static constexpr long kRequiredAgentVersion = 0x00000002;
static constexpr const char* kDeviceAgentPath = "/data/local/tmp/";
static constexpr const char* kDeviceAgentFile = "/data/local/tmp/deployagent.jar";
static constexpr const char* kDeviceAgentScript = "/data/local/tmp/deployagent";
static bool g_use_localagent = false;
@ -71,46 +76,32 @@ void fastdeploy_set_local_agent(bool use_localagent) {
g_use_localagent = use_localagent;
}
// local_path - must start with a '/' and be relative to $ANDROID_PRODUCT_OUT
static std::string get_agent_component_host_path(const char* local_path, const char* sdk_path) {
std::string adb_dir = android::base::GetExecutableDirectory();
if (adb_dir.empty()) {
error_exit("Could not determine location of adb!");
}
if (g_use_localagent) {
const char* product_out = getenv("ANDROID_PRODUCT_OUT");
if (product_out == nullptr) {
error_exit("Could not locate %s because $ANDROID_PRODUCT_OUT is not defined",
local_path);
}
return android::base::StringPrintf("%s%s", product_out, local_path);
} else {
return adb_dir + sdk_path;
}
}
static bool deploy_agent(bool checkTimeStamps) {
std::vector<const char*> srcs;
std::string jar_path =
get_agent_component_host_path("/system/framework/deployagent.jar", "/deployagent.jar");
std::string script_path =
get_agent_component_host_path("/system/bin/deployagent", "/deployagent");
srcs.push_back(jar_path.c_str());
srcs.push_back(script_path.c_str());
if (do_sync_push(srcs, kDeviceAgentPath, checkTimeStamps)) {
// on windows the shell script might have lost execute permission
// so need to set this explicitly
const char* kChmodCommandPattern = "chmod 777 %sdeployagent";
std::string chmodCommand =
android::base::StringPrintf(kChmodCommandPattern, kDeviceAgentPath);
int ret = send_shell_command(chmodCommand);
if (ret != 0) {
error_exit("Error executing %s returncode: %d", chmodCommand.c_str(), ret);
}
} else {
error_exit("Error pushing agent files to device");
// TODO: Deploy agent from bin2c directly instead of writing to disk first.
TemporaryFile tempAgent;
android::base::WriteFully(tempAgent.fd, kDeployAgent, sizeof(kDeployAgent));
srcs.push_back(tempAgent.path);
if (!do_sync_push(srcs, kDeviceAgentFile, checkTimeStamps)) {
error_exit("Failed to push fastdeploy agent to device.");
}
srcs.clear();
// TODO: Deploy agent from bin2c directly instead of writing to disk first.
TemporaryFile tempAgentScript;
android::base::WriteFully(tempAgentScript.fd, kDeployAgentScript, sizeof(kDeployAgentScript));
srcs.push_back(tempAgentScript.path);
if (!do_sync_push(srcs, kDeviceAgentScript, checkTimeStamps)) {
error_exit("Failed to push fastdeploy agent script to device.");
}
srcs.clear();
// on windows the shell script might have lost execute permission
// so need to set this explicitly
const char* kChmodCommandPattern = "chmod 777 %s";
std::string chmodCommand =
android::base::StringPrintf(kChmodCommandPattern, kDeviceAgentScript);
int ret = send_shell_command(chmodCommand);
if (ret != 0) {
error_exit("Error executing %s returncode: %d", chmodCommand.c_str(), ret);
}
return true;
@ -238,34 +229,15 @@ void extract_metadata(const char* apkPath, FILE* outputFp) {
}
}
static std::string get_patch_generator_command() {
if (g_use_localagent) {
// This should never happen on a Windows machine
const char* host_out = getenv("ANDROID_HOST_OUT");
if (host_out == nullptr) {
error_exit(
"Could not locate deploypatchgenerator.jar because $ANDROID_HOST_OUT "
"is not defined");
}
return android::base::StringPrintf("java -jar %s/framework/deploypatchgenerator.jar",
host_out);
}
std::string adb_dir = android::base::GetExecutableDirectory();
if (adb_dir.empty()) {
error_exit("Could not locate deploypatchgenerator.jar");
}
return android::base::StringPrintf(R"(java -jar "%s/deploypatchgenerator.jar")",
adb_dir.c_str());
}
void create_patch(const char* apkPath, const char* metadataPath, const char* patchPath) {
std::string generatePatchCommand = android::base::StringPrintf(
R"(%s "%s" "%s" > "%s")", get_patch_generator_command().c_str(), apkPath, metadataPath,
patchPath);
int returnCode = system(generatePatchCommand.c_str());
if (returnCode != 0) {
error_exit("Executing %s returned %d", generatePatchCommand.c_str(), returnCode);
DeployPatchGenerator generator(false);
unique_fd patchFd(adb_open(patchPath, O_WRONLY | O_CREAT | O_CLOEXEC));
if (patchFd < 0) {
perror_exit("adb: failed to create %s", patchPath);
}
bool success = generator.CreatePatch(apkPath, metadataPath, patchFd);
if (!success) {
error_exit("Failed to create patch for %s", apkPath);
}
}

14
adb/fastdeploy/Android.bp
View file

@ -13,27 +13,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//
java_binary {
name: "deployagent",
sdk_version: "24",
srcs: ["deployagent/src/**/*.java", "deploylib/src/**/*.java", "proto/**/*.proto"],
static_libs: ["apkzlib_zip"],
wrapper: "deployagent/deployagent.sh",
proto: {
type: "lite",
},
dex_preopt: {
enabled: false,
}
}
java_binary_host {
name: "deploypatchgenerator",
srcs: ["deploypatchgenerator/src/**/*.java", "deploylib/src/**/*.java", "proto/**/*.proto"],
static_libs: ["apkzlib"],
manifest: "deploypatchgenerator/manifest.txt",
proto: {
type: "full",
}
}
}

4
adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java
View file

@ -181,7 +181,7 @@ public final class DeployAgent {
private static void extractMetaData(String packageName) throws IOException {
File apkFile = getFileFromPackageName(packageName);
APKMetaData apkMetaData = PatchUtils.getAPKMetaData(apkFile);
apkMetaData.writeDelimitedTo(System.out);
apkMetaData.writeTo(System.out);
}
private static int createInstallSession(String[] args) throws IOException {
@ -223,7 +223,6 @@ public final class DeployAgent {
}
int writeExitCode = writePatchedDataToSession(new RandomAccessFile(deviceFile, "r"), deltaStream, sessionId);
if (writeExitCode == 0) {
return commitInstallSession(sessionId);
} else {
@ -285,7 +284,6 @@ public final class DeployAgent {
if (oldDataLen > 0) {
PatchUtils.pipe(oldData, outputStream, buffer, (int) oldDataLen);
}
newDataBytesWritten += copyLen + oldDataLen;
}

2
adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchUtils.java
View file

@ -39,7 +39,7 @@ import com.android.fastdeploy.APKEntry;
class PatchUtils {
private static final long NEGATIVE_MASK = 1L << 63;
private static final long NEGATIVE_LONG_SIGN_MASK = 1L << 63;
public static final String SIGNATURE = "HAMADI/IHD";
public static final String SIGNATURE = "FASTDEPLOY";
private static long getOffsetFromEntry(StoredEntry entry) {
return entry.getCentralDirectoryHeader().getOffset() + entry.getLocalHeaderSize();

167
adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp Normal file
View file

@ -0,0 +1,167 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "deploy_patch_generator.h"
#include <inttypes.h>
#include <stdio.h>
#include <algorithm>
#include <fstream>
#include <functional>
#include <iostream>
#include <sstream>
#include <string>
#include "adb_unique_fd.h"
#include "android-base/file.h"
#include "patch_utils.h"
#include "sysdeps.h"
using namespace com::android::fastdeploy;
void DeployPatchGenerator::Log(const char* fmt, ...) {
if (!is_verbose_) {
return;
}
va_list ap;
va_start(ap, fmt);
vprintf(fmt, ap);
printf("\n");
va_end(ap);
}
void DeployPatchGenerator::APKEntryToLog(const APKEntry& entry) {
Log("Filename: %s", entry.filename().c_str());
Log("CRC32: 0x%08llX", entry.crc32());
Log("Data Offset: %lld", entry.dataoffset());
Log("Compressed Size: %lld", entry.compressedsize());
Log("Uncompressed Size: %lld", entry.uncompressedsize());
}
void DeployPatchGenerator::APKMetaDataToLog(const char* file, const APKMetaData& metadata) {
if (!is_verbose_) {
return;
}
Log("APK Metadata: %s", file);
for (int i = 0; i < metadata.entries_size(); i++) {
const APKEntry& entry = metadata.entries(i);
APKEntryToLog(entry);
}
}
void DeployPatchGenerator::ReportSavings(const std::vector<SimpleEntry>& identicalEntries,
uint64_t totalSize) {
long totalEqualBytes = 0;
int totalEqualFiles = 0;
for (size_t i = 0; i < identicalEntries.size(); i++) {
if (identicalEntries[i].deviceEntry != nullptr) {
totalEqualBytes += identicalEntries[i].localEntry->compressedsize();
totalEqualFiles++;
}
}
float savingPercent = (totalEqualBytes * 100.0f) / totalSize;
fprintf(stderr, "Detected %d equal APK entries\n", totalEqualFiles);
fprintf(stderr, "%ld bytes are equal out of %" PRIu64 " (%.2f%%)\n", totalEqualBytes, totalSize,
savingPercent);
}
void DeployPatchGenerator::GeneratePatch(const std::vector<SimpleEntry>& entriesToUseOnDevice,
const char* localApkPath, borrowed_fd output) {
unique_fd input(adb_open(localApkPath, O_RDONLY | O_CLOEXEC));
size_t newApkSize = adb_lseek(input, 0L, SEEK_END);
adb_lseek(input, 0L, SEEK_SET);
PatchUtils::WriteSignature(output);
PatchUtils::WriteLong(newApkSize, output);
size_t currentSizeOut = 0;
// Write data from the host upto the first entry we have that matches a device entry. Then write
// the metadata about the device entry and repeat for all entries that match on device. Finally
// write out any data left. If the device and host APKs are exactly the same this ends up
// writing out zip metadata from the local APK followed by offsets to the data to use from the
// device APK.
for (auto&& entry : entriesToUseOnDevice) {
int64_t deviceDataOffset = entry.deviceEntry->dataoffset();
int64_t hostDataOffset = entry.localEntry->dataoffset();
int64_t deviceDataLength = entry.deviceEntry->compressedsize();
int64_t deltaFromDeviceDataStart = hostDataOffset - currentSizeOut;
PatchUtils::WriteLong(deltaFromDeviceDataStart, output);
if (deltaFromDeviceDataStart > 0) {
PatchUtils::Pipe(input, output, deltaFromDeviceDataStart);
}
PatchUtils::WriteLong(deviceDataOffset, output);
PatchUtils::WriteLong(deviceDataLength, output);
adb_lseek(input, deviceDataLength, SEEK_CUR);
currentSizeOut += deltaFromDeviceDataStart + deviceDataLength;
}
if (currentSizeOut != newApkSize) {
PatchUtils::WriteLong(newApkSize - currentSizeOut, output);
PatchUtils::Pipe(input, output, newApkSize - currentSizeOut);
PatchUtils::WriteLong(0, output);
PatchUtils::WriteLong(0, output);
}
}
bool DeployPatchGenerator::CreatePatch(const char* localApkPath, const char* deviceApkMetadataPath,
borrowed_fd output) {
std::string content;
APKMetaData deviceApkMetadata;
if (android::base::ReadFileToString(deviceApkMetadataPath, &content)) {
deviceApkMetadata.ParsePartialFromString(content);
} else {
// TODO: What do we want to do if we don't find any metadata.
// The current fallback behavior is to build a patch with the contents of |localApkPath|.
}
APKMetaData localApkMetadata = PatchUtils::GetAPKMetaData(localApkPath);
// Log gathered metadata info.
APKMetaDataToLog(deviceApkMetadataPath, deviceApkMetadata);
APKMetaDataToLog(localApkPath, localApkMetadata);
std::vector<SimpleEntry> identicalEntries;
uint64_t totalSize =
BuildIdenticalEntries(identicalEntries, localApkMetadata, deviceApkMetadata);
ReportSavings(identicalEntries, totalSize);
GeneratePatch(identicalEntries, localApkPath, output);
return true;
}
uint64_t DeployPatchGenerator::BuildIdenticalEntries(std::vector<SimpleEntry>& outIdenticalEntries,
const APKMetaData& localApkMetadata,
const APKMetaData& deviceApkMetadata) {
uint64_t totalSize = 0;
for (int i = 0; i < localApkMetadata.entries_size(); i++) {
const APKEntry& localEntry = localApkMetadata.entries(i);
totalSize += localEntry.compressedsize();
for (int j = 0; j < deviceApkMetadata.entries_size(); j++) {
const APKEntry& deviceEntry = deviceApkMetadata.entries(j);
if (deviceEntry.crc32() == localEntry.crc32() &&
deviceEntry.filename().compare(localEntry.filename()) == 0) {
SimpleEntry simpleEntry;
simpleEntry.localEntry = const_cast<APKEntry*>(&localEntry);
simpleEntry.deviceEntry = const_cast<APKEntry*>(&deviceEntry);
APKEntryToLog(localEntry);
outIdenticalEntries.push_back(simpleEntry);
break;
}
}
}
std::sort(outIdenticalEntries.begin(), outIdenticalEntries.end(),
[](const SimpleEntry& lhs, const SimpleEntry& rhs) {
return lhs.localEntry->dataoffset() < rhs.localEntry->dataoffset();
});
return totalSize;
}

102
adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.h Normal file
View file

@ -0,0 +1,102 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <vector>
#include "adb_unique_fd.h"
#include "fastdeploy/proto/ApkEntry.pb.h"
/**
* This class is responsible for creating a patch that can be accepted by the deployagent. The
* patch format is documented in GeneratePatch.
*/
class DeployPatchGenerator {
public:
/**
* Simple struct to hold mapping between local metadata and device metadata.
*/
struct SimpleEntry {
com::android::fastdeploy::APKEntry* localEntry;
com::android::fastdeploy::APKEntry* deviceEntry;
};
/**
* If |is_verbose| is true ApkEntries that are similar between device and host are written to
* the console.
*/
explicit DeployPatchGenerator(bool is_verbose) : is_verbose_(is_verbose) {}
/**
* Given a |localApkPath|, and the |deviceApkMetadataPath| from an installed APK this function
* writes a patch to the given |output|.
*/
bool CreatePatch(const char* localApkPath, const char* deviceApkMetadataPath,
android::base::borrowed_fd output);
private:
bool is_verbose_;
/**
* Log function only logs data to stdout when |is_verbose_| is true.
*/
void Log(const char* fmt, ...) __attribute__((__format__(__printf__, 2, 3)));
/**
* Helper function to log the APKMetaData structure. If |is_verbose_| is false this function
* early outs. |file| is the path to the file represented by |metadata|. This function is used
* for debugging / information.
*/
void APKMetaDataToLog(const char* file, const com::android::fastdeploy::APKMetaData& metadata);
/**
* Helper function to log APKEntry.
*/
void APKEntryToLog(const com::android::fastdeploy::APKEntry& entry);
/**
* Helper function to report savings by fastdeploy. This function prints out savings even with
* |is_verbose_| set to false. |totalSize| is used to show a percentage of savings. Note:
* |totalSize| is the size of the ZipEntries. Not the size of the entire file. The metadata of
* the zip data needs to be sent across with every iteration.
* [Patch format]
* |Fixed String| Signature
* |long| New Size of Apk
* |Packets[]| Array of Packets
*
* [Packet Format]
* |long| Size of data to use from patch
* |byte[]| Patch data
* |long| Offset of data to use already on device
* |long| Length of data to read from device APK
* TODO(b/138306784): Move the patch format to a proto.
*/
void ReportSavings(const std::vector<SimpleEntry>& identicalEntries, uint64_t totalSize);
/**
* This enumerates each entry in |entriesToUseOnDevice| and builds a patch file copying data
* from |localApkPath| where we are unable to use entries already on the device. The new patch
* is written to |output|. The entries are expected to be sorted by data offset from lowest to
* highest.
*/
void GeneratePatch(const std::vector<SimpleEntry>& entriesToUseOnDevice,
const char* localApkPath, android::base::borrowed_fd output);
protected:
uint64_t BuildIdenticalEntries(
std::vector<SimpleEntry>& outIdenticalEntries,
const com::android::fastdeploy::APKMetaData& localApkMetadata,
const com::android::fastdeploy::APKMetaData& deviceApkMetadataPath);
};

72
adb/fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp Normal file
View file

@ -0,0 +1,72 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "deploy_patch_generator.h"
#include "patch_utils.h"
#include <android-base/file.h>
#include <gtest/gtest.h>
#include <stdlib.h>
#include <string.h>
#include <string>
#include "sysdeps.h"
using namespace com::android::fastdeploy;
static std::string GetTestFile(const std::string& name) {
return "fastdeploy/testdata/" + name;
}
class TestPatchGenerator : DeployPatchGenerator {
public:
TestPatchGenerator() : DeployPatchGenerator(false) {}
void GatherIdenticalEntries(std::vector<DeployPatchGenerator::SimpleEntry>& outIdenticalEntries,
const APKMetaData& metadataA, const APKMetaData& metadataB) {
BuildIdenticalEntries(outIdenticalEntries, metadataA, metadataB);
}
};
TEST(DeployPatchGeneratorTest, IdenticalFileEntries) {
std::string apkPath = GetTestFile("rotating_cube-release.apk");
APKMetaData metadataA = PatchUtils::GetAPKMetaData(apkPath.c_str());
TestPatchGenerator generator;
std::vector<DeployPatchGenerator::SimpleEntry> entries;
generator.GatherIdenticalEntries(entries, metadataA, metadataA);
// Expect the entry count to match the number of entries in the metadata.
const uint32_t identicalCount = entries.size();
const uint32_t entriesCount = metadataA.entries_size();
EXPECT_EQ(identicalCount, entriesCount);
}
TEST(DeployPatchGeneratorTest, NoDeviceMetadata) {
std::string apkPath = GetTestFile("rotating_cube-release.apk");
// Get size of our test apk.
long apkSize = 0;
{
unique_fd apkFile(adb_open(apkPath.c_str(), O_RDWR));
apkSize = adb_lseek(apkFile, 0L, SEEK_END);
}
// Create a patch that is 100% different.
TemporaryFile output;
DeployPatchGenerator generator(true);
generator.CreatePatch(apkPath.c_str(), "", output.fd);
// Expect a patch file that has a size at least the size of our initial APK.
long patchSize = adb_lseek(output.fd, 0L, SEEK_END);
EXPECT_GT(patchSize, apkSize);
}

1
adb/fastdeploy/deploypatchgenerator/manifest.txt
View file

@ -1 +0,0 @@
Main-Class: com.android.fastdeploy.DeployPatchGenerator

87
adb/fastdeploy/deploypatchgenerator/patch_utils.cpp Normal file
View file

@ -0,0 +1,87 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "patch_utils.h"
#include <androidfw/ZipFileRO.h>
#include <stdio.h>
#include "adb_io.h"
#include "android-base/endian.h"
#include "sysdeps.h"
using namespace com::android;
using namespace com::android::fastdeploy;
using namespace android::base;
static constexpr char kSignature[] = "FASTDEPLOY";
APKMetaData PatchUtils::GetAPKMetaData(const char* apkPath) {
APKMetaData apkMetaData;
#undef open
std::unique_ptr<android::ZipFileRO> zipFile(android::ZipFileRO::open(apkPath));
#define open ___xxx_unix_open
if (zipFile == nullptr) {
printf("Could not open %s", apkPath);
exit(1);
}
void* cookie;
if (zipFile->startIteration(&cookie)) {
android::ZipEntryRO entry;
while ((entry = zipFile->nextEntry(cookie)) != NULL) {
char fileName[256];
// Make sure we have a file name.
// TODO: Handle filenames longer than 256.
if (zipFile->getEntryFileName(entry, fileName, sizeof(fileName))) {
continue;
}
uint32_t uncompressedSize, compressedSize, crc32;
int64_t dataOffset;
zipFile->getEntryInfo(entry, nullptr, &uncompressedSize, &compressedSize, &dataOffset,
nullptr, &crc32);
APKEntry* apkEntry = apkMetaData.add_entries();
apkEntry->set_crc32(crc32);
apkEntry->set_filename(fileName);
apkEntry->set_compressedsize(compressedSize);
apkEntry->set_uncompressedsize(uncompressedSize);
apkEntry->set_dataoffset(dataOffset);
}
}
return apkMetaData;
}
void PatchUtils::WriteSignature(borrowed_fd output) {
WriteFdExactly(output, kSignature, sizeof(kSignature) - 1);
}
void PatchUtils::WriteLong(int64_t value, borrowed_fd output) {
int64_t toLittleEndian = htole64(value);
WriteFdExactly(output, &toLittleEndian, sizeof(int64_t));
}
void PatchUtils::Pipe(borrowed_fd input, borrowed_fd output, size_t amount) {
constexpr static int BUFFER_SIZE = 128 * 1024;
char buffer[BUFFER_SIZE];
size_t transferAmount = 0;
while (transferAmount != amount) {
long chunkAmount =
amount - transferAmount > BUFFER_SIZE ? BUFFER_SIZE : amount - transferAmount;
long readAmount = adb_read(input, buffer, chunkAmount);
WriteFdExactly(output, buffer, readAmount);
transferAmount += readAmount;
}
}

46
adb/fastdeploy/deploypatchgenerator/patch_utils.h Normal file
View file

@ -0,0 +1,46 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "adb_unique_fd.h"
#include "fastdeploy/proto/ApkEntry.pb.h"
/**
* Helper class that mirrors the PatchUtils from deploy agent.
*/
class PatchUtils {
public:
/**
* This function takes a local APK file and builds the APKMetaData required by the patching
* algorithm. The if this function has an error a string is printed to the terminal and exit(1)
* is called.
*/
static com::android::fastdeploy::APKMetaData GetAPKMetaData(const char* file);
/**
* Writes a fixed signature string to the header of the patch.
*/
static void WriteSignature(android::base::borrowed_fd output);
/**
* Writes an int64 to the |output| reversing the bytes.
*/
static void WriteLong(int64_t value, android::base::borrowed_fd output);
/**
* Copy |amount| of data from |input| to |output|.
*/
static void Pipe(android::base::borrowed_fd input, android::base::borrowed_fd output,
size_t amount);
};

96
adb/fastdeploy/deploypatchgenerator/patch_utils_test.cpp Normal file
View file

@ -0,0 +1,96 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "patch_utils.h"
#include <android-base/file.h>
#include <gtest/gtest.h>
#include <stdio.h>
#include <stdlib.h>
#include <sstream>
#include <string>
#include "adb_io.h"
#include "sysdeps.h"
using namespace com::android::fastdeploy;
static std::string GetTestFile(const std::string& name) {
return "fastdeploy/testdata/" + name;
}
bool FileMatchesContent(android::base::borrowed_fd input, const char* contents,
ssize_t contentsSize) {
adb_lseek(input, 0, SEEK_SET);
// Use a temp buffer larger than any test contents.
constexpr int BUFFER_SIZE = 2048;
char buffer[BUFFER_SIZE];
bool result = true;
// Validate size of files is equal.
ssize_t readAmount = adb_read(input, buffer, BUFFER_SIZE);
EXPECT_EQ(readAmount, contentsSize);
result = memcmp(buffer, contents, readAmount) == 0;
for (int i = 0; i < readAmount; i++) {
printf("%x", buffer[i]);
}
printf(" == ");
for (int i = 0; i < contentsSize; i++) {
printf("%x", contents[i]);
}
printf("\n");
return result;
}
TEST(PatchUtilsTest, SwapLongWrites) {
TemporaryFile output;
PatchUtils::WriteLong(0x0011223344556677, output.fd);
adb_lseek(output.fd, 0, SEEK_SET);
const char expected[] = {0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x00};
EXPECT_TRUE(FileMatchesContent(output.fd, expected, 8));
}
TEST(PatchUtilsTest, PipeWritesAmountToOutput) {
std::string expected("Some Data");
TemporaryFile input;
TemporaryFile output;
// Populate input file.
WriteFdExactly(input.fd, expected);
adb_lseek(input.fd, 0, SEEK_SET);
// Open input file for read, and output file for write.
PatchUtils::Pipe(input.fd, output.fd, expected.size());
// Validate pipe worked
EXPECT_TRUE(FileMatchesContent(output.fd, expected.c_str(), expected.size()));
}
TEST(PatchUtilsTest, SignatureConstMatches) {
std::string apkFile = GetTestFile("rotating_cube-release.apk");
TemporaryFile output;
PatchUtils::WriteSignature(output.fd);
std::string contents("FASTDEPLOY");
EXPECT_TRUE(FileMatchesContent(output.fd, contents.c_str(), contents.size()));
}
TEST(PatchUtilsTest, GatherMetadata) {
std::string apkFile = GetTestFile("rotating_cube-release.apk");
APKMetaData metadata = PatchUtils::GetAPKMetaData(apkFile.c_str());
std::string expectedMetadata;
android::base::ReadFileToString(GetTestFile("rotating_cube-metadata-release.data"),
&expectedMetadata);
std::string actualMetadata;
metadata.SerializeToString(&actualMetadata);
EXPECT_EQ(expectedMetadata, actualMetadata);
}

208
adb/fastdeploy/deploypatchgenerator/src/com/android/fastdeploy/DeployPatchGenerator.java
View file

@ -1,208 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.fastdeploy;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.StringBuilder;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.ArrayList;
import java.nio.charset.StandardCharsets;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.AbstractMap.SimpleEntry;
import com.android.fastdeploy.APKMetaData;
import com.android.fastdeploy.APKEntry;
public final class DeployPatchGenerator {
private static final int BUFFER_SIZE = 128 * 1024;
public static void main(String[] args) {
try {
if (args.length < 2) {
showUsage(0);
}
boolean verbose = false;
if (args.length > 2) {
String verboseFlag = args[2];
if (verboseFlag.compareTo("--verbose") == 0) {
verbose = true;
}
}
StringBuilder sb = null;
String apkPath = args[0];
String deviceMetadataPath = args[1];
File hostFile = new File(apkPath);
List<APKEntry> deviceZipEntries = getMetadataFromFile(deviceMetadataPath);
System.err.println("Device Entries (" + deviceZipEntries.size() + ")");
if (verbose) {
sb = new StringBuilder();
for (APKEntry entry : deviceZipEntries) {
APKEntryToString(entry, sb);
}
System.err.println(sb.toString());
}
List<APKEntry> hostFileEntries = PatchUtils.getAPKMetaData(hostFile).getEntriesList();
System.err.println("Host Entries (" + hostFileEntries.size() + ")");
if (verbose) {
sb = new StringBuilder();
for (APKEntry entry : hostFileEntries) {
APKEntryToString(entry, sb);
}
System.err.println(sb.toString());
}
List<SimpleEntry<APKEntry, APKEntry>> identicalContentsEntrySet =
getIdenticalContents(deviceZipEntries, hostFileEntries);
reportIdenticalContents(identicalContentsEntrySet, hostFile);
if (verbose) {
sb = new StringBuilder();
for (SimpleEntry<APKEntry, APKEntry> identicalEntry : identicalContentsEntrySet) {
APKEntry entry = identicalEntry.getValue();
APKEntryToString(entry, sb);
}
System.err.println("Identical Entries (" + identicalContentsEntrySet.size() + ")");
System.err.println(sb.toString());
}
createPatch(identicalContentsEntrySet, hostFile, System.out);
} catch (Exception e) {
System.err.println("Error: " + e);
e.printStackTrace();
System.exit(2);
}
System.exit(0);
}
private static void showUsage(int exitCode) {
System.err.println("usage: deploypatchgenerator <apkpath> <deviceapkmetadata> [--verbose]");
System.err.println("");
System.exit(exitCode);
}
private static void APKEntryToString(APKEntry entry, StringBuilder outputString) {
outputString.append(String.format("Filename: %s\n", entry.getFileName()));
outputString.append(String.format("CRC32: 0x%08X\n", entry.getCrc32()));
outputString.append(String.format("Data Offset: %d\n", entry.getDataOffset()));
outputString.append(String.format("Compressed Size: %d\n", entry.getCompressedSize()));
outputString.append(String.format("Uncompressed Size: %d\n", entry.getUncompressedSize()));
}
private static List<APKEntry> getMetadataFromFile(String deviceMetadataPath) throws IOException {
InputStream is = new FileInputStream(new File(deviceMetadataPath));
APKMetaData apkMetaData = APKMetaData.parseDelimitedFrom(is);
return apkMetaData.getEntriesList();
}
private static List<SimpleEntry<APKEntry, APKEntry>> getIdenticalContents(
List<APKEntry> deviceZipEntries, List<APKEntry> hostZipEntries) throws IOException {
List<SimpleEntry<APKEntry, APKEntry>> identicalContents =
new ArrayList<SimpleEntry<APKEntry, APKEntry>>();
for (APKEntry deviceZipEntry : deviceZipEntries) {
for (APKEntry hostZipEntry : hostZipEntries) {
if (deviceZipEntry.getCrc32() == hostZipEntry.getCrc32() &&
deviceZipEntry.getFileName().equals(hostZipEntry.getFileName())) {
identicalContents.add(new SimpleEntry(deviceZipEntry, hostZipEntry));
}
}
}
Collections.sort(identicalContents, new Comparator<SimpleEntry<APKEntry, APKEntry>>() {
@Override
public int compare(
SimpleEntry<APKEntry, APKEntry> p1, SimpleEntry<APKEntry, APKEntry> p2) {
return Long.compare(p1.getValue().getDataOffset(), p2.getValue().getDataOffset());
}
});
return identicalContents;
}
private static void reportIdenticalContents(
List<SimpleEntry<APKEntry, APKEntry>> identicalContentsEntrySet, File hostFile)
throws IOException {
long totalEqualBytes = 0;
int totalEqualFiles = 0;
for (SimpleEntry<APKEntry, APKEntry> entries : identicalContentsEntrySet) {
APKEntry hostAPKEntry = entries.getValue();
totalEqualBytes += hostAPKEntry.getCompressedSize();
totalEqualFiles++;
}
float savingPercent = (float) (totalEqualBytes * 100) / hostFile.length();
System.err.println("Detected " + totalEqualFiles + " equal APK entries");
System.err.println(totalEqualBytes + " bytes are equal out of " + hostFile.length() + " ("
+ savingPercent + "%)");
}
static void createPatch(List<SimpleEntry<APKEntry, APKEntry>> zipEntrySimpleEntrys,
File hostFile, OutputStream patchStream) throws IOException, PatchFormatException {
FileInputStream hostFileInputStream = new FileInputStream(hostFile);
patchStream.write(PatchUtils.SIGNATURE.getBytes(StandardCharsets.US_ASCII));
PatchUtils.writeFormattedLong(hostFile.length(), patchStream);
byte[] buffer = new byte[BUFFER_SIZE];
long totalBytesWritten = 0;
Iterator<SimpleEntry<APKEntry, APKEntry>> entrySimpleEntryIterator =
zipEntrySimpleEntrys.iterator();
while (entrySimpleEntryIterator.hasNext()) {
SimpleEntry<APKEntry, APKEntry> entrySimpleEntry = entrySimpleEntryIterator.next();
APKEntry deviceAPKEntry = entrySimpleEntry.getKey();
APKEntry hostAPKEntry = entrySimpleEntry.getValue();
long newDataLen = hostAPKEntry.getDataOffset() - totalBytesWritten;
long oldDataOffset = deviceAPKEntry.getDataOffset();
long oldDataLen = deviceAPKEntry.getCompressedSize();
PatchUtils.writeFormattedLong(newDataLen, patchStream);
PatchUtils.pipe(hostFileInputStream, patchStream, buffer, newDataLen);
PatchUtils.writeFormattedLong(oldDataOffset, patchStream);
PatchUtils.writeFormattedLong(oldDataLen, patchStream);
long skip = hostFileInputStream.skip(oldDataLen);
if (skip != oldDataLen) {
throw new PatchFormatException("skip error: attempted to skip " + oldDataLen
+ " bytes but return code was " + skip);
}
totalBytesWritten += oldDataLen + newDataLen;
}
long remainderLen = hostFile.length() - totalBytesWritten;
PatchUtils.writeFormattedLong(remainderLen, patchStream);
PatchUtils.pipe(hostFileInputStream, patchStream, buffer, remainderLen);
PatchUtils.writeFormattedLong(0, patchStream);
PatchUtils.writeFormattedLong(0, patchStream);
patchStream.flush();
}
}

6
adb/fastdeploy/testdata/rotating_cube-metadata-release.data vendored Normal file
View file

@ -0,0 +1,6 @@
#Ç‚Ï« META-INF/MANIFEST.MFÇ Q(W
#ƒ<>•ŽAndroidManifest.xml1 ä(è
6¦µ€>#lib/armeabi-v7a/libvulkan_sample.so<18> ÀÒQ(Œ²ì
—ã—‘resources.arscôàQ ´(´
‹œÂÉ classes.dexÁ ÿ(ô

BIN
adb/fastdeploy/testdata/rotating_cube-release.apk vendored Normal file
View file

Binary file not shown.