Remove the old log files if cache space is insufficient for OTA
We set the limit of the max stash size to 80% of cache size. But the cache space can still be insufficient for the update if the log files occupy a large chunk of /cache. So remove the old logs for now to make room for the update. Bug: 77528881 Test: unit tests pass Change-Id: Ia8bcb0ace11f8164ad9290bfb360e08e31d282cb
This commit is contained in:
Tianjie Xu
parent
ea8a6a9af2
commit
d5fbcc1ba9
6 changed files with 277 additions and 66 deletions
|
|
@ -436,13 +436,13 @@ static size_t FileSink(const unsigned char* data, size_t len, int fd) {
|
|||
|
||||
// Return the amount of free space (in bytes) on the filesystem
|
||||
// containing filename. filename must exist. Return -1 on error.
|
||||
size_t FreeSpaceForFile(const char* filename) {
|
||||
struct statfs sf;
|
||||
if (statfs(filename, &sf) != 0) {
|
||||
printf("failed to statfs %s: %s\n", filename, strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
return sf.f_bsize * sf.f_bavail;
|
||||
size_t FreeSpaceForFile(const std::string& filename) {
|
||||
struct statfs sf;
|
||||
if (statfs(filename.c_str(), &sf) != 0) {
|
||||
printf("failed to statfs %s: %s\n", filename.c_str(), strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
return sf.f_bsize * sf.f_bavail;
|
||||
}
|
||||
|
||||
int CacheSizeCheck(size_t bytes) {
|
||||
|
|
|
|||
|
|
@ -14,7 +14,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <ctype.h>
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <error.h>
|
||||
#include <libgen.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
|
@ -22,20 +25,22 @@
|
|||
#include <sys/stat.h>
|
||||
#include <sys/statfs.h>
|
||||
#include <unistd.h>
|
||||
#include <dirent.h>
|
||||
#include <ctype.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
#include <android-base/file.h>
|
||||
#include <android-base/parseint.h>
|
||||
#include <android-base/stringprintf.h>
|
||||
#include <android-base/strings.h>
|
||||
|
||||
#include "applypatch/applypatch.h"
|
||||
#include "otautil/cache_location.h"
|
||||
|
||||
static int EliminateOpenFiles(std::set<std::string>* files) {
|
||||
static int EliminateOpenFiles(const std::string& dirname, std::set<std::string>* files) {
|
||||
std::unique_ptr<DIR, decltype(&closedir)> d(opendir("/proc"), closedir);
|
||||
if (!d) {
|
||||
printf("error opening /proc: %s\n", strerror(errno));
|
||||
|
|
@ -62,7 +67,7 @@ static int EliminateOpenFiles(std::set<std::string>* files) {
|
|||
int count = readlink(fd_path.c_str(), link, sizeof(link)-1);
|
||||
if (count >= 0) {
|
||||
link[count] = '\0';
|
||||
if (strncmp(link, "/cache/", 7) == 0) {
|
||||
if (android::base::StartsWith(link, dirname)) {
|
||||
if (files->erase(link) > 0) {
|
||||
printf("%s is open by %s\n", link, de->d_name);
|
||||
}
|
||||
|
|
@ -73,77 +78,138 @@ static int EliminateOpenFiles(std::set<std::string>* files) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static std::set<std::string> FindExpendableFiles() {
|
||||
static std::vector<std::string> FindExpendableFiles(
|
||||
const std::string& dirname, const std::function<bool(const std::string&)>& name_filter) {
|
||||
std::set<std::string> files;
|
||||
// We're allowed to delete unopened regular files in any of these
|
||||
// directories.
|
||||
const char* dirs[2] = {"/cache", "/cache/recovery/otatest"};
|
||||
|
||||
for (size_t i = 0; i < sizeof(dirs)/sizeof(dirs[0]); ++i) {
|
||||
std::unique_ptr<DIR, decltype(&closedir)> d(opendir(dirs[i]), closedir);
|
||||
if (!d) {
|
||||
printf("error opening %s: %s\n", dirs[i], strerror(errno));
|
||||
std::unique_ptr<DIR, decltype(&closedir)> d(opendir(dirname.c_str()), closedir);
|
||||
if (!d) {
|
||||
printf("error opening %s: %s\n", dirname.c_str(), strerror(errno));
|
||||
return {};
|
||||
}
|
||||
|
||||
// Look for regular files in the directory (not in any subdirectories).
|
||||
struct dirent* de;
|
||||
while ((de = readdir(d.get())) != 0) {
|
||||
std::string path = dirname + "/" + de->d_name;
|
||||
|
||||
// We can't delete cache_temp_source; if it's there we might have restarted during
|
||||
// installation and could be depending on it to be there.
|
||||
if (path == CacheLocation::location().cache_temp_source()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Look for regular files in the directory (not in any subdirectories).
|
||||
struct dirent* de;
|
||||
while ((de = readdir(d.get())) != 0) {
|
||||
std::string path = std::string(dirs[i]) + "/" + de->d_name;
|
||||
// Do not delete the file if it doesn't have the expected format.
|
||||
if (name_filter != nullptr && !name_filter(de->d_name)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// We can't delete cache_temp_source; if it's there we might have restarted during
|
||||
// installation and could be depending on it to be there.
|
||||
if (path == CacheLocation::location().cache_temp_source()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
struct stat st;
|
||||
if (stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) {
|
||||
files.insert(path);
|
||||
}
|
||||
struct stat st;
|
||||
if (stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) {
|
||||
files.insert(path);
|
||||
}
|
||||
}
|
||||
|
||||
printf("%zu regular files in deletable directories\n", files.size());
|
||||
if (EliminateOpenFiles(&files) < 0) {
|
||||
return std::set<std::string>();
|
||||
printf("%zu regular files in deletable directory\n", files.size());
|
||||
if (EliminateOpenFiles(dirname, &files) < 0) {
|
||||
return {};
|
||||
}
|
||||
return files;
|
||||
|
||||
return std::vector<std::string>(files.begin(), files.end());
|
||||
}
|
||||
|
||||
// Parses the index of given log file, e.g. 3 for last_log.3; returns max number if the log name
|
||||
// doesn't have the expected format so that we'll delete these ones first.
|
||||
static unsigned int GetLogIndex(const std::string& log_name) {
|
||||
if (log_name == "last_log" || log_name == "last_kmsg") {
|
||||
return 0;
|
||||
}
|
||||
|
||||
unsigned int index;
|
||||
if (sscanf(log_name.c_str(), "last_log.%u", &index) == 1 ||
|
||||
sscanf(log_name.c_str(), "last_kmsg.%u", &index) == 1) {
|
||||
return index;
|
||||
}
|
||||
|
||||
return std::numeric_limits<unsigned int>::max();
|
||||
}
|
||||
|
||||
int MakeFreeSpaceOnCache(size_t bytes_needed) {
|
||||
#ifndef __ANDROID__
|
||||
// TODO (xunchang) implement a heuristic cache size check during host simulation.
|
||||
printf("Skip making (%zu) bytes free space on cache; program is running on host\n", bytes_needed);
|
||||
printf("Skip making (%zu) bytes free space on /cache; program is running on host\n",
|
||||
bytes_needed);
|
||||
return 0;
|
||||
#endif
|
||||
|
||||
size_t free_now = FreeSpaceForFile("/cache");
|
||||
printf("%zu bytes free on /cache (%zu needed)\n", free_now, bytes_needed);
|
||||
|
||||
if (free_now >= bytes_needed) {
|
||||
return 0;
|
||||
}
|
||||
std::set<std::string> files = FindExpendableFiles();
|
||||
if (files.empty()) {
|
||||
// nothing we can delete to free up space!
|
||||
printf("no files can be deleted to free space on /cache\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// We could try to be smarter about which files to delete: the
|
||||
// biggest ones? the smallest ones that will free up enough space?
|
||||
// the oldest? the newest?
|
||||
//
|
||||
// Instead, we'll be dumb.
|
||||
|
||||
for (const auto& file : files) {
|
||||
unlink(file.c_str());
|
||||
free_now = FreeSpaceForFile("/cache");
|
||||
printf("deleted %s; now %zu bytes free\n", file.c_str(), free_now);
|
||||
if (free_now < bytes_needed) {
|
||||
break;
|
||||
std::vector<std::string> dirs = { "/cache", CacheLocation::location().cache_log_directory() };
|
||||
for (const auto& dirname : dirs) {
|
||||
if (RemoveFilesInDirectory(bytes_needed, dirname, FreeSpaceForFile)) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return (free_now >= bytes_needed) ? 0 : -1;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
bool RemoveFilesInDirectory(size_t bytes_needed, const std::string& dirname,
|
||||
const std::function<size_t(const std::string&)>& space_checker) {
|
||||
struct stat st;
|
||||
if (stat(dirname.c_str(), &st) != 0) {
|
||||
error(0, errno, "Unable to free space on %s", dirname.c_str());
|
||||
return false;
|
||||
}
|
||||
if (!S_ISDIR(st.st_mode)) {
|
||||
printf("%s is not a directory\n", dirname.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t free_now = space_checker(dirname);
|
||||
printf("%zu bytes free on %s (%zu needed)\n", free_now, dirname.c_str(), bytes_needed);
|
||||
|
||||
if (free_now >= bytes_needed) {
|
||||
return true;
|
||||
}
|
||||
|
||||
std::vector<std::string> files;
|
||||
if (dirname == CacheLocation::location().cache_log_directory()) {
|
||||
// Deletes the log files only.
|
||||
auto log_filter = [](const std::string& file_name) {
|
||||
return android::base::StartsWith(file_name, "last_log") ||
|
||||
android::base::StartsWith(file_name, "last_kmsg");
|
||||
};
|
||||
|
||||
files = FindExpendableFiles(dirname, log_filter);
|
||||
|
||||
// Older logs will come to the top of the queue.
|
||||
auto comparator = [](const std::string& name1, const std::string& name2) -> bool {
|
||||
unsigned int index1 = GetLogIndex(android::base::Basename(name1));
|
||||
unsigned int index2 = GetLogIndex(android::base::Basename(name2));
|
||||
if (index1 == index2) {
|
||||
return name1 < name2;
|
||||
}
|
||||
|
||||
return index1 > index2;
|
||||
};
|
||||
|
||||
std::sort(files.begin(), files.end(), comparator);
|
||||
} else {
|
||||
// We're allowed to delete unopened regular files in the directory.
|
||||
files = FindExpendableFiles(dirname, nullptr);
|
||||
}
|
||||
|
||||
for (const auto& file : files) {
|
||||
if (unlink(file.c_str()) == -1) {
|
||||
error(0, errno, "Failed to delete %s", file.c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
free_now = space_checker(dirname);
|
||||
printf("deleted %s; now %zu bytes free\n", file.c_str(), free_now);
|
||||
if (free_now >= bytes_needed) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ using SinkFn = std::function<size_t(const unsigned char*, size_t)>;
|
|||
// applypatch.cpp
|
||||
|
||||
int ShowLicenses();
|
||||
size_t FreeSpaceForFile(const char* filename);
|
||||
size_t FreeSpaceForFile(const std::string& filename);
|
||||
int CacheSizeCheck(size_t bytes);
|
||||
int ParseSha1(const char* str, uint8_t* digest);
|
||||
|
||||
|
|
@ -79,5 +79,9 @@ int ApplyImagePatch(const unsigned char* old_data, size_t old_size, const Value&
|
|||
// freecache.cpp
|
||||
|
||||
int MakeFreeSpaceOnCache(size_t bytes_needed);
|
||||
// Removes the files in |dirname| until we have at least |bytes_needed| bytes of free space on
|
||||
// the partition. The size of the free space is returned by calling |space_checker|.
|
||||
bool RemoveFilesInDirectory(size_t bytes_needed, const std::string& dirname,
|
||||
const std::function<size_t(const std::string&)>& space_checker);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
constexpr const char kDefaultCacheTempSource[] = "/cache/saved.file";
|
||||
constexpr const char kDefaultLastCommandFile[] = "/cache/recovery/last_command";
|
||||
constexpr const char kDefaultStashDirectoryBase[] = "/cache/recovery";
|
||||
constexpr const char kDefaultCacheLogDirectory[] = "/cache/recovery";
|
||||
|
||||
CacheLocation& CacheLocation::location() {
|
||||
static CacheLocation cache_location;
|
||||
|
|
@ -28,4 +29,5 @@ CacheLocation& CacheLocation::location() {
|
|||
CacheLocation::CacheLocation()
|
||||
: cache_temp_source_(kDefaultCacheTempSource),
|
||||
last_command_file_(kDefaultLastCommandFile),
|
||||
stash_directory_base_(kDefaultStashDirectoryBase) {}
|
||||
stash_directory_base_(kDefaultStashDirectoryBase),
|
||||
cache_log_directory_(kDefaultCacheLogDirectory) {}
|
||||
|
|
|
|||
|
|
@ -49,6 +49,13 @@ class CacheLocation {
|
|||
stash_directory_base_ = base;
|
||||
}
|
||||
|
||||
std::string cache_log_directory() const {
|
||||
return cache_log_directory_;
|
||||
}
|
||||
void set_cache_log_directory(const std::string& log_dir) {
|
||||
cache_log_directory_ = log_dir;
|
||||
}
|
||||
|
||||
private:
|
||||
CacheLocation();
|
||||
DISALLOW_COPY_AND_ASSIGN(CacheLocation);
|
||||
|
|
@ -64,6 +71,9 @@ class CacheLocation {
|
|||
|
||||
// The base directory to write stashes during update.
|
||||
std::string stash_directory_base_;
|
||||
|
||||
// The location of last_log & last_kmsg.
|
||||
std::string cache_log_directory_;
|
||||
};
|
||||
|
||||
#endif // _OTAUTIL_OTAUTIL_CACHE_LOCATION_H_
|
||||
|
|
|
|||
|
|
@ -14,8 +14,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <dirent.h>
|
||||
#include <fcntl.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include <libgen.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
|
|
@ -23,6 +25,7 @@
|
|||
#include <sys/types.h>
|
||||
#include <time.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
|
@ -30,6 +33,7 @@
|
|||
#include <android-base/file.h>
|
||||
#include <android-base/stringprintf.h>
|
||||
#include <android-base/test_utils.h>
|
||||
#include <android-base/unique_fd.h>
|
||||
#include <bsdiff/bsdiff.h>
|
||||
#include <openssl/sha.h>
|
||||
|
||||
|
|
@ -110,6 +114,52 @@ class ApplyPatchModesTest : public ::testing::Test {
|
|||
TemporaryFile cache_source;
|
||||
};
|
||||
|
||||
class FreeCacheTest : public ::testing::Test {
|
||||
protected:
|
||||
static constexpr size_t PARTITION_SIZE = 4096 * 10;
|
||||
|
||||
// Returns a sorted list of files in |dirname|.
|
||||
static std::vector<std::string> FindFilesInDir(const std::string& dirname) {
|
||||
std::vector<std::string> file_list;
|
||||
|
||||
std::unique_ptr<DIR, decltype(&closedir)> d(opendir(dirname.c_str()), closedir);
|
||||
struct dirent* de;
|
||||
while ((de = readdir(d.get())) != 0) {
|
||||
std::string path = dirname + "/" + de->d_name;
|
||||
|
||||
struct stat st;
|
||||
if (stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) {
|
||||
file_list.emplace_back(de->d_name);
|
||||
}
|
||||
}
|
||||
|
||||
std::sort(file_list.begin(), file_list.end());
|
||||
return file_list;
|
||||
}
|
||||
|
||||
static void AddFilesToDir(const std::string& dir, const std::vector<std::string>& files) {
|
||||
std::string zeros(4096, 0);
|
||||
for (const auto& file : files) {
|
||||
std::string path = dir + "/" + file;
|
||||
ASSERT_TRUE(android::base::WriteStringToFile(zeros, path));
|
||||
}
|
||||
}
|
||||
|
||||
void SetUp() override {
|
||||
CacheLocation::location().set_cache_log_directory(mock_log_dir.path);
|
||||
}
|
||||
|
||||
// A mock method to calculate the free space. It assumes the partition has a total size of 40960
|
||||
// bytes and all files are 4096 bytes in size.
|
||||
size_t MockFreeSpaceChecker(const std::string& dirname) {
|
||||
std::vector<std::string> files = FindFilesInDir(dirname);
|
||||
return PARTITION_SIZE - 4096 * files.size();
|
||||
}
|
||||
|
||||
TemporaryDir mock_cache;
|
||||
TemporaryDir mock_log_dir;
|
||||
};
|
||||
|
||||
TEST_F(ApplyPatchTest, CheckModeSkip) {
|
||||
std::vector<std::string> sha1s;
|
||||
ASSERT_EQ(0, applypatch_check(&old_file[0], sha1s));
|
||||
|
|
@ -414,3 +464,82 @@ TEST_F(ApplyPatchModesTest, CheckModeInvalidArgs) {
|
|||
TEST_F(ApplyPatchModesTest, ShowLicenses) {
|
||||
ASSERT_EQ(0, applypatch_modes(2, (const char* []){ "applypatch", "-l" }));
|
||||
}
|
||||
|
||||
TEST_F(FreeCacheTest, FreeCacheSmoke) {
|
||||
std::vector<std::string> files = { "file1", "file2", "file3" };
|
||||
AddFilesToDir(mock_cache.path, files);
|
||||
ASSERT_EQ(files, FindFilesInDir(mock_cache.path));
|
||||
ASSERT_EQ(4096 * 7, MockFreeSpaceChecker(mock_cache.path));
|
||||
|
||||
ASSERT_TRUE(RemoveFilesInDirectory(4096 * 9, mock_cache.path, [&](const std::string& dir) {
|
||||
return this->MockFreeSpaceChecker(dir);
|
||||
}));
|
||||
|
||||
ASSERT_EQ(std::vector<std::string>{ "file3" }, FindFilesInDir(mock_cache.path));
|
||||
ASSERT_EQ(4096 * 9, MockFreeSpaceChecker(mock_cache.path));
|
||||
}
|
||||
|
||||
TEST_F(FreeCacheTest, FreeCacheOpenFile) {
|
||||
std::vector<std::string> files = { "file1", "file2" };
|
||||
AddFilesToDir(mock_cache.path, files);
|
||||
ASSERT_EQ(files, FindFilesInDir(mock_cache.path));
|
||||
ASSERT_EQ(4096 * 8, MockFreeSpaceChecker(mock_cache.path));
|
||||
|
||||
std::string file1_path = mock_cache.path + "/file1"s;
|
||||
android::base::unique_fd fd(open(file1_path.c_str(), O_RDONLY));
|
||||
|
||||
// file1 can't be deleted as it's opened by us.
|
||||
ASSERT_FALSE(RemoveFilesInDirectory(4096 * 10, mock_cache.path, [&](const std::string& dir) {
|
||||
return this->MockFreeSpaceChecker(dir);
|
||||
}));
|
||||
|
||||
ASSERT_EQ(std::vector<std::string>{ "file1" }, FindFilesInDir(mock_cache.path));
|
||||
}
|
||||
|
||||
TEST_F(FreeCacheTest, FreeCacheLogsSmoke) {
|
||||
std::vector<std::string> log_files = { "last_log", "last_log.1", "last_kmsg.2", "last_log.5",
|
||||
"last_log.10" };
|
||||
AddFilesToDir(mock_log_dir.path, log_files);
|
||||
ASSERT_EQ(4096 * 5, MockFreeSpaceChecker(mock_log_dir.path));
|
||||
|
||||
ASSERT_TRUE(RemoveFilesInDirectory(4096 * 8, mock_log_dir.path, [&](const std::string& dir) {
|
||||
return this->MockFreeSpaceChecker(dir);
|
||||
}));
|
||||
|
||||
// Logs with a higher index will be deleted first
|
||||
std::vector<std::string> expected = { "last_log", "last_log.1" };
|
||||
ASSERT_EQ(expected, FindFilesInDir(mock_log_dir.path));
|
||||
ASSERT_EQ(4096 * 8, MockFreeSpaceChecker(mock_log_dir.path));
|
||||
}
|
||||
|
||||
TEST_F(FreeCacheTest, FreeCacheLogsStringComparison) {
|
||||
std::vector<std::string> log_files = { "last_log.1", "last_kmsg.1", "last_log.not_number",
|
||||
"last_kmsgrandom" };
|
||||
AddFilesToDir(mock_log_dir.path, log_files);
|
||||
ASSERT_EQ(4096 * 6, MockFreeSpaceChecker(mock_log_dir.path));
|
||||
|
||||
ASSERT_TRUE(RemoveFilesInDirectory(4096 * 9, mock_log_dir.path, [&](const std::string& dir) {
|
||||
return this->MockFreeSpaceChecker(dir);
|
||||
}));
|
||||
|
||||
// Logs with incorrect format will be deleted first; and the last_kmsg with the same index is
|
||||
// deleted before last_log.
|
||||
std::vector<std::string> expected = { "last_log.1" };
|
||||
ASSERT_EQ(expected, FindFilesInDir(mock_log_dir.path));
|
||||
ASSERT_EQ(4096 * 9, MockFreeSpaceChecker(mock_log_dir.path));
|
||||
}
|
||||
|
||||
TEST_F(FreeCacheTest, FreeCacheLogsOtherFiles) {
|
||||
std::vector<std::string> log_files = { "last_install", "command", "block.map", "last_log",
|
||||
"last_kmsg.1" };
|
||||
AddFilesToDir(mock_log_dir.path, log_files);
|
||||
ASSERT_EQ(4096 * 5, MockFreeSpaceChecker(mock_log_dir.path));
|
||||
|
||||
ASSERT_FALSE(RemoveFilesInDirectory(4096 * 8, mock_log_dir.path, [&](const std::string& dir) {
|
||||
return this->MockFreeSpaceChecker(dir);
|
||||
}));
|
||||
|
||||
// Non log files in /cache/recovery won't be deleted.
|
||||
std::vector<std::string> expected = { "block.map", "command", "last_install" };
|
||||
ASSERT_EQ(expected, FindFilesInDir(mock_log_dir.path));
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue