From 5ed96f4a1a7dfc7633400e97e719696997efd843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20=C5=BBenczykowski?= Date: Fri, 3 Jun 2022 08:41:04 -0700 Subject: [PATCH] bpfloader: support 'shared' maps and per-map/program selinux context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit allow bpf .o files to specify desired selinux context Bug: 218408035 Test: TreeHugger Signed-off-by: Maciej Żenczykowski Change-Id: I9d7449e477c371053a34191a2a9a935ba927d4b3 (cherry picked from commit 41817139636bdb58022d4206662b25f375587c68) Merged-In: I9d7449e477c371053a34191a2a9a935ba927d4b3 --- bpfloader/BpfLoader.cpp | 29 +++- libbpf_android/BpfLoadTest.cpp | 2 +- libbpf_android/Loader.cpp | 219 +++++++++++++++++++++--- libbpf_android/include/libbpf_android.h | 54 ++++++ 4 files changed, 277 insertions(+), 27 deletions(-) diff --git a/bpfloader/BpfLoader.cpp b/bpfloader/BpfLoader.cpp index 4e5a6ce..bc72811 100644 --- a/bpfloader/BpfLoader.cpp +++ b/bpfloader/BpfLoader.cpp @@ -51,8 +51,16 @@ #include "bpf/BpfUtils.h" using android::base::EndsWith; +using android::bpf::domain; using std::string; +constexpr unsigned long long kTetheringApexDomainBitmask = + domainToBitmask(domain::tethering) | + domainToBitmask(domain::net_private) | + domainToBitmask(domain::net_shared) | + domainToBitmask(domain::netd_readonly) | + domainToBitmask(domain::netd_shared); + // see b/162057235. For arbitrary program types, the concern is that due to the lack of // SELinux access controls over BPF program attachpoints, we have no way to control the // attachment of programs to shared resources (or to detect when a shared resource @@ -64,6 +72,7 @@ constexpr bpf_prog_type kVendorAllowedProgTypes[] = { struct Location { const char* const dir; const char* const prefix; + unsigned long long allowedDomainBitmask; const bpf_prog_type* allowedProgTypes = nullptr; size_t allowedProgTypesLength = 0; }; @@ -73,38 +82,45 @@ const Location locations[] = { { .dir = "/apex/com.android.tethering/etc/bpf/", .prefix = "tethering/", + .allowedDomainBitmask = kTetheringApexDomainBitmask, }, // T+ Tethering mainline module (shared with netd & system server) // netutils_wrapper (for iptables xt_bpf) has access to programs { .dir = "/apex/com.android.tethering/etc/bpf/netd_shared/", .prefix = "netd_shared/", + .allowedDomainBitmask = kTetheringApexDomainBitmask, }, // T+ Tethering mainline module (shared with netd & system server) // netutils_wrapper has no access, netd has read only access { .dir = "/apex/com.android.tethering/etc/bpf/netd_readonly/", .prefix = "netd_readonly/", + .allowedDomainBitmask = kTetheringApexDomainBitmask, }, // T+ Tethering mainline module (shared with system server) { .dir = "/apex/com.android.tethering/etc/bpf/net_shared/", .prefix = "net_shared/", + .allowedDomainBitmask = kTetheringApexDomainBitmask, }, // T+ Tethering mainline module (not shared, just network_stack) { .dir = "/apex/com.android.tethering/etc/bpf/net_private/", .prefix = "net_private/", + .allowedDomainBitmask = kTetheringApexDomainBitmask, }, // Core operating system { .dir = "/system/etc/bpf/", .prefix = "", + .allowedDomainBitmask = domainToBitmask(domain::platform), }, // Vendor operating system { .dir = "/vendor/etc/bpf/", .prefix = "vendor/", + .allowedDomainBitmask = domainToBitmask(domain::vendor), .allowedProgTypes = kVendorAllowedProgTypes, .allowedProgTypesLength = arraysize(kVendorAllowedProgTypes), }, @@ -124,7 +140,9 @@ int loadAllElfObjects(const Location& location) { progPath += s; bool critical; - int ret = android::bpf::loadProg(progPath.c_str(), &critical, location.prefix, + int ret = android::bpf::loadProg(progPath.c_str(), &critical, + location.prefix, + location.allowedDomainBitmask, location.allowedProgTypes, location.allowedProgTypesLength); if (ret) { @@ -160,9 +178,16 @@ int main(int argc, char** argv) { (void)argc; android::base::InitLogging(argv, &android::base::KernelLogger); - // Load all ELF objects, create programs and maps, and pin them + // Create all the pin subdirectories + // (this must be done first to allow selinux_context and pin_subdir functionality, + // which could otherwise fail with ENOENT during object pinning or renaming, + // due to ordering issues) for (const auto& location : locations) { createSysFsBpfSubDir(location.prefix); + } + + // Load all ELF objects, create programs and maps, and pin them + for (const auto& location : locations) { if (loadAllElfObjects(location) != 0) { ALOGE("=== CRITICAL FAILURE LOADING BPF PROGRAMS FROM %s ===", location.dir); ALOGE("If this triggers reliably, you're probably missing kernel options or patches."); diff --git a/libbpf_android/BpfLoadTest.cpp b/libbpf_android/BpfLoadTest.cpp index db45da1..d5e5814 100644 --- a/libbpf_android/BpfLoadTest.cpp +++ b/libbpf_android/BpfLoadTest.cpp @@ -53,7 +53,7 @@ class BpfLoadTest : public TestWithParam { bpf_prog_type kAllowed[] = { BPF_PROG_TYPE_UNSPEC, }; - EXPECT_EQ(android::bpf::loadProg(progPath.c_str(), &critical, "", kAllowed, + EXPECT_EQ(android::bpf::loadProg(progPath.c_str(), &critical, "", 0, kAllowed, arraysize(kAllowed)), -1); diff --git a/libbpf_android/Loader.cpp b/libbpf_android/Loader.cpp index f06d20f..db00634 100644 --- a/libbpf_android/Loader.cpp +++ b/libbpf_android/Loader.cpp @@ -30,9 +30,9 @@ #include #include -// This is BpfLoader v0.17 +// This is BpfLoader v0.18 #define BPFLOADER_VERSION_MAJOR 0u -#define BPFLOADER_VERSION_MINOR 17u +#define BPFLOADER_VERSION_MINOR 18u #define BPFLOADER_VERSION ((BPFLOADER_VERSION_MAJOR << 16) | BPFLOADER_VERSION_MINOR) #include "bpf/BpfUtils.h" @@ -73,6 +73,68 @@ using std::vector; namespace android { namespace bpf { +constexpr const char* lookupSelinuxContext(const domain d, const char* const unspecified = "") { + switch (d) { + case domain::unspecified: return unspecified; + case domain::platform: return "fs_bpf"; + case domain::tethering: return "fs_bpf_tethering"; + case domain::net_private: return "fs_bpf_net_private"; + case domain::net_shared: return "fs_bpf_net_shared"; + case domain::netd_readonly: return "fs_bpf_netd_readonly"; + case domain::netd_shared: return "fs_bpf_netd_shared"; + case domain::vendor: return "fs_bpf_vendor"; + default: return "(unrecognized)"; + } +} + +domain getDomainFromSelinuxContext(const char s[BPF_SELINUX_CONTEXT_CHAR_ARRAY_SIZE]) { + for (domain d : AllDomains) { + // Not sure how to enforce this at compile time, so abort() bpfloader at boot instead + if (strlen(lookupSelinuxContext(d)) >= BPF_SELINUX_CONTEXT_CHAR_ARRAY_SIZE) abort(); + if (!strncmp(s, lookupSelinuxContext(d), BPF_SELINUX_CONTEXT_CHAR_ARRAY_SIZE)) return d; + } + ALOGW("ignoring unrecognized selinux_context '%32s'", s); + // We should return 'unrecognized' here, however: returning unspecified will + // result in the system simply using the default context, which in turn + // will allow future expansion by adding more restrictive selinux types. + // Older bpfloader will simply ignore that, and use the less restrictive default. + // This does mean you CANNOT later add a *less* restrictive type than the default. + // + // Note: we cannot just abort() here as this might be a mainline module shipped optional update + return domain::unspecified; +} + +constexpr const char* lookupPinSubdir(const domain d, const char* const unspecified = "") { + switch (d) { + case domain::unspecified: return unspecified; + case domain::platform: return "/"; + case domain::tethering: return "tethering/"; + case domain::net_private: return "net_private/"; + case domain::net_shared: return "net_shared/"; + case domain::netd_readonly: return "netd_readonly/"; + case domain::netd_shared: return "netd_shared/"; + case domain::vendor: return "vendor/"; + default: return "(unrecognized)"; + } +}; + +domain getDomainFromPinSubdir(const char s[BPF_PIN_SUBDIR_CHAR_ARRAY_SIZE]) { + for (domain d : AllDomains) { + // Not sure how to enforce this at compile time, so abort() bpfloader at boot instead + if (strlen(lookupPinSubdir(d)) >= BPF_PIN_SUBDIR_CHAR_ARRAY_SIZE) abort(); + if (!strncmp(s, lookupPinSubdir(d), BPF_PIN_SUBDIR_CHAR_ARRAY_SIZE)) return d; + } + ALOGE("unrecognized pin_subdir '%32s'", s); + // pin_subdir affects the object's full pathname, + // and thus using the default would change the location and thus our code's ability to find it, + // hence this seems worth treating as a true error condition. + // + // Note: we cannot just abort() here as this might be a mainline module shipped optional update + // However, our callers will treat this as an error, and stop loading the specific .o, + // which will fail bpfloader if the .o is marked critical. + return domain::unrecognized; +} + static string pathToFilename(const string& path, bool noext = false) { vector spath = android::base::Split(path, "/"); string ret = spath.back(); @@ -623,7 +685,8 @@ static bool mapMatchesExpectations(unique_fd& fd, string& mapName, struct bpf_ma } static int createMaps(const char* elfPath, ifstream& elfFile, vector& mapFds, - const char* prefix, size_t sizeOfBpfMapDef) { + const char* prefix, const unsigned long long allowedDomainBitmask, + const size_t sizeOfBpfMapDef) { int ret; vector mdData, btfData; vector md; @@ -719,9 +782,35 @@ static int createMaps(const char* elfPath, ifstream& elfFile, vector& type = BPF_MAP_TYPE_HASH; } - // Format of pin location is /sys/fs/bpf/map__ - string mapPinLoc = - string(BPF_FS_PATH) + prefix + "map_" + fname + "_" + string(mapNames[i]); + domain selinux_context = getDomainFromSelinuxContext(md[i].selinux_context); + if (specified(selinux_context)) { + if (!inDomainBitmask(selinux_context, allowedDomainBitmask)) { + ALOGE("map %s has invalid selinux_context of %d (allowed bitmask 0x%llx)", + mapNames[i].c_str(), selinux_context, allowedDomainBitmask); + return -EINVAL; + } + ALOGI("map %s selinux_context [%32s] -> %d -> '%s' (%s)", mapNames[i].c_str(), + md[i].selinux_context, selinux_context, lookupSelinuxContext(selinux_context), + lookupPinSubdir(selinux_context)); + } + + domain pin_subdir = getDomainFromPinSubdir(md[i].pin_subdir); + if (unrecognized(pin_subdir)) return -ENOTDIR; + if (specified(pin_subdir)) { + if (!inDomainBitmask(pin_subdir, allowedDomainBitmask)) { + ALOGE("map %s has invalid pin_subdir of %d (allowed bitmask 0x%llx)", + mapNames[i].c_str(), pin_subdir, allowedDomainBitmask); + return -EINVAL; + } + ALOGI("map %s pin_subdir [%32s] -> %d -> '%s'", mapNames[i].c_str(), md[i].pin_subdir, + pin_subdir, lookupPinSubdir(pin_subdir)); + } + + // Format of pin location is /sys/fs/bpf/map__ + // except that maps shared across .o's have empty + // Note: refers to the extension-less basename of the .o file. + string mapPinLoc = string(BPF_FS_PATH) + lookupPinSubdir(pin_subdir, prefix) + "map_" + + (md[i].shared ? "" : fname) + "_" + mapNames[i]; bool reuse = false; unique_fd fd; int saved_errno; @@ -758,12 +847,40 @@ static int createMaps(const char* elfPath, ifstream& elfFile, vector& if (!mapMatchesExpectations(fd, mapNames[i], md[i], type)) return -ENOTUNIQ; if (!reuse) { - ret = bpf_obj_pin(fd, mapPinLoc.c_str()); - if (ret) return -errno; + if (specified(selinux_context)) { + string createLoc = string(BPF_FS_PATH) + lookupPinSubdir(selinux_context) + + "tmp_map_" + fname + "_" + mapNames[i]; + ret = bpf_obj_pin(fd, createLoc.c_str()); + if (ret) { + int err = errno; + ALOGE("create %s -> %d [%d:%s]", createLoc.c_str(), ret, err, strerror(err)); + return -err; + } + ret = rename(createLoc.c_str(), mapPinLoc.c_str()); + if (ret) { + int err = errno; + ALOGE("rename %s %s -> %d [%d:%s]", createLoc.c_str(), mapPinLoc.c_str(), ret, + err, strerror(err)); + return -err; + } + } else { + ret = bpf_obj_pin(fd, mapPinLoc.c_str()); + if (ret) return -errno; + } ret = chown(mapPinLoc.c_str(), (uid_t)md[i].uid, (gid_t)md[i].gid); - if (ret) return -errno; + if (ret) { + int err = errno; + ALOGE("chown(%s, %u, %u) = %d [%d:%s]", mapPinLoc.c_str(), md[i].uid, md[i].gid, + ret, err, strerror(err)); + return -err; + } ret = chmod(mapPinLoc.c_str(), md[i].mode); - if (ret) return -errno; + if (ret) { + int err = errno; + ALOGE("chmod(%s, 0%o) = %d [%d:%s]", mapPinLoc.c_str(), md[i].mode, ret, err, + strerror(err)); + return -err; + } } struct bpf_map_info map_info = {}; @@ -855,7 +972,7 @@ static void applyMapRelo(ifstream& elfFile, vector &mapFds, vector& cs, const string& license, - const char* prefix) { + const char* prefix, const unsigned long long allowedDomainBitmask) { unsigned kvers = kernelVersion(); int ret, fd; @@ -867,6 +984,8 @@ static int loadCodeSections(const char* elfPath, vector& cs, const string name = cs[i].name; unsigned bpfMinVer = DEFAULT_BPFLOADER_MIN_VER; // v0.0 unsigned bpfMaxVer = DEFAULT_BPFLOADER_MAX_VER; // v1.0 + domain selinux_context = domain::unspecified; + domain pin_subdir = domain::unspecified; if (cs[i].prog_def.has_value()) { unsigned min_kver = cs[i].prog_def->min_kver; @@ -878,12 +997,38 @@ static int loadCodeSections(const char* elfPath, vector& cs, const bpfMinVer = cs[i].prog_def->bpfloader_min_ver; bpfMaxVer = cs[i].prog_def->bpfloader_max_ver; + selinux_context = getDomainFromSelinuxContext(cs[i].prog_def->selinux_context); + pin_subdir = getDomainFromPinSubdir(cs[i].prog_def->pin_subdir); + // Note: make sure to only check for unrecognized *after* verifying bpfloader + // version limits include this bpfloader's version. } ALOGD("cs[%d].name:%s requires bpfloader version [0x%05x,0x%05x)", i, name.c_str(), bpfMinVer, bpfMaxVer); if (BPFLOADER_VERSION < bpfMinVer) continue; if (BPFLOADER_VERSION >= bpfMaxVer) continue; + if (unrecognized(pin_subdir)) return -ENOTDIR; + + if (specified(selinux_context)) { + if (!inDomainBitmask(selinux_context, allowedDomainBitmask)) { + ALOGE("prog %s has invalid selinux_context of %d (allowed bitmask 0x%llx)", + name.c_str(), selinux_context, allowedDomainBitmask); + return -EINVAL; + } + ALOGI("prog %s selinux_context [%32s] -> %d -> '%s' (%s)", name.c_str(), + cs[i].prog_def->selinux_context, selinux_context, + lookupSelinuxContext(selinux_context), lookupPinSubdir(selinux_context)); + } + + if (specified(pin_subdir)) { + if (!inDomainBitmask(pin_subdir, allowedDomainBitmask)) { + ALOGE("prog %s has invalid pin_subdir of %d (allowed bitmask 0x%llx)", name.c_str(), + pin_subdir, allowedDomainBitmask); + return -EINVAL; + } + ALOGI("prog %s pin_subdir [%32s] -> %d -> '%s'", name.c_str(), + cs[i].prog_def->pin_subdir, pin_subdir, lookupPinSubdir(pin_subdir)); + } // strip any potential $foo suffix // this can be used to provide duplicate programs @@ -893,12 +1038,8 @@ static int loadCodeSections(const char* elfPath, vector& cs, const bool reuse = false; // Format of pin location is // /sys/fs/bpf/prog__ - string progPinLoc = BPF_FS_PATH; - progPinLoc += prefix; - progPinLoc += "prog_"; - progPinLoc += fname; - progPinLoc += '_'; - progPinLoc += name; + string progPinLoc = string(BPF_FS_PATH) + lookupPinSubdir(pin_subdir, prefix) + "prog_" + + fname + '_' + string(name); if (access(progPinLoc.c_str(), F_OK) == 0) { fd = retrieveProgram(progPinLoc.c_str()); ALOGD("New bpf prog load reusing prog %s, ret: %d (%s)", progPinLoc.c_str(), fd, @@ -941,13 +1082,42 @@ static int loadCodeSections(const char* elfPath, vector& cs, const if (fd == 0) return -EINVAL; if (!reuse) { - ret = bpf_obj_pin(fd, progPinLoc.c_str()); - if (ret) return -errno; - if (chmod(progPinLoc.c_str(), 0440)) return -errno; + if (specified(selinux_context)) { + string createLoc = string(BPF_FS_PATH) + lookupPinSubdir(selinux_context) + + "tmp_prog_" + fname + '_' + string(name); + ret = bpf_obj_pin(fd, createLoc.c_str()); + if (ret) { + int err = errno; + ALOGE("create %s -> %d [%d:%s]", createLoc.c_str(), ret, err, strerror(err)); + return -err; + } + ret = rename(createLoc.c_str(), progPinLoc.c_str()); + if (ret) { + int err = errno; + ALOGE("rename %s %s -> %d [%d:%s]", createLoc.c_str(), progPinLoc.c_str(), ret, + err, strerror(err)); + return -err; + } + } else { + ret = bpf_obj_pin(fd, progPinLoc.c_str()); + if (ret) { + int err = errno; + ALOGE("create %s -> %d [%d:%s]", progPinLoc.c_str(), ret, err, strerror(err)); + return -err; + } + } + if (chmod(progPinLoc.c_str(), 0440)) { + int err = errno; + ALOGE("chmod %s 0440 -> [%d:%s]", progPinLoc.c_str(), err, strerror(err)); + return -err; + } if (cs[i].prog_def.has_value()) { if (chown(progPinLoc.c_str(), (uid_t)cs[i].prog_def->uid, (gid_t)cs[i].prog_def->gid)) { - return -errno; + int err = errno; + ALOGE("chown %s %d %d -> [%d:%s]", progPinLoc.c_str(), cs[i].prog_def->uid, + cs[i].prog_def->gid, err, strerror(err)); + return -err; } } } @@ -968,7 +1138,8 @@ static int loadCodeSections(const char* elfPath, vector& cs, const } int loadProg(const char* elfPath, bool* isCritical, const char* prefix, - const bpf_prog_type* allowed, size_t numAllowed) { + const unsigned long long allowedDomainBitmask, const bpf_prog_type* allowed, + size_t numAllowed) { vector license; vector critical; vector cs; @@ -1042,7 +1213,7 @@ int loadProg(const char* elfPath, bool* isCritical, const char* prefix, /* Just for future debugging */ if (0) dumpAllCs(cs); - ret = createMaps(elfPath, elfFile, mapFds, prefix, sizeOfBpfMapDef); + ret = createMaps(elfPath, elfFile, mapFds, prefix, allowedDomainBitmask, sizeOfBpfMapDef); if (ret) { ALOGE("Failed to create maps: (ret=%d) in %s", ret, elfPath); return ret; @@ -1053,7 +1224,7 @@ int loadProg(const char* elfPath, bool* isCritical, const char* prefix, applyMapRelo(elfFile, mapFds, cs); - ret = loadCodeSections(elfPath, cs, string(license.data()), prefix); + ret = loadCodeSections(elfPath, cs, string(license.data()), prefix, allowedDomainBitmask); if (ret) ALOGE("Failed to load programs, loadCodeSections ret=%d", ret); return ret; diff --git a/libbpf_android/include/libbpf_android.h b/libbpf_android/include/libbpf_android.h index 3fb777b..2218215 100644 --- a/libbpf_android/include/libbpf_android.h +++ b/libbpf_android/include/libbpf_android.h @@ -23,8 +23,62 @@ namespace android { namespace bpf { +// Bpf programs may specify per-program & per-map selinux_context and pin_subdir. +// +// The BpfLoader needs to convert these bpf.o specified strings into an enum +// for internal use (to check that valid values were specified for the specific +// location of the bpf.o file). +// +// It also needs to map selinux_context's into pin_subdir's. +// This is because of how selinux_context is actually implemented via pin+rename. +// +// Thus 'domain' enumerates all selinux_context's/pin_subdir's that the BpfLoader +// is aware of. Thus there currently needs to be a 1:1 mapping between the two. +// +enum class domain : int { + unrecognized = -1, // invalid for this version of the bpfloader + unspecified = 0, // means just use the default for that specific pin location + platform, // fs_bpf /sys/fs/bpf + tethering, // (S+) fs_bpf_tethering /sys/fs/bpf/tethering + net_private, // (T+) fs_bpf_net_private /sys/fs/bpf/net_private + net_shared, // (T+) fs_bpf_net_shared /sys/fs/bpf/net_shared + netd_readonly, // (T+) fs_bpf_netd_readonly /sys/fs/bpf/netd_readonly + netd_shared, // (T+) fs_bpf_netd_shared /sys/fs/bpf/netd_shared + vendor, // (T+) fs_bpf_vendor /sys/fs/bpf/vendor +}; + +// Note: this does not include domain::unrecognized, but does include domain::unspecified +static constexpr domain AllDomains[] = { + domain::unspecified, + domain::platform, + domain::tethering, + domain::net_private, + domain::net_shared, + domain::netd_readonly, + domain::netd_shared, + domain::vendor, +}; + +static constexpr bool unrecognized(domain d) { + return d == domain::unrecognized; +} + +// Note: this doesn't handle unrecognized, handle it first. +static constexpr bool specified(domain d) { + return d != domain::unspecified; +} + +static constexpr unsigned long long domainToBitmask(domain d) { + return specified(d) ? 1uLL << (static_cast(d) - 1) : 0; +} + +static constexpr bool inDomainBitmask(domain d, unsigned long long v) { + return domainToBitmask(d) & v; +} + // BPF loader implementation. Loads an eBPF ELF object int loadProg(const char* elfPath, bool* isCritical, const char* prefix = "", + const unsigned long long allowedDomainBitmask = 0, const bpf_prog_type* allowed = nullptr, size_t numAllowed = 0); // Exposed for testing