From 11526e2fc607624dbb8bd4acf2ffdd3a187e9c02 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Thu, 14 Oct 2021 22:44:47 +0000 Subject: [PATCH] Add execinfo functionality. Bug: 27877410 Test: Add new unit tests. Change-Id: Id5d7eb27a23f50e99a04f5ee1ab64047ba269bab --- libc/Android.bp | 1 + libc/bionic/execinfo.cpp | 187 ++++++++++++++++++++++++++++++ libc/include/execinfo.h | 74 ++++++++++++ libc/libc.map.txt | 3 + tests/Android.bp | 5 + tests/NOTICE | 28 +++++ tests/execinfo_test.cpp | 244 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 542 insertions(+) create mode 100644 libc/bionic/execinfo.cpp create mode 100644 libc/include/execinfo.h create mode 100644 tests/execinfo_test.cpp diff --git a/libc/Android.bp b/libc/Android.bp index 2a8df9a47..ee8c280fc 100644 --- a/libc/Android.bp +++ b/libc/Android.bp @@ -1054,6 +1054,7 @@ cc_library_static { "bionic/error.cpp", "bionic/eventfd.cpp", "bionic/exec.cpp", + "bionic/execinfo.cpp", "bionic/faccessat.cpp", "bionic/fchmod.cpp", "bionic/fchmodat.cpp", diff --git a/libc/bionic/execinfo.cpp b/libc/bionic/execinfo.cpp new file mode 100644 index 000000000..d129f7cae --- /dev/null +++ b/libc/bionic/execinfo.cpp @@ -0,0 +1,187 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "private/ScopedFd.h" + +struct StackState { + void** frames; + int frame_count; + int cur_frame = 0; + + StackState(void** frames, int frame_count) : frames(frames), frame_count(frame_count) {} +}; + +static _Unwind_Reason_Code TraceFunction(_Unwind_Context* context, void* arg) { + // The instruction pointer is pointing at the instruction after the return + // call on all architectures. + // Modify the pc to point at the real function. + uintptr_t ip = _Unwind_GetIP(context); + if (ip != 0) { +#if defined(__arm__) + // If the ip is suspiciously low, do nothing to avoid a segfault trying + // to access this memory. + if (ip >= 4096) { + // Check bits [15:11] of the first halfword assuming the instruction + // is 32 bits long. If the bits are any of these values, then our + // assumption was correct: + // b11101 + // b11110 + // b11111 + // Otherwise, this is a 16 bit instruction. + uint16_t value = (*reinterpret_cast(ip - 2)) >> 11; + if (value == 0x1f || value == 0x1e || value == 0x1d) { + ip -= 4; + } else { + ip -= 2; + } + } +#elif defined(__aarch64__) + // All instructions are 4 bytes long, skip back one instruction. + ip -= 4; +#elif defined(__i386__) || defined(__x86_64__) + // It's difficult to decode exactly where the previous instruction is, + // so subtract 1 to estimate where the instruction lives. + ip--; +#endif + } + + StackState* state = static_cast(arg); + state->frames[state->cur_frame++] = reinterpret_cast(ip); + return (state->cur_frame >= state->frame_count) ? _URC_END_OF_STACK : _URC_NO_REASON; +} + +int backtrace(void** buffer, int size) { + if (size <= 0) { + return 0; + } + + StackState state(buffer, size); + _Unwind_Backtrace(TraceFunction, &state); + return state.cur_frame; +} + +char** backtrace_symbols(void* const* buffer, int size) { + if (size <= 0) { + return nullptr; + } + // Do this calculation first in case the user passes in a bad value. + size_t ptr_size; + if (__builtin_mul_overflow(sizeof(char*), size, &ptr_size)) { + return nullptr; + } + + ScopedFd fd(memfd_create("backtrace_symbols_fd", MFD_CLOEXEC)); + if (fd.get() == -1) { + return nullptr; + } + backtrace_symbols_fd(buffer, size, fd.get()); + + // Get the size of the file. + off_t file_size = lseek(fd.get(), 0, SEEK_END); + if (file_size <= 0) { + return nullptr; + } + + // The interface for backtrace_symbols indicates that only the single + // returned pointer must be freed by the caller. Therefore, allocate a + // buffer that includes the memory for the strings and all of the pointers. + // Add one byte at the end just in case the file didn't end with a '\n'. + size_t symbol_data_size; + if (__builtin_add_overflow(ptr_size, file_size, &symbol_data_size) || + __builtin_add_overflow(symbol_data_size, 1, &symbol_data_size)) { + return nullptr; + } + + uint8_t* symbol_data = reinterpret_cast(malloc(symbol_data_size)); + if (symbol_data == nullptr) { + return nullptr; + } + + // Copy the string data into the buffer. + char* cur_string = reinterpret_cast(&symbol_data[ptr_size]); + // If this fails, the read won't read back the correct number of bytes. + lseek(fd.get(), 0, SEEK_SET); + ssize_t num_read = read(fd.get(), cur_string, file_size); + fd.reset(-1); + if (num_read != file_size) { + free(symbol_data); + return nullptr; + } + + // Make sure the last character in the file is '\n'. + if (cur_string[file_size] != '\n') { + cur_string[file_size++] = '\n'; + } + + for (int i = 0; i < size; i++) { + (reinterpret_cast(symbol_data))[i] = cur_string; + cur_string = strchr(cur_string, '\n'); + if (cur_string == nullptr) { + free(symbol_data); + return nullptr; + } + cur_string[0] = '\0'; + cur_string++; + } + return reinterpret_cast(symbol_data); +} + +// This function should do no allocations if possible. +void backtrace_symbols_fd(void* const* buffer, int size, int fd) { + if (size <= 0 || fd < 0) { + return; + } + + for (int frame_num = 0; frame_num < size; frame_num++) { + void* address = buffer[frame_num]; + Dl_info info; + if (dladdr(address, &info) != 0) { + if (info.dli_fname != nullptr) { + write(fd, info.dli_fname, strlen(info.dli_fname)); + } + if (info.dli_sname != nullptr) { + dprintf(fd, "(%s+0x%" PRIxPTR ") ", info.dli_sname, + reinterpret_cast(address) - reinterpret_cast(info.dli_saddr)); + } else { + dprintf(fd, "(+%p) ", info.dli_saddr); + } + } + + dprintf(fd, "[%p]\n", address); + } +} diff --git a/libc/include/execinfo.h b/libc/include/execinfo.h new file mode 100644 index 000000000..347ae9289 --- /dev/null +++ b/libc/include/execinfo.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#pragma once + +#include + +/** + * @file execinfo.h + * @brief Functions to do in process backtracing. + */ + +__BEGIN_DECLS + +/** + * [backtrace(3)](https://man7.org/linux/man-pages/man3/backtrace.3.html) + * Saves a backtrace for the current call in the array pointed to by buffer. + * "size" indicates the maximum number of void* pointers that can be set. + * + * Returns the number of addresses stored in "buffer", which is not greater + * than "size". If the return value is equal to "size" then the number of + * addresses may have been truncated. + * + * Available since API level 33. + */ +int backtrace(void** buffer, int size) __INTRODUCED_IN(33); + +/** + * [backtrace_symbols(3)](https://man7.org/linux/man-pages/man3/backtrace_symbols.3.html) + * Given an array of void* pointers, translate the addresses into an array + * of strings that represent the backtrace. + * + * Returns a pointer to allocated memory, on error NULL is returned. It is + * the responsibility of the caller to free the returned memory. + * + * Available since API level 33. + */ +char** backtrace_symbols(void* const* buffer, int size) __INTRODUCED_IN(33); + +/** + * [backtrace_symbols_fd(3)](https://man7.org/linux/man-pages/man3/backtrace_symbols_fd.3.html) + * Given an array of void* pointers, translate the addresses into an array + * of strings that represent the backtrace and write to the file represented + * by "fd". The file is written such that one line equals one void* address. + * + * Available since API level 33. + */ +void backtrace_symbols_fd(void* const* buffer, int size, int fd) __INTRODUCED_IN(33); + +__END_DECLS diff --git a/libc/libc.map.txt b/libc/libc.map.txt index 12b811461..00b25d395 100644 --- a/libc/libc.map.txt +++ b/libc/libc.map.txt @@ -1567,6 +1567,9 @@ LIBC_S { # introduced=S LIBC_T { # introduced=Tiramisu global: + backtrace; + backtrace_symbols; + backtrace_symbols_fd; preadv2; preadv64v2; pwritev2; diff --git a/tests/Android.bp b/tests/Android.bp index cbf0a9f36..8a57dbbf4 100644 --- a/tests/Android.bp +++ b/tests/Android.bp @@ -474,6 +474,9 @@ cc_test_library { // musl doesn't have error.h "error_test.cpp", + // musl doesn't have execinfo.h + "execinfo_test.cpp", + // musl doesn't define noreturn for C++ "stdnoreturn_test.cpp", @@ -745,6 +748,7 @@ cc_test_library { "dl_test.cpp", "dlfcn_symlink_support.cpp", "dlfcn_test.cpp", + "execinfo_test.cpp", "link_test.cpp", "pthread_dlfcn_test.cpp", ], @@ -1141,6 +1145,7 @@ cc_test_host { "dlfcn_symlink_support.cpp", "dlfcn_test.cpp", "dl_test.cpp", + "execinfo_test.cpp", "gtest_globals.cpp", "gtest_main.cpp", "pthread_dlfcn_test.cpp", diff --git a/tests/NOTICE b/tests/NOTICE index a58cf46d7..c9b65d07b 100644 --- a/tests/NOTICE +++ b/tests/NOTICE @@ -354,3 +354,31 @@ limitations under the License. ------------------------------------------------------------------- +Copyright (C) 2021 The Android Open Source Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +------------------------------------------------------------------- + diff --git a/tests/execinfo_test.cpp b/tests/execinfo_test.cpp new file mode 100644 index 000000000..b8e13252d --- /dev/null +++ b/tests/execinfo_test.cpp @@ -0,0 +1,244 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +TEST(execinfo, backtrace_errors) { + void* frames[20]; + ASSERT_EQ(0, backtrace(frames, 0)); + ASSERT_EQ(0, backtrace(frames, -1)); +} + +static constexpr int kMaxFrames = 50; + +// Disable optimizations so that these functions show up properly in +// the backtrace. +#pragma clang optimize off +extern "C" __attribute__((__noinline__)) void CallTwo(std::vector& frames) { + int num_frames = backtrace(frames.data(), static_cast(frames.size())); + ASSERT_LT(0, num_frames); + frames.resize(static_cast(num_frames)); +} + +extern "C" __attribute__((__noinline__)) void CallOne(std::vector& frames) { + CallTwo(frames); +} +#pragma clang optimize on + +static std::string DumpFrames(std::vector& frames) { + std::string frame_data; + for (auto frame : frames) { + frame_data += android::base::StringPrintf("[%p]", frame); + Dl_info info; + if (dladdr(frame, &info) != 0 && info.dli_sname != nullptr) { + frame_data += ' '; + frame_data += info.dli_sname; + } + frame_data += '\n'; + } + return frame_data; +} + +static size_t FindFunction(std::vector& frames, uintptr_t func_addr) { + for (size_t i = 0; i < frames.size(); i++) { + uintptr_t frame_addr = reinterpret_cast(frames[i]); + if (frame_addr >= func_addr && frame_addr <= func_addr + 0x100) { + return i + 1; + } + } + return 0; +} + +static void VerifyCalls(std::vector& frames, size_t* one_idx = nullptr, + size_t* two_idx = nullptr) { + // Try and find the CallOne and CallTwo function addresses. + size_t call_one_idx = FindFunction(frames, reinterpret_cast(&CallOne)); + ASSERT_TRUE(call_one_idx != 0) << DumpFrames(frames); + size_t call_two_idx = FindFunction(frames, reinterpret_cast(&CallTwo)); + ASSERT_TRUE(call_two_idx != 0) << DumpFrames(frames); + + ASSERT_LT(call_two_idx, call_one_idx) << "CallTwo function found after CallOne\n" + << DumpFrames(frames); + + if (one_idx != nullptr) *one_idx = call_one_idx; + if (two_idx != nullptr) *two_idx = call_two_idx; +} + +TEST(execinfo, backtrace) { + std::vector frames(kMaxFrames); + ASSERT_NO_FATAL_FAILURE(CallOne(frames)); + + // Verfiy that there are at least two frames. + ASSERT_LT(3U, frames.size()) << DumpFrames(frames); + + VerifyCalls(frames); +} + +TEST(execinfo, backtrace_cutoff_frames) { + // Verify the max frames is handled properly + std::vector frames(1); + ASSERT_NO_FATAL_FAILURE(CallOne(frames)); + ASSERT_EQ(1U, frames.size()) << DumpFrames(frames); +} + +TEST(execinfo, backtrace_symbols_errors) { + void* frames[kMaxFrames]; + // glibc incorrectly returns memory when a zero is passed in. + // Since we know this works properly on bionic, only verify + // this there. +#if defined(__BIONIC__) + ASSERT_EQ(nullptr, backtrace_symbols(frames, 0)); +#endif + ASSERT_EQ(nullptr, backtrace_symbols(frames, -1)); +} + +static void VerifyLineFormat(std::string& line) { + // Verify that the format of the line is one of these: + // elf_file(FuncName+0xFuncAddr) [0xAddress] + // elf_file(+0xRelAddress) [0xAddress] + // elf_file [0xAddress] + // [0xAddress] +#if defined(__GLIBC__) + // For some reason, glibc will print a space before [0xAddress] for + // backtrace symbols, and no space for backtrace_symbols_fd. Allow this + // only for glibc. + std::regex format1("[^\\(\\s]+\\([^\\+]+\\+0x[0-9a-fA-F]+\\) ?\\[0x[0-9a-fA-F]+\\]"); + std::regex format2("[^\\(\\s]+\\(+\\+0x[0-9a-fA-F]+\\) ?\\[0x[0-9a-fA-F]+\\]"); + std::regex format3("[^\\(\\s]+ ?\\[0x[0-9a-fA-F]+\\]"); +#else + std::regex format1("[^\\(\\s]+\\([^\\+]+\\+0x[0-9a-fA-F]+\\) \\[0x[0-9a-fA-F]+\\]"); + std::regex format2("[^\\(\\s]+\\(+\\+0x[0-9a-fA-F]+\\) \\[0x[0-9a-fA-F]+\\]"); + std::regex format3("[^\\(\\s]+ \\[0x[0-9a-fA-F]+\\]"); +#endif + std::regex format4("\\[0x[0-9a-fA-F]+\\]"); + + EXPECT_TRUE(std::regex_match(line, format1) || std::regex_match(line, format2) || + std::regex_match(line, format3) || std::regex_match(line, format4)) + << "Unknown format of line:\n" + << line; +} + +static void VerifyLineFormat(char* raw_line, size_t length) { + std::string line(raw_line, length); + VerifyLineFormat(line); +} + +TEST(execinfo, backtrace_symbols) { + std::vector frames(kMaxFrames); + ASSERT_NO_FATAL_FAILURE(CallOne(frames)); + ASSERT_LT(3U, frames.size()) << DumpFrames(frames); + + char** symbols = backtrace_symbols(frames.data(), static_cast(frames.size())); + ASSERT_TRUE(symbols != nullptr); + for (size_t i = 0; i < frames.size(); i++) { + ASSERT_TRUE(frames[i] != nullptr); + VerifyLineFormat(symbols[i], strlen(symbols[i])); + } + + size_t call_one_idx; + size_t call_two_idx; + ASSERT_NO_FATAL_FAILURE(VerifyCalls(frames, &call_one_idx, &call_two_idx)); + // Now verify that those frames contain the function names we expect. + SCOPED_TRACE(DumpFrames(frames)); + ASSERT_MATCH(symbols[call_one_idx - 1], "\\(CallOne+"); + ASSERT_MATCH(symbols[call_two_idx - 1], "\\(CallTwo+"); + free(symbols); +} + +TEST(execinfo, backtrace_symbols_fd_errors) { + void* frames[kMaxFrames]; + frames[0] = reinterpret_cast(&backtrace_symbols); + + { + TemporaryFile tf; + backtrace_symbols_fd(frames, 0, tf.fd); + close(tf.fd); + std::string content; + ASSERT_TRUE(android::base::ReadFileToString(tf.path, &content)); + // Verify that no data is written to the file. + ASSERT_TRUE(content.empty()); + } + + { + TemporaryFile tf; + backtrace_symbols_fd(frames, -1, tf.fd); + close(tf.fd); + std::string content; + ASSERT_TRUE(android::base::ReadFileToString(tf.path, &content)); + // Verify that no data is written to the file. + ASSERT_TRUE(content.empty()); + } + + // Verify that there isn't a crash. + backtrace_symbols_fd(frames, 0, -1); +} + +TEST(execinfo, backtrace_symbols_fd) { + std::vector frames(kMaxFrames); + ASSERT_NO_FATAL_FAILURE(CallOne(frames)); + ASSERT_LT(3U, frames.size()) << DumpFrames(frames); + + TemporaryFile tf; + backtrace_symbols_fd(frames.data(), static_cast(frames.size()), tf.fd); + close(tf.fd); + + size_t call_one_idx; + size_t call_two_idx; + ASSERT_NO_FATAL_FAILURE(VerifyCalls(frames, &call_one_idx, &call_two_idx)); + + std::ifstream frame_stream(tf.path); + ASSERT_TRUE(frame_stream.is_open()); + size_t num_lines = 0; + std::string line; + while (std::getline(frame_stream, line)) { + ASSERT_FALSE(line.empty()); + VerifyLineFormat(line); + num_lines++; + + if (num_lines == call_one_idx) { + EXPECT_MATCH(line, "\\(CallOne+"); + } else if (num_lines == call_two_idx) { + EXPECT_MATCH(line, "\\(CallTwo+"); + } + } + ASSERT_EQ(num_lines, frames.size()) << "Number of lines in file does not match number of frames."; +}