tequilaOS/platform_bionic
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Add support for removing structs that cross blocks." am: 0ba4b34206

Browse source 
Original change: https://android-review.googlesource.com/c/platform/bionic/+/2175190

Change-Id: I94abfec1aa97a9dce5d3537ebd8f1a49725ec4c8
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Christopher Ferris 2022-08-10 20:17:48 +00:00 committed by Automerger Merge Worker
commit 97417cb356
3 changed files with 196 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

216
libc/kernel/tools/cpp.py
View file

@ -155,6 +155,11 @@ class BadExpectedToken(Exception):
pass
class UnparseableStruct(Exception):
"""An exception that will be raised for structs that cannot be parsed."""
pass
# The __contains__ function in libclang SourceRange class contains a bug. It
# gives wrong result when dealing with single line range.
# Bug filed with upstream:
@ -1197,22 +1202,38 @@ class BlockList(object):
def removeStructs(self, structs):
"""Remove structs."""
for b in self.blocks:
extra_includes = []
block_num = 0
num_blocks = len(self.blocks)
while block_num < num_blocks:
b = self.blocks[block_num]
block_num += 1
# Have to look in each block for a top-level struct definition.
if b.directive:
continue
num_tokens = len(b.tokens)
# A struct definition has at least 5 tokens:
# A struct definition usually looks like:
# struct
# ident
# {
# }
# ;
if num_tokens < 5:
# However, the structure might be spread across multiple blocks
# if the structure looks like this:
# struct ident
# {
# #ifdef VARIABLE
# pid_t pid;
# #endif
# }:
# So the total number of tokens in the block might be less than
# five but assume at least three.
if num_tokens < 3:
continue
# This is a simple struct finder, it might fail if a top-level
# structure has an #if type directives that confuses the algorithm
# for finding th end of the structure. Or if there is another
# for finding the end of the structure. Or if there is another
# structure definition embedded in the structure.
i = 0
while i < num_tokens - 2:
@ -1223,24 +1244,58 @@ class BlockList(object):
if (b.tokens[i + 1].kind == TokenKind.IDENTIFIER and
b.tokens[i + 2].kind == TokenKind.PUNCTUATION and
b.tokens[i + 2].id == "{" and b.tokens[i + 1].id in structs):
# Add an include for the structure to be removed of the form:
# #include <bits/STRUCT_NAME.h>
struct_token = b.tokens[i + 1]
if not structs[struct_token.id]:
extra_includes.append("<bits/%s.h>" % struct_token.id)
# Search forward for the end of the structure.
# Very simple search, look for } and ; tokens. If something
# more complicated is needed we can add it later.
# Very simple search, look for } and ; tokens.
# If we hit the end of the block, we'll need to start
# looking at the next block.
j = i + 3
while j < num_tokens - 1:
if (b.tokens[j].kind == TokenKind.PUNCTUATION and
b.tokens[j].id == "}" and
b.tokens[j + 1].kind == TokenKind.PUNCTUATION and
b.tokens[j + 1].id == ";"):
b.tokens = b.tokens[0:i] + b.tokens[j + 2:num_tokens]
depth = 1
struct_removed = False
while not struct_removed:
while j < num_tokens:
if b.tokens[j].kind == TokenKind.PUNCTUATION:
if b.tokens[j].id == '{':
depth += 1
elif b.tokens[j].id == '}':
depth -= 1
elif b.tokens[j].id == ';' and depth == 0:
b.tokens = b.tokens[0:i] + b.tokens[j + 1:num_tokens]
num_tokens = len(b.tokens)
j = i
struct_removed = True
break
j += 1
i = j
if not struct_removed:
b.tokens = b.tokens[0:i]
# Skip directive blocks.
start_block = block_num
while block_num < num_blocks:
if not self.blocks[block_num].directive:
break
block_num += 1
if block_num >= num_blocks:
# Unparsable struct, error out.
raise UnparseableStruct("Cannot remove struct %s: %s" % (struct_token.id, struct_token.location))
self.blocks = self.blocks[0:start_block] + self.blocks[block_num:num_blocks]
num_blocks = len(self.blocks)
b = self.blocks[start_block]
block_num = start_block + 1
num_tokens = len(b.tokens)
i = 0
j = 0
continue
i += 1
for extra_include in extra_includes:
replacement = CppStringTokenizer(extra_include)
self.blocks.insert(2, Block(replacement.tokens, directive='include'))
def optimizeAll(self, macros):
self.optimizeMacros(macros)
self.optimizeIf01()
@ -1404,35 +1459,12 @@ class BlockList(object):
def replaceTokens(self, replacements):
"""Replace tokens according to the given dict."""
extra_includes = []
for b in self.blocks:
made_change = False
if b.isInclude() is None:
i = 0
while i < len(b.tokens):
tok = b.tokens[i]
if (tok.kind == TokenKind.KEYWORD and tok.id == 'struct'
and (i + 2) < len(b.tokens) and b.tokens[i + 2].id == '{'):
struct_name = b.tokens[i + 1].id
if struct_name in kernel_struct_replacements:
extra_includes.append("<bits/%s.h>" % struct_name)
end = i + 2
depth = 1
while end < len(b.tokens) and depth > 0:
if b.tokens[end].id == '}':
depth -= 1
elif b.tokens[end].id == '{':
depth += 1
end += 1
end += 1 # Swallow last '}'
while end < len(b.tokens) and b.tokens[end].id != ';':
end += 1
end += 1 # Swallow ';'
# Remove these tokens. We'll replace them later with a #include block.
b.tokens[i:end] = []
made_change = True
# We've just modified b.tokens, so revisit the current offset.
continue
if tok.kind == TokenKind.IDENTIFIER:
if tok.id in replacements:
tok.id = replacements[tok.id]
@ -1447,10 +1479,6 @@ class BlockList(object):
# Keep 'expr' in sync with 'tokens'.
b.expr = CppExpr(b.tokens)
for extra_include in extra_includes:
replacement = CppStringTokenizer(extra_include)
self.blocks.insert(2, Block(replacement.tokens, directive='include'))
def strip_space(s):
@ -2020,7 +2048,7 @@ struct something {
struct timeval val2;
};
"""
self.assertEqual(self.parse(text, set(["remove"])), expected)
self.assertEqual(self.parse(text, {"remove": True}), expected)
def test_remove_struct_from_end(self):
text = """\
@ -2039,7 +2067,7 @@ struct something {
struct timeval val2;
};
"""
self.assertEqual(self.parse(text, set(["remove"])), expected)
self.assertEqual(self.parse(text, {"remove": True}), expected)
def test_remove_minimal_struct(self):
text = """\
@ -2047,7 +2075,7 @@ struct remove {
};
"""
expected = "";
self.assertEqual(self.parse(text, set(["remove"])), expected)
self.assertEqual(self.parse(text, {"remove": True}), expected)
def test_remove_struct_with_struct_fields(self):
text = """\
@ -2067,7 +2095,7 @@ struct something {
struct remove val2;
};
"""
self.assertEqual(self.parse(text, set(["remove"])), expected)
self.assertEqual(self.parse(text, {"remove": True}), expected)
def test_remove_consecutive_structs(self):
text = """\
@ -2099,7 +2127,7 @@ struct keep2 {
struct timeval val2;
};
"""
self.assertEqual(self.parse(text, set(["remove1", "remove2"])), expected)
self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
def test_remove_multiple_structs(self):
text = """\
@ -2132,7 +2160,101 @@ struct keep3 {
int val;
};
"""
self.assertEqual(self.parse(text, set(["remove1", "remove2"])), expected)
self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
def test_remove_struct_with_inline_structs(self):
text = """\
struct remove {
int val1;
int val2;
struct {
int val1;
struct {
int val1;
} level2;
} level1;
};
struct something {
struct timeval val1;
struct timeval val2;
};
"""
expected = """\
struct something {
struct timeval val1;
struct timeval val2;
};
"""
self.assertEqual(self.parse(text, {"remove": True}), expected)
def test_remove_struct_across_blocks(self):
text = """\
struct remove {
int val1;
int val2;
#ifdef PARAMETER1
PARAMETER1
#endif
#ifdef PARAMETER2
PARAMETER2
#endif
};
struct something {
struct timeval val1;
struct timeval val2;
};
"""
expected = """\
struct something {
struct timeval val1;
struct timeval val2;
};
"""
self.assertEqual(self.parse(text, {"remove": True}), expected)
def test_remove_struct_across_blocks_multiple_structs(self):
text = """\
struct remove1 {
int val1;
int val2;
#ifdef PARAMETER1
PARAMETER1
#endif
#ifdef PARAMETER2
PARAMETER2
#endif
};
struct remove2 {
};
struct something {
struct timeval val1;
struct timeval val2;
};
"""
expected = """\
struct something {
struct timeval val1;
struct timeval val2;
};
"""
self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
def test_remove_multiple_struct_and_add_includes(self):
text = """\
struct remove1 {
int val1;
int val2;
};
struct remove2 {
struct timeval val1;
struct timeval val2;
};
"""
expected = """\
#include <bits/remove1.h>
#include <bits/remove2.h>
"""
self.assertEqual(self.parse(text, {"remove1": False, "remove2": False}), expected)
class FullPathTest(unittest.TestCase):

37
libc/kernel/tools/defaults.py
View file

@ -35,16 +35,23 @@ kernel_known_macros = {
"__kernel_old_timeval": "1",
}
# this is the set of known kernel data structures we want to remove from
# the final headers
kernel_structs_to_remove = set(
[
# This is the set of known kernel data structures we want to remove from
# the final headers. If the map value is False, that means that in
# addition to removing the structure, add an #include <bits/STRUCT.h>
# to the file.
kernel_structs_to_remove = {
# Remove the structures since they are still the same as
# timeval, itimerval.
"__kernel_old_timeval",
"__kernel_old_itimerval",
]
)
"__kernel_old_timeval": True,
"__kernel_old_itimerval": True,
# Replace all of the below structures with #include <bits/STRUCT.h>
"epoll_event": False,
"flock": False,
"flock64": False,
"in_addr": False,
"ip_mreq_source": False,
"ip_msfilter": False,
}
# define to true if you want to remove all defined(CONFIG_FOO) tests
# from the clean headers. testing shows that this is not strictly necessary
@ -100,20 +107,6 @@ kernel_token_replacements = {
}
# This is the set of struct definitions that we want to replace with
# a #include of <bits/struct.h> instead.
kernel_struct_replacements = set(
[
"epoll_event",
"flock",
"flock64",
"in_addr",
"ip_mreq_source",
"ip_msfilter",
]
)
# This is the set of known static inline functions that we want to keep
# in the final kernel headers.
kernel_known_generic_statics = set(

10
libc/kernel/tools/update_all.py
View file

@ -27,8 +27,14 @@ def Usage():
def ProcessFiles(updater, original_dir, modified_dir, src_rel_dir, update_rel_dir):
# Delete the old headers before updating to the new headers.
update_dir = os.path.join(get_kernel_dir(), update_rel_dir)
shutil.rmtree(update_dir)
os.mkdir(update_dir, 0o755)
for root, dirs, files in os.walk(update_dir, topdown=True):
for entry in files:
# BUILD is a special file that needs to be preserved.
if entry == "BUILD":
continue
os.remove(os.path.join(root, entry))
for entry in dirs:
shutil.rmtree(os.path.join(root, entry))
src_dir = os.path.normpath(os.path.join(original_dir, src_rel_dir))
src_dir_len = len(src_dir) + 1