Merge "Add support for removing structs that cross blocks." am: 0ba4b34206
Original change: https://android-review.googlesource.com/c/platform/bionic/+/2175190 Change-Id: I94abfec1aa97a9dce5d3537ebd8f1a49725ec4c8 Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Christopher Ferris
Automerger Merge Worker
commit
97417cb356
3 changed files with 196 additions and 75 deletions
|
|
@ -155,6 +155,11 @@ class BadExpectedToken(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class UnparseableStruct(Exception):
|
||||
"""An exception that will be raised for structs that cannot be parsed."""
|
||||
pass
|
||||
|
||||
|
||||
# The __contains__ function in libclang SourceRange class contains a bug. It
|
||||
# gives wrong result when dealing with single line range.
|
||||
# Bug filed with upstream:
|
||||
|
|
@ -1197,22 +1202,38 @@ class BlockList(object):
|
|||
|
||||
def removeStructs(self, structs):
|
||||
"""Remove structs."""
|
||||
for b in self.blocks:
|
||||
extra_includes = []
|
||||
block_num = 0
|
||||
num_blocks = len(self.blocks)
|
||||
while block_num < num_blocks:
|
||||
b = self.blocks[block_num]
|
||||
block_num += 1
|
||||
# Have to look in each block for a top-level struct definition.
|
||||
if b.directive:
|
||||
continue
|
||||
num_tokens = len(b.tokens)
|
||||
# A struct definition has at least 5 tokens:
|
||||
# A struct definition usually looks like:
|
||||
# struct
|
||||
# ident
|
||||
# {
|
||||
# }
|
||||
# ;
|
||||
if num_tokens < 5:
|
||||
# However, the structure might be spread across multiple blocks
|
||||
# if the structure looks like this:
|
||||
# struct ident
|
||||
# {
|
||||
# #ifdef VARIABLE
|
||||
# pid_t pid;
|
||||
# #endif
|
||||
# }:
|
||||
# So the total number of tokens in the block might be less than
|
||||
# five but assume at least three.
|
||||
if num_tokens < 3:
|
||||
continue
|
||||
|
||||
# This is a simple struct finder, it might fail if a top-level
|
||||
# structure has an #if type directives that confuses the algorithm
|
||||
# for finding th end of the structure. Or if there is another
|
||||
# for finding the end of the structure. Or if there is another
|
||||
# structure definition embedded in the structure.
|
||||
i = 0
|
||||
while i < num_tokens - 2:
|
||||
|
|
@ -1223,24 +1244,58 @@ class BlockList(object):
|
|||
if (b.tokens[i + 1].kind == TokenKind.IDENTIFIER and
|
||||
b.tokens[i + 2].kind == TokenKind.PUNCTUATION and
|
||||
b.tokens[i + 2].id == "{" and b.tokens[i + 1].id in structs):
|
||||
# Add an include for the structure to be removed of the form:
|
||||
# #include <bits/STRUCT_NAME.h>
|
||||
struct_token = b.tokens[i + 1]
|
||||
if not structs[struct_token.id]:
|
||||
extra_includes.append("<bits/%s.h>" % struct_token.id)
|
||||
|
||||
# Search forward for the end of the structure.
|
||||
# Very simple search, look for } and ; tokens. If something
|
||||
# more complicated is needed we can add it later.
|
||||
# Very simple search, look for } and ; tokens.
|
||||
# If we hit the end of the block, we'll need to start
|
||||
# looking at the next block.
|
||||
j = i + 3
|
||||
while j < num_tokens - 1:
|
||||
if (b.tokens[j].kind == TokenKind.PUNCTUATION and
|
||||
b.tokens[j].id == "}" and
|
||||
b.tokens[j + 1].kind == TokenKind.PUNCTUATION and
|
||||
b.tokens[j + 1].id == ";"):
|
||||
b.tokens = b.tokens[0:i] + b.tokens[j + 2:num_tokens]
|
||||
depth = 1
|
||||
struct_removed = False
|
||||
while not struct_removed:
|
||||
while j < num_tokens:
|
||||
if b.tokens[j].kind == TokenKind.PUNCTUATION:
|
||||
if b.tokens[j].id == '{':
|
||||
depth += 1
|
||||
elif b.tokens[j].id == '}':
|
||||
depth -= 1
|
||||
elif b.tokens[j].id == ';' and depth == 0:
|
||||
b.tokens = b.tokens[0:i] + b.tokens[j + 1:num_tokens]
|
||||
num_tokens = len(b.tokens)
|
||||
struct_removed = True
|
||||
break
|
||||
j += 1
|
||||
if not struct_removed:
|
||||
b.tokens = b.tokens[0:i]
|
||||
|
||||
# Skip directive blocks.
|
||||
start_block = block_num
|
||||
while block_num < num_blocks:
|
||||
if not self.blocks[block_num].directive:
|
||||
break
|
||||
block_num += 1
|
||||
if block_num >= num_blocks:
|
||||
# Unparsable struct, error out.
|
||||
raise UnparseableStruct("Cannot remove struct %s: %s" % (struct_token.id, struct_token.location))
|
||||
self.blocks = self.blocks[0:start_block] + self.blocks[block_num:num_blocks]
|
||||
num_blocks = len(self.blocks)
|
||||
b = self.blocks[start_block]
|
||||
block_num = start_block + 1
|
||||
num_tokens = len(b.tokens)
|
||||
j = i
|
||||
break
|
||||
j += 1
|
||||
i = j
|
||||
i = 0
|
||||
j = 0
|
||||
continue
|
||||
i += 1
|
||||
|
||||
for extra_include in extra_includes:
|
||||
replacement = CppStringTokenizer(extra_include)
|
||||
self.blocks.insert(2, Block(replacement.tokens, directive='include'))
|
||||
|
||||
def optimizeAll(self, macros):
|
||||
self.optimizeMacros(macros)
|
||||
self.optimizeIf01()
|
||||
|
|
@ -1404,35 +1459,12 @@ class BlockList(object):
|
|||
|
||||
def replaceTokens(self, replacements):
|
||||
"""Replace tokens according to the given dict."""
|
||||
extra_includes = []
|
||||
for b in self.blocks:
|
||||
made_change = False
|
||||
if b.isInclude() is None:
|
||||
i = 0
|
||||
while i < len(b.tokens):
|
||||
tok = b.tokens[i]
|
||||
if (tok.kind == TokenKind.KEYWORD and tok.id == 'struct'
|
||||
and (i + 2) < len(b.tokens) and b.tokens[i + 2].id == '{'):
|
||||
struct_name = b.tokens[i + 1].id
|
||||
if struct_name in kernel_struct_replacements:
|
||||
extra_includes.append("<bits/%s.h>" % struct_name)
|
||||
end = i + 2
|
||||
depth = 1
|
||||
while end < len(b.tokens) and depth > 0:
|
||||
if b.tokens[end].id == '}':
|
||||
depth -= 1
|
||||
elif b.tokens[end].id == '{':
|
||||
depth += 1
|
||||
end += 1
|
||||
end += 1 # Swallow last '}'
|
||||
while end < len(b.tokens) and b.tokens[end].id != ';':
|
||||
end += 1
|
||||
end += 1 # Swallow ';'
|
||||
# Remove these tokens. We'll replace them later with a #include block.
|
||||
b.tokens[i:end] = []
|
||||
made_change = True
|
||||
# We've just modified b.tokens, so revisit the current offset.
|
||||
continue
|
||||
if tok.kind == TokenKind.IDENTIFIER:
|
||||
if tok.id in replacements:
|
||||
tok.id = replacements[tok.id]
|
||||
|
|
@ -1447,10 +1479,6 @@ class BlockList(object):
|
|||
# Keep 'expr' in sync with 'tokens'.
|
||||
b.expr = CppExpr(b.tokens)
|
||||
|
||||
for extra_include in extra_includes:
|
||||
replacement = CppStringTokenizer(extra_include)
|
||||
self.blocks.insert(2, Block(replacement.tokens, directive='include'))
|
||||
|
||||
|
||||
|
||||
def strip_space(s):
|
||||
|
|
@ -2020,7 +2048,7 @@ struct something {
|
|||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
self.assertEqual(self.parse(text, set(["remove"])), expected)
|
||||
self.assertEqual(self.parse(text, {"remove": True}), expected)
|
||||
|
||||
def test_remove_struct_from_end(self):
|
||||
text = """\
|
||||
|
|
@ -2039,7 +2067,7 @@ struct something {
|
|||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
self.assertEqual(self.parse(text, set(["remove"])), expected)
|
||||
self.assertEqual(self.parse(text, {"remove": True}), expected)
|
||||
|
||||
def test_remove_minimal_struct(self):
|
||||
text = """\
|
||||
|
|
@ -2047,7 +2075,7 @@ struct remove {
|
|||
};
|
||||
"""
|
||||
expected = "";
|
||||
self.assertEqual(self.parse(text, set(["remove"])), expected)
|
||||
self.assertEqual(self.parse(text, {"remove": True}), expected)
|
||||
|
||||
def test_remove_struct_with_struct_fields(self):
|
||||
text = """\
|
||||
|
|
@ -2067,7 +2095,7 @@ struct something {
|
|||
struct remove val2;
|
||||
};
|
||||
"""
|
||||
self.assertEqual(self.parse(text, set(["remove"])), expected)
|
||||
self.assertEqual(self.parse(text, {"remove": True}), expected)
|
||||
|
||||
def test_remove_consecutive_structs(self):
|
||||
text = """\
|
||||
|
|
@ -2099,7 +2127,7 @@ struct keep2 {
|
|||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
self.assertEqual(self.parse(text, set(["remove1", "remove2"])), expected)
|
||||
self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
|
||||
|
||||
def test_remove_multiple_structs(self):
|
||||
text = """\
|
||||
|
|
@ -2132,7 +2160,101 @@ struct keep3 {
|
|||
int val;
|
||||
};
|
||||
"""
|
||||
self.assertEqual(self.parse(text, set(["remove1", "remove2"])), expected)
|
||||
self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
|
||||
|
||||
def test_remove_struct_with_inline_structs(self):
|
||||
text = """\
|
||||
struct remove {
|
||||
int val1;
|
||||
int val2;
|
||||
struct {
|
||||
int val1;
|
||||
struct {
|
||||
int val1;
|
||||
} level2;
|
||||
} level1;
|
||||
};
|
||||
struct something {
|
||||
struct timeval val1;
|
||||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
expected = """\
|
||||
struct something {
|
||||
struct timeval val1;
|
||||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
self.assertEqual(self.parse(text, {"remove": True}), expected)
|
||||
|
||||
def test_remove_struct_across_blocks(self):
|
||||
text = """\
|
||||
struct remove {
|
||||
int val1;
|
||||
int val2;
|
||||
#ifdef PARAMETER1
|
||||
PARAMETER1
|
||||
#endif
|
||||
#ifdef PARAMETER2
|
||||
PARAMETER2
|
||||
#endif
|
||||
};
|
||||
struct something {
|
||||
struct timeval val1;
|
||||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
expected = """\
|
||||
struct something {
|
||||
struct timeval val1;
|
||||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
self.assertEqual(self.parse(text, {"remove": True}), expected)
|
||||
|
||||
def test_remove_struct_across_blocks_multiple_structs(self):
|
||||
text = """\
|
||||
struct remove1 {
|
||||
int val1;
|
||||
int val2;
|
||||
#ifdef PARAMETER1
|
||||
PARAMETER1
|
||||
#endif
|
||||
#ifdef PARAMETER2
|
||||
PARAMETER2
|
||||
#endif
|
||||
};
|
||||
struct remove2 {
|
||||
};
|
||||
struct something {
|
||||
struct timeval val1;
|
||||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
expected = """\
|
||||
struct something {
|
||||
struct timeval val1;
|
||||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
|
||||
|
||||
def test_remove_multiple_struct_and_add_includes(self):
|
||||
text = """\
|
||||
struct remove1 {
|
||||
int val1;
|
||||
int val2;
|
||||
};
|
||||
struct remove2 {
|
||||
struct timeval val1;
|
||||
struct timeval val2;
|
||||
};
|
||||
"""
|
||||
expected = """\
|
||||
#include <bits/remove1.h>
|
||||
#include <bits/remove2.h>
|
||||
"""
|
||||
self.assertEqual(self.parse(text, {"remove1": False, "remove2": False}), expected)
|
||||
|
||||
|
||||
class FullPathTest(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -35,16 +35,23 @@ kernel_known_macros = {
|
|||
"__kernel_old_timeval": "1",
|
||||
}
|
||||
|
||||
# this is the set of known kernel data structures we want to remove from
|
||||
# the final headers
|
||||
kernel_structs_to_remove = set(
|
||||
[
|
||||
# Remove the structures since they are still the same as
|
||||
# timeval, itimerval.
|
||||
"__kernel_old_timeval",
|
||||
"__kernel_old_itimerval",
|
||||
]
|
||||
)
|
||||
# This is the set of known kernel data structures we want to remove from
|
||||
# the final headers. If the map value is False, that means that in
|
||||
# addition to removing the structure, add an #include <bits/STRUCT.h>
|
||||
# to the file.
|
||||
kernel_structs_to_remove = {
|
||||
# Remove the structures since they are still the same as
|
||||
# timeval, itimerval.
|
||||
"__kernel_old_timeval": True,
|
||||
"__kernel_old_itimerval": True,
|
||||
# Replace all of the below structures with #include <bits/STRUCT.h>
|
||||
"epoll_event": False,
|
||||
"flock": False,
|
||||
"flock64": False,
|
||||
"in_addr": False,
|
||||
"ip_mreq_source": False,
|
||||
"ip_msfilter": False,
|
||||
}
|
||||
|
||||
# define to true if you want to remove all defined(CONFIG_FOO) tests
|
||||
# from the clean headers. testing shows that this is not strictly necessary
|
||||
|
|
@ -100,20 +107,6 @@ kernel_token_replacements = {
|
|||
}
|
||||
|
||||
|
||||
# This is the set of struct definitions that we want to replace with
|
||||
# a #include of <bits/struct.h> instead.
|
||||
kernel_struct_replacements = set(
|
||||
[
|
||||
"epoll_event",
|
||||
"flock",
|
||||
"flock64",
|
||||
"in_addr",
|
||||
"ip_mreq_source",
|
||||
"ip_msfilter",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# This is the set of known static inline functions that we want to keep
|
||||
# in the final kernel headers.
|
||||
kernel_known_generic_statics = set(
|
||||
|
|
|
|||
|
|
@ -27,8 +27,14 @@ def Usage():
|
|||
def ProcessFiles(updater, original_dir, modified_dir, src_rel_dir, update_rel_dir):
|
||||
# Delete the old headers before updating to the new headers.
|
||||
update_dir = os.path.join(get_kernel_dir(), update_rel_dir)
|
||||
shutil.rmtree(update_dir)
|
||||
os.mkdir(update_dir, 0o755)
|
||||
for root, dirs, files in os.walk(update_dir, topdown=True):
|
||||
for entry in files:
|
||||
# BUILD is a special file that needs to be preserved.
|
||||
if entry == "BUILD":
|
||||
continue
|
||||
os.remove(os.path.join(root, entry))
|
||||
for entry in dirs:
|
||||
shutil.rmtree(os.path.join(root, entry))
|
||||
|
||||
src_dir = os.path.normpath(os.path.join(original_dir, src_rel_dir))
|
||||
src_dir_len = len(src_dir) + 1
|
||||
|
|
|
|||
Loading…
Reference in a new issue