Merge "Add support for removing structs that cross blocks." am: 0ba4b34206

Original change: https://android-review.googlesource.com/c/platform/bionic/+/2175190

Change-Id: I94abfec1aa97a9dce5d3537ebd8f1a49725ec4c8
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

libc/kernel/tools/cpp.py

@@ -155,6 +155,11 @@ class BadExpectedToken(Exception):
     pass
 
 
+class UnparseableStruct(Exception):
+    """An exception that will be raised for structs that cannot be parsed."""
+    pass
+
+
 # The __contains__ function in libclang SourceRange class contains a bug. It
 # gives wrong result when dealing with single line range.
 # Bug filed with upstream:
@@ -1197,22 +1202,38 @@ class BlockList(object):
 
     def removeStructs(self, structs):
         """Remove structs."""
-        for b in self.blocks:
+        extra_includes = []
+        block_num = 0
+        num_blocks = len(self.blocks)
+        while block_num < num_blocks:
+            b = self.blocks[block_num]
+            block_num += 1
             # Have to look in each block for a top-level struct definition.
             if b.directive:
                 continue
             num_tokens = len(b.tokens)
-            # A struct definition has at least 5 tokens:
+            # A struct definition usually looks like:
             #   struct
             #   ident
             #   {
             #   }
             #   ;
-            if num_tokens < 5:
+            # However, the structure might be spread across multiple blocks
+            # if the structure looks like this:
+            #   struct ident
+            #   {
+            #   #ifdef VARIABLE
+            #     pid_t pid;
+            #   #endif
+            #   }:
+            # So the total number of tokens in the block might be less than
+            # five but assume at least three.
+            if num_tokens < 3:
                 continue
+
             # This is a simple struct finder, it might fail if a top-level
             # structure has an #if type directives that confuses the algorithm
-            # for finding th end of the structure. Or if there is another
+            # for finding the end of the structure. Or if there is another
             # structure definition embedded in the structure.
             i = 0
             while i < num_tokens - 2:
@@ -1223,24 +1244,58 @@ class BlockList(object):
                 if (b.tokens[i + 1].kind == TokenKind.IDENTIFIER and
                     b.tokens[i + 2].kind == TokenKind.PUNCTUATION and
                     b.tokens[i + 2].id == "{" and b.tokens[i + 1].id in structs):
+                    # Add an include for the structure to be removed of the form:
+                    #  #include <bits/STRUCT_NAME.h>
+                    struct_token = b.tokens[i + 1]
+                    if not structs[struct_token.id]:
+                        extra_includes.append("<bits/%s.h>" % struct_token.id)
+
                     # Search forward for the end of the structure.
-                    # Very simple search, look for } and ; tokens. If something
+                    # Very simple search, look for } and ; tokens.
-                    # more complicated is needed we can add it later.
+                    # If we hit the end of the block, we'll need to start
+                    # looking at the next block.
                     j = i + 3
-                    while j < num_tokens - 1:
+                    depth = 1
-                        if (b.tokens[j].kind == TokenKind.PUNCTUATION and
+                    struct_removed = False
-                            b.tokens[j].id == "}" and
+                    while not struct_removed:
-                            b.tokens[j + 1].kind == TokenKind.PUNCTUATION and
+                        while j < num_tokens:
-                            b.tokens[j + 1].id == ";"):
+                            if b.tokens[j].kind == TokenKind.PUNCTUATION:
-                            b.tokens = b.tokens[0:i] + b.tokens[j + 2:num_tokens]
+                                if b.tokens[j].id == '{':
+                                    depth += 1
+                                elif b.tokens[j].id == '}':
+                                    depth -= 1
+                                elif b.tokens[j].id == ';' and depth == 0:
+                                    b.tokens = b.tokens[0:i] + b.tokens[j + 1:num_tokens]
                                     num_tokens = len(b.tokens)
-                            j = i
+                                    struct_removed = True
                                     break
                             j += 1
-                    i = j
+                        if not struct_removed:
+                            b.tokens = b.tokens[0:i]
+
+                            # Skip directive blocks.
+                            start_block = block_num
+                            while block_num < num_blocks:
+                                if not self.blocks[block_num].directive:
+                                    break
+                                block_num += 1
+                            if block_num >= num_blocks:
+                                # Unparsable struct, error out.
+                                raise UnparseableStruct("Cannot remove struct %s: %s" % (struct_token.id, struct_token.location))
+                            self.blocks = self.blocks[0:start_block] + self.blocks[block_num:num_blocks]
+                            num_blocks = len(self.blocks)
+                            b = self.blocks[start_block]
+                            block_num = start_block + 1
+                            num_tokens = len(b.tokens)
+                            i = 0
+                            j = 0
                     continue
                 i += 1
 
+        for extra_include in extra_includes:
+            replacement = CppStringTokenizer(extra_include)
+            self.blocks.insert(2, Block(replacement.tokens, directive='include'))
+
     def optimizeAll(self, macros):
         self.optimizeMacros(macros)
         self.optimizeIf01()
@@ -1404,35 +1459,12 @@ class BlockList(object):
 
     def replaceTokens(self, replacements):
         """Replace tokens according to the given dict."""
-        extra_includes = []
         for b in self.blocks:
             made_change = False
             if b.isInclude() is None:
                 i = 0
                 while i < len(b.tokens):
                     tok = b.tokens[i]
-                    if (tok.kind == TokenKind.KEYWORD and tok.id == 'struct'
-                        and (i + 2) < len(b.tokens) and b.tokens[i + 2].id == '{'):
-                        struct_name = b.tokens[i + 1].id
-                        if struct_name in kernel_struct_replacements:
-                            extra_includes.append("<bits/%s.h>" % struct_name)
-                            end = i + 2
-                            depth = 1
-                            while end < len(b.tokens) and depth > 0:
-                                if b.tokens[end].id == '}':
-                                    depth -= 1
-                                elif b.tokens[end].id == '{':
-                                    depth += 1
-                                end += 1
-                            end += 1 # Swallow last '}'
-                            while end < len(b.tokens) and b.tokens[end].id != ';':
-                                end += 1
-                            end += 1 # Swallow ';'
-                            # Remove these tokens. We'll replace them later with a #include block.
-                            b.tokens[i:end] = []
-                            made_change = True
-                            # We've just modified b.tokens, so revisit the current offset.
-                            continue
                     if tok.kind == TokenKind.IDENTIFIER:
                         if tok.id in replacements:
                             tok.id = replacements[tok.id]
@@ -1447,10 +1479,6 @@ class BlockList(object):
                 # Keep 'expr' in sync with 'tokens'.
                 b.expr = CppExpr(b.tokens)
 
-        for extra_include in extra_includes:
-            replacement = CppStringTokenizer(extra_include)
-            self.blocks.insert(2, Block(replacement.tokens, directive='include'))
-
 
 
 def strip_space(s):
@@ -2020,7 +2048,7 @@ struct something {
   struct timeval val2;
 };
 """
-        self.assertEqual(self.parse(text, set(["remove"])), expected)
+        self.assertEqual(self.parse(text, {"remove": True}), expected)
 
     def test_remove_struct_from_end(self):
         text = """\
@@ -2039,7 +2067,7 @@ struct something {
   struct timeval val2;
 };
 """
-        self.assertEqual(self.parse(text, set(["remove"])), expected)
+        self.assertEqual(self.parse(text, {"remove": True}), expected)
 
     def test_remove_minimal_struct(self):
         text = """\
@@ -2047,7 +2075,7 @@ struct remove {
 };
 """
         expected = "";
-        self.assertEqual(self.parse(text, set(["remove"])), expected)
+        self.assertEqual(self.parse(text, {"remove": True}), expected)
 
     def test_remove_struct_with_struct_fields(self):
         text = """\
@@ -2067,7 +2095,7 @@ struct something {
   struct remove val2;
 };
 """
-        self.assertEqual(self.parse(text, set(["remove"])), expected)
+        self.assertEqual(self.parse(text, {"remove": True}), expected)
 
     def test_remove_consecutive_structs(self):
         text = """\
@@ -2099,7 +2127,7 @@ struct keep2 {
   struct timeval val2;
 };
 """
-        self.assertEqual(self.parse(text, set(["remove1", "remove2"])), expected)
+        self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
 
     def test_remove_multiple_structs(self):
         text = """\
@@ -2132,7 +2160,101 @@ struct keep3 {
   int val;
 };
 """
-        self.assertEqual(self.parse(text, set(["remove1", "remove2"])), expected)
+        self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
+
+    def test_remove_struct_with_inline_structs(self):
+        text = """\
+struct remove {
+  int val1;
+  int val2;
+  struct {
+    int val1;
+    struct {
+      int val1;
+    } level2;
+  } level1;
+};
+struct something {
+  struct timeval val1;
+  struct timeval val2;
+};
+"""
+        expected = """\
+struct something {
+  struct timeval val1;
+  struct timeval val2;
+};
+"""
+        self.assertEqual(self.parse(text, {"remove": True}), expected)
+
+    def test_remove_struct_across_blocks(self):
+        text = """\
+struct remove {
+  int val1;
+  int val2;
+#ifdef PARAMETER1
+  PARAMETER1
+#endif
+#ifdef PARAMETER2
+  PARAMETER2
+#endif
+};
+struct something {
+  struct timeval val1;
+  struct timeval val2;
+};
+"""
+        expected = """\
+struct something {
+  struct timeval val1;
+  struct timeval val2;
+};
+"""
+        self.assertEqual(self.parse(text, {"remove": True}), expected)
+
+    def test_remove_struct_across_blocks_multiple_structs(self):
+        text = """\
+struct remove1 {
+  int val1;
+  int val2;
+#ifdef PARAMETER1
+  PARAMETER1
+#endif
+#ifdef PARAMETER2
+  PARAMETER2
+#endif
+};
+struct remove2 {
+};
+struct something {
+  struct timeval val1;
+  struct timeval val2;
+};
+"""
+        expected = """\
+struct something {
+  struct timeval val1;
+  struct timeval val2;
+};
+"""
+        self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
+
+    def test_remove_multiple_struct_and_add_includes(self):
+        text = """\
+struct remove1 {
+  int val1;
+  int val2;
+};
+struct remove2 {
+  struct timeval val1;
+  struct timeval val2;
+};
+"""
+        expected = """\
+#include <bits/remove1.h>
+#include <bits/remove2.h>
+"""
+        self.assertEqual(self.parse(text, {"remove1": False, "remove2": False}), expected)
 
 
 class FullPathTest(unittest.TestCase):

libc/kernel/tools/defaults.py

@@ -35,16 +35,23 @@ kernel_known_macros = {
     "__kernel_old_timeval": "1",
     }
 
-# this is the set of known kernel data structures we want to remove from
+# This is the set of known kernel data structures we want to remove from
-# the final headers
+# the final headers. If the map value is False, that means that in
-kernel_structs_to_remove = set(
+# addition to removing the structure, add an #include <bits/STRUCT.h>
-        [
+# to the file.
+kernel_structs_to_remove = {
     # Remove the structures since they are still the same as
     # timeval, itimerval.
-          "__kernel_old_timeval",
+    "__kernel_old_timeval": True,
-          "__kernel_old_itimerval",
+    "__kernel_old_itimerval": True,
-        ]
+    # Replace all of the below structures with #include <bits/STRUCT.h>
-    )
+    "epoll_event": False,
+    "flock": False,
+    "flock64": False,
+    "in_addr": False,
+    "ip_mreq_source": False,
+    "ip_msfilter": False,
+    }
 
 # define to true if you want to remove all defined(CONFIG_FOO) tests
 # from the clean headers. testing shows that this is not strictly necessary
@@ -100,20 +107,6 @@ kernel_token_replacements = {
     }
 
 
-# This is the set of struct definitions that we want to replace with
-# a #include of <bits/struct.h> instead.
-kernel_struct_replacements = set(
-        [
-          "epoll_event",
-          "flock",
-          "flock64",
-          "in_addr",
-          "ip_mreq_source",
-          "ip_msfilter",
-        ]
-    )
-
-
 # This is the set of known static inline functions that we want to keep
 # in the final kernel headers.
 kernel_known_generic_statics = set(

libc/kernel/tools/update_all.py

@@ -27,8 +27,14 @@ def Usage():
 def ProcessFiles(updater, original_dir, modified_dir, src_rel_dir, update_rel_dir):
     # Delete the old headers before updating to the new headers.
     update_dir = os.path.join(get_kernel_dir(), update_rel_dir)
-    shutil.rmtree(update_dir)
+    for root, dirs, files in os.walk(update_dir, topdown=True):
-    os.mkdir(update_dir, 0o755)
+        for entry in files:
+            # BUILD is a special file that needs to be preserved.
+            if entry == "BUILD":
+                continue
+            os.remove(os.path.join(root, entry))
+        for entry in dirs:
+            shutil.rmtree(os.path.join(root, entry))
 
     src_dir = os.path.normpath(os.path.join(original_dir, src_rel_dir))
     src_dir_len = len(src_dir) + 1