From d57789fde83d97eaf8ab0b973afd08ffaa93d883 Mon Sep 17 00:00:00 2001 From: Tri Vo Date: Mon, 17 Dec 2018 16:35:41 -0800 Subject: [PATCH] Replace "grep -f" with python util. grep can potentially run out of memory on Mac builds for large input files. So we add a python util to handle filtering out files. We will also need this util to filter plat_sepolicy.cil out of product_sepolicy.cil Bug: 119305624 Test: boot aosp_taimen Change-Id: I61cd68f407ea5de43a06bf522a5fc149e5067e8c --- Android.mk | 16 ++++++++++------ build/build_sepolicy.py | 20 +++++++++++++++++++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/Android.mk b/Android.mk index fcf9d9ac7..36009d30a 100644 --- a/Android.mk +++ b/Android.mk @@ -392,10 +392,12 @@ $(PLAT_PUBLIC_POLICY) $(REQD_MASK_POLICY)) plat_pub_policy.cil := $(intermediates)/plat_pub_policy.cil $(plat_pub_policy.cil): PRIVATE_POL_CONF := $(plat_pub_policy.conf) $(plat_pub_policy.cil): PRIVATE_REQD_MASK := $(reqd_policy_mask.cil) -$(plat_pub_policy.cil): $(HOST_OUT_EXECUTABLES)/checkpolicy $(plat_pub_policy.conf) $(reqd_policy_mask.cil) +$(plat_pub_policy.cil): $(HOST_OUT_EXECUTABLES)/checkpolicy \ +$(HOST_OUT_EXECUTABLES)/build_sepolicy $(plat_pub_policy.conf) $(reqd_policy_mask.cil) @mkdir -p $(dir $@) - $(hide) $(CHECKPOLICY_ASAN_OPTIONS) $< -C -M -c $(POLICYVERS) -o $@.tmp $(PRIVATE_POL_CONF) - $(hide) grep -Fxv -f $(PRIVATE_REQD_MASK) $@.tmp > $@ + $(hide) $(CHECKPOLICY_ASAN_OPTIONS) $< -C -M -c $(POLICYVERS) -o $@ $(PRIVATE_POL_CONF) + $(hide) $(HOST_OUT_EXECUTABLES)/build_sepolicy -a $(HOST_OUT_EXECUTABLES) filter_out \ + -f $(PRIVATE_REQD_MASK) -t $@ plat_pub_policy.conf := @@ -1641,10 +1643,12 @@ $(BASE_PLAT_PUBLIC_POLICY) $(REQD_MASK_POLICY)) base_plat_pub_policy.cil := $(intermediates)/base_plat_pub_policy.cil $(base_plat_pub_policy.cil): PRIVATE_POL_CONF := $(base_plat_pub_policy.conf) $(base_plat_pub_policy.cil): PRIVATE_REQD_MASK := $(reqd_policy_mask.cil) -$(base_plat_pub_policy.cil): $(HOST_OUT_EXECUTABLES)/checkpolicy $(base_plat_pub_policy.conf) $(reqd_policy_mask.cil) +$(base_plat_pub_policy.cil): $(HOST_OUT_EXECUTABLES)/checkpolicy \ +$(HOST_OUT_EXECUTABLES)/build_sepolicy $(base_plat_pub_policy.conf) $(reqd_policy_mask.cil) @mkdir -p $(dir $@) - $(hide) $(CHECKPOLICY_ASAN_OPTIONS) $< -C -M -c $(POLICYVERS) -o $@.tmp $(PRIVATE_POL_CONF) - $(hide) grep -Fxv -f $(PRIVATE_REQD_MASK) $@.tmp > $@ + $(hide) $(CHECKPOLICY_ASAN_OPTIONS) $< -C -M -c $(POLICYVERS) -o $@ $(PRIVATE_POL_CONF) + $(hide) $(HOST_OUT_EXECUTABLES)/build_sepolicy -a $(HOST_OUT_EXECUTABLES) filter_out \ + -f $(PRIVATE_REQD_MASK) -t $@ all_fc_files := $(built_plat_fc) $(built_vendor_fc) ifdef BOARD_ODM_SEPOLICY_DIRS diff --git a/build/build_sepolicy.py b/build/build_sepolicy.py index ff2ff075d..285bfea52 100644 --- a/build/build_sepolicy.py +++ b/build/build_sepolicy.py @@ -27,7 +27,7 @@ import file_utils # - setup_build_cil() # - Sets up command parsers and sets default function to do_build_cil(). # - do_build_cil() -_SUPPORTED_COMMANDS = ('build_cil',) +_SUPPORTED_COMMANDS = ('build_cil', 'filter_out') def run_host_command(args, **kwargs): @@ -119,6 +119,24 @@ def setup_build_cil(subparsers): parser.set_defaults(func=do_build_cil) +def do_filter_out(args): + """Removes all lines in one file that match any line in another file. + + Args: + args: the parsed command arguments. + """ + file_utils.filter_out(args.filter_out_files, args.target_file) + +def setup_filter_out(subparsers): + """Sets up command args for 'filter_out' command.""" + parser = subparsers.add_parser('filter_out', help='filter CIL files') + parser.add_argument('-f', '--filter_out_files', required=True, nargs='+', + help='the pattern files to filter out the output cil') + parser.add_argument('-t', '--target_file', required=True, + help='target file to filter') + parser.set_defaults(func=do_filter_out) + + def run(argv): """Sets up command parser and execuates sub-command.""" parser = argparse.ArgumentParser()