Merge "Add support for manually modified kernel headers."

libc/kernel/tools/clean_header.py

@@ -73,90 +73,77 @@ import sys, cpp, kernel, glob, os, re, getopt
 from defaults import *
 from utils import *
 
-noUpdate = 1
+def print_error(no_update, msg):
+    if no_update:
+        panic(msg)
+    sys.stderr.write("warning: " + msg)
 
-def cleanupFile(path, original_path):
+
+def cleanupFile(dst_dir, src_dir, rel_path, no_update = True):
     """reads an original header and perform the cleanup operation on it
        this functions returns the destination path and the clean header
        as a single string"""
     # check the header path
-    src_path = path
+    full_path = os.path.join(src_dir, rel_path)
 
-    if not os.path.exists(src_path):
+    if not os.path.exists(full_path):
-        if noUpdate:
+        print_error(no_update, "file does not exist: '%s'\n" % full_path)
-            panic( "file does not exist: '%s'\n" % path )
-        sys.stderr.write( "warning: file does not exit: %s\n" % path )
         return None, None
 
-    if not os.path.isfile(src_path):
+    if not os.path.isfile(full_path):
-        if noUpdate:
+        print_error(no_update, "path is not a file: '%s'\n" % full_path)
-            panic( "path is not a file: '%s'\n" % path )
-        sys.stderr.write( "warning: not a file: %s\n" % path )
         return None, None
 
-    if os.path.commonprefix( [ src_path, original_path ] ) != original_path:
-        if noUpdate:
-            panic( "file is not in 'original' directory: %s\n" % path );
-        sys.stderr.write( "warning: file not in 'original' ignored: %s\n" % path )
-        return None, None
-
-    src_path = src_path[len(original_path):]
-    if len(src_path) > 0 and src_path[0] == '/':
-        src_path = src_path[1:]
-
-    if len(src_path) == 0:
-        panic( "oops, internal error, can't extract correct relative path\n" )
-
     # convert into destination path, extracting architecture if needed
     # and the corresponding list of known static functions
     #
     arch = None
     statics = kernel_known_generic_statics
-    m = re.match(r"asm-([\w\d_\+\.\-]+)(/.*)", src_path)
+    m = re.match(r"asm-([\w\d_\+\.\-]+)(/.*)", rel_path)
     if m and m.group(1) != 'generic':
         dst_path = "arch-%s/asm/%s" % m.groups()
-        arch     = m.group(1)
+        arch = m.group(1)
-        statics  = statics.union( kernel_known_statics.get( arch, set() ) )
+        statics  = statics.union(kernel_known_statics.get(arch, set()))
     else:
         # process headers under the uapi directory
         # note the "asm" level has been explicitly added in the original
         # kernel header tree for architectural-dependent uapi headers
-        m_uapi = re.match(r"(uapi)/([\w\d_\+\.\-]+)(/.*)", src_path)
+        m_uapi = re.match(r"(uapi)/([\w\d_\+\.\-]+)(/.*)", rel_path)
         if m_uapi:
-            dst_path = src_path
+            dst_path = rel_path
             m_uapi_arch = re.match(r"asm-([\w\d_\+\.\-]+)", m_uapi.group(2))
             if m_uapi_arch and m_uapi_arch.group(1) != 'generic':
-                arch     = m_uapi_arch.group(1)
+                arch = m_uapi_arch.group(1)
-                statics  = statics.union( kernel_known_statics.get( arch, set() ) )
+                statics = statics.union(kernel_known_statics.get(arch, set()))
         # common headers (ie non-asm and non-uapi)
         else:
-            dst_path = "common/" + src_path
+            dst_path = os.path.join("common", rel_path)
 
-    dst_path = os.path.normpath( kernel_cleaned_path + "/" + dst_path )
+    dst_path = os.path.join(dst_dir, dst_path)
 
     # now, let's parse the file
     #
     parser = cpp.BlockParser()
-    blocks = parser.parseFile(path)
+    blocks = parser.parseFile(full_path)
     if not parser.parsed:
-        sys.stderr.write( "error: can't parse '%s'" % path )
+        print_error(no_update, "can't parse '%s'%" % full_path)
-        sys.exit(1)
+        return None, None
 
     macros = kernel_known_macros.copy()
     if arch and arch in kernel_default_arch_macros:
         macros.update(kernel_default_arch_macros[arch])
 
     if arch and arch in kernel_arch_token_replacements:
-        blocks.replaceTokens( kernel_arch_token_replacements[arch] )
+        blocks.replaceTokens(kernel_arch_token_replacements[arch])
 
-    blocks.optimizeMacros( macros )
+    blocks.optimizeMacros(macros)
     blocks.optimizeIf01()
-    blocks.removeVarsAndFuncs( statics )
+    blocks.removeVarsAndFuncs(statics)
-    blocks.replaceTokens( kernel_token_replacements )
+    blocks.replaceTokens(kernel_token_replacements)
-    blocks.removeMacroDefines( kernel_ignored_macros )
+    blocks.removeMacroDefines(kernel_ignored_macros)
 
     out = StringOutput()
-    out.write( kernel_disclaimer )
+    out.write(kernel_disclaimer)
     blocks.writeWithWarning(out, kernel_warning, 4)
     return dst_path, out.get()
 
@@ -183,28 +170,31 @@ if __name__ == "__main__":
         sys.exit(1)
 
     try:
-        optlist, args = getopt.getopt( sys.argv[1:], 'uvk:d:' )
+        optlist, args = getopt.getopt(sys.argv[1:], 'uvk:d:')
     except:
         # unrecognized option
-        sys.stderr.write( "error: unrecognized option\n" )
+        sys.stderr.write("error: unrecognized option\n")
         usage()
 
+    no_update = True
+    dst_dir = get_kernel_dir()
+    src_dir = get_kernel_headers_original_dir()
     for opt, arg in optlist:
         if opt == '-u':
-            noUpdate = 0
+            no_update = False
         elif opt == '-v':
             logging.basicConfig(level=logging.DEBUG)
         elif opt == '-k':
-            kernel_original_path = arg
+            src_dir = arg
         elif opt == '-d':
-            kernel_cleaned_path = arg
+            dst_dir = arg
 
     if len(args) == 0:
         usage()
 
-    if noUpdate:
+    if no_update:
         for path in args:
-            dst_path, newdata = cleanupFile(path,kernel_original_path)
+            dst_path, newdata = cleanupFile(dst_dir, src_dir, path)
             print newdata
 
         sys.exit(0)
@@ -214,12 +204,12 @@ if __name__ == "__main__":
     b = BatchFileUpdater()
 
     for path in args:
-        dst_path, newdata = cleanupFile(path,kernel_original_path)
+        dst_path, newdata = cleanupFile(dst_dir, src_dir, path, no_update)
         if not dst_path:
             continue
 
-        b.readFile( dst_path )
+        b.readFile(dst_path)
-        r = b.editFile( dst_path, newdata )
+        r = b.editFile(dst_path, newdata)
         if r == 0:
             r = "unchanged"
         elif r == 1:
@@ -227,7 +217,7 @@ if __name__ == "__main__":
         else:
             r = "added"
 
-        print "cleaning: %-*s -> %-*s (%s)" % ( 35, path, 35, dst_path, r )
+        print "cleaning: %-*s -> %-*s (%s)" % (35, path, 35, dst_path, r)
 
 
     b.updateGitFiles()

libc/kernel/tools/defaults.py

@@ -12,12 +12,6 @@ kernel_archs = [ 'arm', 'arm64', 'mips', 'x86' ]
 # tree. used when looking for sources...
 kernel_dirs = [ "linux", "asm", "asm-generic", "mtd" ]
 
-# path to the directory containing the original kernel headers
-kernel_original_path = os.path.normpath( find_program_dir() + '/../../../../external/kernel-headers/original' )
-
-# path to the default location of the cleaned-up headers
-kernel_cleaned_path = os.path.normpath( find_program_dir() + '/..' )
-
 # a special value that is used to indicate that a given macro is known to be
 # undefined during optimization
 kCppUndefinedMacro = "<<<undefined>>>"

libc/kernel/tools/generate_uapi_headers.sh

@@ -99,6 +99,35 @@ function copy_if_exists () {
   done
 }
 
+function check_hdrs () {
+  local src_dir=$1
+  local tgt_dir=$2
+  local kernel_dir=$3
+
+  local search_dirs=()
+
+  # This only works if none of the filenames have spaces.
+  for file in $(ls -d ${src_dir}/* 2> /dev/null); do
+    if [[ -d "${file}" ]]; then
+      search_dirs+=("${file}")
+    elif [[ -f  "${file}" ]] && [[ "${file}" =~ .h$ ]]; then
+      tgt_file=${tgt_dir}/$(basename ${file})
+      if [[ -e ${tgt_file} ]] && ! diff "${file}" "${tgt_file}" > /dev/null; then
+        if [[ ${file} =~ ${kernel_dir}/*(.+) ]]; then
+          echo "New version of ${BASH_REMATCH[1]} found in kernel headers."
+        else
+          echo "New version of ${file} found in kernel headers."
+        fi
+        echo "This file needs to be updated manually."
+      fi
+    fi
+  done
+
+  for dir in "${search_dirs[@]}"; do
+    check_hdrs "${dir}" ${tgt_dir}/$(basename ${dir}) "${kernel_dir}"
+  done
+}
+
 trap cleanup EXIT
 # This automatically triggers a call to cleanup.
 trap "exit 1" HUP INT TERM TSTP
@@ -207,3 +236,8 @@ for arch in "${ARCH_LIST[@]}"; do
                  "${KERNEL_DIR}/${src_dir}/arch/${arch}/include/generated/asm" \
                  "${ANDROID_KERNEL_DIR}/uapi/asm-${arch}/asm"
 done
+
+# Verify if modified headers have changed.
+check_hdrs "${KERNEL_DIR}/${src_dir}/include/scsi" \
+           "${ANDROID_KERNEL_DIR}/scsi" \
+           "${KERNEL_DIR}/${src_dir}"

libc/kernel/tools/update_all.py

@@ -6,72 +6,93 @@ from utils import *
 
 def usage():
     print """\
-  usage: %(progname)s [kernel-original-path]
+  usage: %(progname)s [kernel-original-path] [kernel-modified-path]
 
     this program is used to update all the auto-generated clean headers
     used by the Bionic C library. it assumes the following:
 
-      - a set of source kernel headers is located in '../original',
+      - a set of source kernel headers is located in
-        relative to the program's directory
+        'external/kernel-headers/original', relative to the current
+        android tree
 
-      - the clean headers will be placed in '../arch-<arch>/asm',
+      - a set of manually modified kernel header files located in
-        '../common/linux', '../common/asm-generic', etc..
+        'external/kernel-headers/modified', relative to the current
+        android tree
+
+      - the clean headers will be placed in 'bionic/libc/kernel/arch-<arch>/asm',
+        'bionic/libc/kernel/common', etc..
 """ % { "progname" : os.path.basename(sys.argv[0]) }
     sys.exit(0)
 
 try:
-    optlist, args = getopt.getopt( sys.argv[1:], '' )
+    optlist, args = getopt.getopt(sys.argv[1:], '')
 except:
     # unrecognized option
-    sys.stderr.write( "error: unrecognized option\n" )
+    sys.stderr.write("error: unrecognized option\n")
     usage()
 
-if len(optlist) > 0 or len(args) > 1:
+if len(optlist) > 0 or len(args) > 2:
     usage()
 
-progdir = find_program_dir()
+modified_dir = get_kernel_headers_modified_dir()
-
+if len(args) == 1 or len(args) == 2:
-if len(args) == 1:
     original_dir = args[0]
     if not os.path.isdir(original_dir):
-        panic( "Not a directory: %s\n" % original_dir )
+        panic("Not a directory: %s\n" % original_dir)
+
+    if len(args) == 2:
+        modified_dir = args[1]
+        if not os.path.isdir(modified_dir):
+            panic("Not a directory: %s\n" % modified_dir)
 else:
-    original_dir = kernel_original_path
+    original_dir = get_kernel_headers_original_dir()
     if not os.path.isdir(original_dir):
-        panic( "Missing directory, please specify one through command-line: %s\n" % original_dir )
+        panic("Missing directory, please specify one through command-line: %s\n" % original_dir)
 
-skip_ion = False
+if not os.path.isdir(modified_dir):
+    modified_dir = None
 
-# find all source files in 'original'
+# Find all source files in 'original'.
-#
+sources = dict()
-sources = []
+original_dir = os.path.normpath(original_dir)
-warning_ion = []
+original_dir_len = len(original_dir) + 1
-for root, dirs, files in os.walk( original_dir ):
+for root, _, files in os.walk(original_dir):
     for file in files:
-        if skip_ion and (file == "ion.h" or file == "ion_test.h"):
+        _, ext = os.path.splitext(file)
-            warning_ion.append("  Skipped file %s/%s" % (root, file))
-            continue
-        base, ext = os.path.splitext(file)
         if ext == ".h":
-            sources.append( "%s/%s" % (root,file) )
+            rel_path = os.path.normpath(os.path.join(root, file))
+            rel_path = rel_path[original_dir_len:]
+            # Check to see if there is a modified header to use instead.
+            if modified_dir and os.path.exists(os.path.join(modified_dir, rel_path)):
+                sources[rel_path] = False
+            else:
+                sources[rel_path] = True
+
 
 b = BatchFileUpdater()
 
+kernel_dir = get_kernel_dir()
 for arch in kernel_archs:
-    b.readDir( os.path.normpath( progdir + "/../arch-%s" % arch ) )
+    b.readDir(os.path.join(kernel_dir, "arch-%s" % arch))
 
-b.readDir( os.path.normpath( progdir + "/../common" ) )
+b.readDir(os.path.join(kernel_dir, "common"))
-
-#print "OLD " + repr(b.old_files)
 
 oldlen = 120
-for path in sources:
+android_root_len = len(get_android_root()) + 1
-    dst_path, newdata = clean_header.cleanupFile(path, original_dir)
+for rel_path in sorted(sources):
+    if sources[rel_path]:
+        src_dir = original_dir
+        src_str = "<original>/"
+    else:
+        src_dir = modified_dir
+        src_str = "<modified>/"
+    dst_path, newdata = clean_header.cleanupFile(kernel_dir, src_dir, rel_path)
     if not dst_path:
         continue
 
-    b.readFile( dst_path )
+    dst_path = os.path.join(kernel_dir, dst_path)
-    r = b.editFile( dst_path, newdata )
+    b.readFile(dst_path)
+    r = b.editFile(dst_path, newdata)
     if r == 0:
         state = "unchanged"
     elif r == 1:
@@ -79,9 +100,11 @@ for path in sources:
     else:
         state = "added"
 
-    str = "cleaning: %-*s -> %-*s (%s)" % ( 35, "<original>" + path[len(original_dir):], 35, dst_path, state )
+    # dst_path is guaranteed to include android root.
+    rel_dst_path = dst_path[android_root_len:]
+    str = "cleaning: %-*s -> %-*s (%s)" % (35, src_str + rel_path, 35, rel_dst_path, state)
     if sys.stdout.isatty():
-        print "%-*s" % (oldlen,str),
+        print "%-*s" % (oldlen, str),
         if (r == 0):
             print "\r",
         else:
@@ -92,11 +115,8 @@ for path in sources:
 
     oldlen = len(str)
 
-print "%-*s" % (oldlen,"Done!")
+print "%-*s" % (oldlen, "Done!")
 
 b.updateGitFiles()
 
-if warning_ion:
-    print "NOTE: Due to import into aosp, some files were not processed."
-    print "\n".join(warning_ion)
 sys.exit(0)

libc/kernel/tools/utils.py

@@ -13,8 +13,26 @@ def panic(msg):
     sys.exit(1)
 
 
-def find_program_dir():
+def get_kernel_headers_dir():
-    return os.path.dirname(sys.argv[0])
+    return os.path.join(get_android_root(), "external/kernel-headers")
+
+
+def get_kernel_headers_original_dir():
+    return os.path.join(get_kernel_headers_dir(), "original")
+
+
+def get_kernel_headers_modified_dir():
+    return os.path.join(get_kernel_headers_dir(), "modified")
+
+
+def get_kernel_dir():
+    return os.path.join(get_android_root(), "bionic/libc/kernel")
+
+
+def get_android_root():
+    if "ANDROID_BUILD_TOP" in os.environ:
+        return os.environ["ANDROID_BUILD_TOP"]
+    panic("Unable to find root of tree, did you forget to lunch a target?")
 
 
 class StringOutput: