Merge "Refactoring fuzz packaging code" am: 0b45f0b90f am: c62f2deb45

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1758711

Change-Id: Ib7b2a903e1f5a8a52c38272704c67b9e92cfef94
This commit is contained in:
Treehugger Robot 2021-07-19 20:34:57 +00:00 committed by Automerger Merge Worker
commit 266d97fa2a
8 changed files with 291 additions and 325 deletions

View file

@ -58,6 +58,7 @@ bootstrap_go_package {
"binary.go", "binary.go",
"binary_sdk_member.go", "binary_sdk_member.go",
"fuzz.go", "fuzz.go",
"fuzz_common.go",
"library.go", "library.go",
"library_headers.go", "library_headers.go",
"library_sdk_member.go", "library_sdk_member.go",

View file

@ -401,24 +401,24 @@ func (fuzz *fuzzBinary) AndroidMkEntries(ctx AndroidMkContext, entries *android.
ctx.subAndroidMk(entries, fuzz.binaryDecorator) ctx.subAndroidMk(entries, fuzz.binaryDecorator)
var fuzzFiles []string var fuzzFiles []string
for _, d := range fuzz.corpus { for _, d := range fuzz.fuzzPackagedModule.Corpus {
fuzzFiles = append(fuzzFiles, fuzzFiles = append(fuzzFiles,
filepath.Dir(fuzz.corpusIntermediateDir.String())+":corpus/"+d.Base()) filepath.Dir(fuzz.fuzzPackagedModule.CorpusIntermediateDir.String())+":corpus/"+d.Base())
} }
for _, d := range fuzz.data { for _, d := range fuzz.fuzzPackagedModule.Data {
fuzzFiles = append(fuzzFiles, fuzzFiles = append(fuzzFiles,
filepath.Dir(fuzz.dataIntermediateDir.String())+":data/"+d.Rel()) filepath.Dir(fuzz.fuzzPackagedModule.DataIntermediateDir.String())+":data/"+d.Rel())
} }
if fuzz.dictionary != nil { if fuzz.fuzzPackagedModule.Dictionary != nil {
fuzzFiles = append(fuzzFiles, fuzzFiles = append(fuzzFiles,
filepath.Dir(fuzz.dictionary.String())+":"+fuzz.dictionary.Base()) filepath.Dir(fuzz.fuzzPackagedModule.Dictionary.String())+":"+fuzz.fuzzPackagedModule.Dictionary.Base())
} }
if fuzz.config != nil { if fuzz.fuzzPackagedModule.Config != nil {
fuzzFiles = append(fuzzFiles, fuzzFiles = append(fuzzFiles,
filepath.Dir(fuzz.config.String())+":config.json") filepath.Dir(fuzz.fuzzPackagedModule.Config.String())+":config.json")
} }
entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {

View file

@ -756,14 +756,13 @@ func IsTestPerSrcDepTag(depTag blueprint.DependencyTag) bool {
// members of the cc.Module to this decorator. Thus, a cc_binary module has custom linker and // members of the cc.Module to this decorator. Thus, a cc_binary module has custom linker and
// installer logic. // installer logic.
type Module struct { type Module struct {
android.ModuleBase FuzzModule
android.DefaultableModuleBase
android.ApexModuleBase
android.SdkBase android.SdkBase
android.BazelModuleBase android.BazelModuleBase
Properties BaseProperties
VendorProperties VendorProperties VendorProperties VendorProperties
Properties BaseProperties
// initialize before calling Init // initialize before calling Init
hod android.HostOrDeviceSupported hod android.HostOrDeviceSupported

View file

@ -94,19 +94,14 @@ type fuzzBinary struct {
*binaryDecorator *binaryDecorator
*baseCompiler *baseCompiler
Properties FuzzProperties fuzzPackagedModule FuzzPackagedModule
dictionary android.Path
corpus android.Paths installedSharedDeps []string
corpusIntermediateDir android.Path
config android.Path
data android.Paths
dataIntermediateDir android.Path
installedSharedDeps []string
} }
func (fuzz *fuzzBinary) linkerProps() []interface{} { func (fuzz *fuzzBinary) linkerProps() []interface{} {
props := fuzz.binaryDecorator.linkerProps() props := fuzz.binaryDecorator.linkerProps()
props = append(props, &fuzz.Properties) props = append(props, &fuzz.fuzzPackagedModule.FuzzProperties)
return props return props
} }
@ -257,41 +252,41 @@ func (fuzz *fuzzBinary) install(ctx ModuleContext, file android.Path) {
"fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName()) "fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName())
fuzz.binaryDecorator.baseInstaller.install(ctx, file) fuzz.binaryDecorator.baseInstaller.install(ctx, file)
fuzz.corpus = android.PathsForModuleSrc(ctx, fuzz.Properties.Corpus) fuzz.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Corpus)
builder := android.NewRuleBuilder(pctx, ctx) builder := android.NewRuleBuilder(pctx, ctx)
intermediateDir := android.PathForModuleOut(ctx, "corpus") intermediateDir := android.PathForModuleOut(ctx, "corpus")
for _, entry := range fuzz.corpus { for _, entry := range fuzz.fuzzPackagedModule.Corpus {
builder.Command().Text("cp"). builder.Command().Text("cp").
Input(entry). Input(entry).
Output(intermediateDir.Join(ctx, entry.Base())) Output(intermediateDir.Join(ctx, entry.Base()))
} }
builder.Build("copy_corpus", "copy corpus") builder.Build("copy_corpus", "copy corpus")
fuzz.corpusIntermediateDir = intermediateDir fuzz.fuzzPackagedModule.CorpusIntermediateDir = intermediateDir
fuzz.data = android.PathsForModuleSrc(ctx, fuzz.Properties.Data) fuzz.fuzzPackagedModule.Data = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Data)
builder = android.NewRuleBuilder(pctx, ctx) builder = android.NewRuleBuilder(pctx, ctx)
intermediateDir = android.PathForModuleOut(ctx, "data") intermediateDir = android.PathForModuleOut(ctx, "data")
for _, entry := range fuzz.data { for _, entry := range fuzz.fuzzPackagedModule.Data {
builder.Command().Text("cp"). builder.Command().Text("cp").
Input(entry). Input(entry).
Output(intermediateDir.Join(ctx, entry.Rel())) Output(intermediateDir.Join(ctx, entry.Rel()))
} }
builder.Build("copy_data", "copy data") builder.Build("copy_data", "copy data")
fuzz.dataIntermediateDir = intermediateDir fuzz.fuzzPackagedModule.DataIntermediateDir = intermediateDir
if fuzz.Properties.Dictionary != nil { if fuzz.fuzzPackagedModule.FuzzProperties.Dictionary != nil {
fuzz.dictionary = android.PathForModuleSrc(ctx, *fuzz.Properties.Dictionary) fuzz.fuzzPackagedModule.Dictionary = android.PathForModuleSrc(ctx, *fuzz.fuzzPackagedModule.FuzzProperties.Dictionary)
if fuzz.dictionary.Ext() != ".dict" { if fuzz.fuzzPackagedModule.Dictionary.Ext() != ".dict" {
ctx.PropertyErrorf("dictionary", ctx.PropertyErrorf("dictionary",
"Fuzzer dictionary %q does not have '.dict' extension", "Fuzzer dictionary %q does not have '.dict' extension",
fuzz.dictionary.String()) fuzz.fuzzPackagedModule.Dictionary.String())
} }
} }
if fuzz.Properties.Fuzz_config != nil { if fuzz.fuzzPackagedModule.FuzzProperties.Fuzz_config != nil {
configPath := android.PathForModuleOut(ctx, "config").Join(ctx, "config.json") configPath := android.PathForModuleOut(ctx, "config").Join(ctx, "config.json")
android.WriteFileRule(ctx, configPath, fuzz.Properties.Fuzz_config.String()) android.WriteFileRule(ctx, configPath, fuzz.fuzzPackagedModule.FuzzProperties.Fuzz_config.String())
fuzz.config = configPath fuzz.fuzzPackagedModule.Config = configPath
} }
// Grab the list of required shared libraries. // Grab the list of required shared libraries.
@ -359,32 +354,20 @@ func NewFuzz(hod android.HostOrDeviceSupported) *Module {
// Responsible for generating GNU Make rules that package fuzz targets into // Responsible for generating GNU Make rules that package fuzz targets into
// their architecture & target/host specific zip file. // their architecture & target/host specific zip file.
type fuzzPackager struct { type ccFuzzPackager struct {
packages android.Paths FuzzPackager
sharedLibInstallStrings []string sharedLibInstallStrings []string
fuzzTargets map[string]bool
} }
func fuzzPackagingFactory() android.Singleton { func fuzzPackagingFactory() android.Singleton {
return &fuzzPackager{} return &ccFuzzPackager{}
} }
type fileToZip struct { func (s *ccFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
SourceFilePath android.Path
DestinationPathPrefix string
}
type archOs struct {
hostOrTarget string
arch string
dir string
}
func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
// Map between each architecture + host/device combination, and the files that // Map between each architecture + host/device combination, and the files that
// need to be packaged (in the tuple of {source file, destination folder in // need to be packaged (in the tuple of {source file, destination folder in
// archive}). // archive}).
archDirs := make(map[archOs][]fileToZip) archDirs := make(map[ArchOs][]FileToZip)
// Map tracking whether each shared library has an install rule to avoid duplicate install rules from // Map tracking whether each shared library has an install rule to avoid duplicate install rules from
// multiple fuzzers that depend on the same shared library. // multiple fuzzers that depend on the same shared library.
@ -392,12 +375,16 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
// List of individual fuzz targets, so that 'make fuzz' also installs the targets // List of individual fuzz targets, so that 'make fuzz' also installs the targets
// to the correct output directories as well. // to the correct output directories as well.
s.fuzzTargets = make(map[string]bool) s.FuzzTargets = make(map[string]bool)
ctx.VisitAllModules(func(module android.Module) { ctx.VisitAllModules(func(module android.Module) {
// Discard non-fuzz targets.
ccModule, ok := module.(*Module) ccModule, ok := module.(*Module)
if !ok { if !ok || ccModule.Properties.PreventInstall {
return
}
// Discard non-fuzz targets.
if ok := IsValid(ccModule.FuzzModule); !ok {
return return
} }
@ -406,18 +393,6 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
return return
} }
// Discard ramdisk + vendor_ramdisk + recovery modules, they're duplicates of
// fuzz targets we're going to package anyway.
if !ccModule.Enabled() || ccModule.Properties.PreventInstall ||
ccModule.InRamdisk() || ccModule.InVendorRamdisk() || ccModule.InRecovery() {
return
}
// Discard modules that are in an unavailable namespace.
if !ccModule.ExportedToMake() {
return
}
hostOrTargetString := "target" hostOrTargetString := "target"
if ccModule.Host() { if ccModule.Host() {
hostOrTargetString = "host" hostOrTargetString = "host"
@ -425,42 +400,21 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
archString := ccModule.Arch().ArchType.String() archString := ccModule.Arch().ArchType.String()
archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString) archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString)
archOs := archOs{hostOrTarget: hostOrTargetString, arch: archString, dir: archDir.String()} archOs := ArchOs{HostOrTarget: hostOrTargetString, Arch: archString, Dir: archDir.String()}
// Grab the list of required shared libraries. // Grab the list of required shared libraries.
sharedLibraries := collectAllSharedDependencies(ctx, module) sharedLibraries := collectAllSharedDependencies(ctx, module)
var files []fileToZip var files []FileToZip
builder := android.NewRuleBuilder(pctx, ctx) builder := android.NewRuleBuilder(pctx, ctx)
// Package the corpora into a zipfile. // Package the corpus, data, dict and config into a zipfile.
if fuzzModule.corpus != nil { files = s.PackageArtifacts(ctx, module, fuzzModule.fuzzPackagedModule, archDir, builder)
corpusZip := archDir.Join(ctx, module.Name()+"_seed_corpus.zip")
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", corpusZip)
rspFile := corpusZip.ReplaceExtension(ctx, "rsp")
command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.corpus)
files = append(files, fileToZip{corpusZip, ""})
}
// Package the data into a zipfile.
if fuzzModule.data != nil {
dataZip := archDir.Join(ctx, module.Name()+"_data.zip")
command := builder.Command().BuiltTool("soong_zip").
FlagWithOutput("-o ", dataZip)
for _, f := range fuzzModule.data {
intermediateDir := strings.TrimSuffix(f.String(), f.Rel())
command.FlagWithArg("-C ", intermediateDir)
command.FlagWithInput("-f ", f)
}
files = append(files, fileToZip{dataZip, ""})
}
// Find and mark all the transiently-dependent shared libraries for // Find and mark all the transiently-dependent shared libraries for
// packaging. // packaging.
for _, library := range sharedLibraries { for _, library := range sharedLibraries {
files = append(files, fileToZip{library, "lib"}) files = append(files, FileToZip{library, "lib"})
// For each architecture-specific shared library dependency, we need to // For each architecture-specific shared library dependency, we need to
// install it to the output directory. Setup the install destination here, // install it to the output directory. Setup the install destination here,
@ -492,83 +446,20 @@ func (s *fuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
} }
// The executable. // The executable.
files = append(files, fileToZip{ccModule.UnstrippedOutputFile(), ""}) files = append(files, FileToZip{ccModule.UnstrippedOutputFile(), ""})
// The dictionary. archDirs[archOs], ok = s.BuildZipFile(ctx, module, fuzzModule.fuzzPackagedModule, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs)
if fuzzModule.dictionary != nil { if !ok {
files = append(files, fileToZip{fuzzModule.dictionary, ""}) return
} }
// Additional fuzz config.
if fuzzModule.config != nil {
files = append(files, fileToZip{fuzzModule.config, ""})
}
fuzzZip := archDir.Join(ctx, module.Name()+".zip")
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", fuzzZip)
for _, file := range files {
if file.DestinationPathPrefix != "" {
command.FlagWithArg("-P ", file.DestinationPathPrefix)
} else {
command.Flag("-P ''")
}
command.FlagWithInput("-f ", file.SourceFilePath)
}
builder.Build("create-"+fuzzZip.String(),
"Package "+module.Name()+" for "+archString+"-"+hostOrTargetString)
// Don't add modules to 'make haiku' that are set to not be exported to the
// fuzzing infrastructure.
if config := fuzzModule.Properties.Fuzz_config; config != nil {
if ccModule.Host() && !BoolDefault(config.Fuzz_on_haiku_host, true) {
return
} else if !BoolDefault(config.Fuzz_on_haiku_device, true) {
return
}
}
s.fuzzTargets[module.Name()] = true
archDirs[archOs] = append(archDirs[archOs], fileToZip{fuzzZip, ""})
}) })
var archOsList []archOs s.CreateFuzzPackage(ctx, archDirs, Cc)
for archOs := range archDirs {
archOsList = append(archOsList, archOs)
}
sort.Slice(archOsList, func(i, j int) bool { return archOsList[i].dir < archOsList[j].dir })
for _, archOs := range archOsList {
filesToZip := archDirs[archOs]
arch := archOs.arch
hostOrTarget := archOs.hostOrTarget
builder := android.NewRuleBuilder(pctx, ctx)
outputFile := android.PathForOutput(ctx, "fuzz-"+hostOrTarget+"-"+arch+".zip")
s.packages = append(s.packages, outputFile)
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", outputFile).
Flag("-L 0") // No need to try and re-compress the zipfiles.
for _, fileToZip := range filesToZip {
if fileToZip.DestinationPathPrefix != "" {
command.FlagWithArg("-P ", fileToZip.DestinationPathPrefix)
} else {
command.Flag("-P ''")
}
command.FlagWithInput("-f ", fileToZip.SourceFilePath)
}
builder.Build("create-fuzz-package-"+arch+"-"+hostOrTarget,
"Create fuzz target packages for "+arch+"-"+hostOrTarget)
}
} }
func (s *fuzzPackager) MakeVars(ctx android.MakeVarsContext) { func (s *ccFuzzPackager) MakeVars(ctx android.MakeVarsContext) {
packages := s.packages.Strings() packages := s.Packages.Strings()
sort.Strings(packages) sort.Strings(packages)
sort.Strings(s.sharedLibInstallStrings) sort.Strings(s.sharedLibInstallStrings)
// TODO(mitchp): Migrate this to use MakeVarsContext::DistForGoal() when it's // TODO(mitchp): Migrate this to use MakeVarsContext::DistForGoal() when it's
@ -580,10 +471,5 @@ func (s *fuzzPackager) MakeVars(ctx android.MakeVarsContext) {
strings.Join(s.sharedLibInstallStrings, " ")) strings.Join(s.sharedLibInstallStrings, " "))
// Preallocate the slice of fuzz targets to minimise memory allocations. // Preallocate the slice of fuzz targets to minimise memory allocations.
fuzzTargets := make([]string, 0, len(s.fuzzTargets)) s.PreallocateSlice(ctx, "ALL_FUZZ_TARGETS")
for target, _ := range s.fuzzTargets {
fuzzTargets = append(fuzzTargets, target)
}
sort.Strings(fuzzTargets)
ctx.Strict("ALL_FUZZ_TARGETS", strings.Join(fuzzTargets, " "))
} }

201
cc/fuzz_common.go Normal file
View file

@ -0,0 +1,201 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cc
// This file contains the common code for compiling C/C++ and Rust fuzzers for Android.
import (
"sort"
"strings"
"android/soong/android"
)
type Lang string
const (
Cc Lang = ""
Rust Lang = "rust"
)
type FuzzModule struct {
android.ModuleBase
android.DefaultableModuleBase
android.ApexModuleBase
}
type FuzzPackager struct {
Packages android.Paths
FuzzTargets map[string]bool
}
type FileToZip struct {
SourceFilePath android.Path
DestinationPathPrefix string
}
type ArchOs struct {
HostOrTarget string
Arch string
Dir string
}
type FuzzPackagedModule struct {
FuzzProperties FuzzProperties
Dictionary android.Path
Corpus android.Paths
CorpusIntermediateDir android.Path
Config android.Path
Data android.Paths
DataIntermediateDir android.Path
}
func IsValid(fuzzModule FuzzModule) bool {
// Discard ramdisk + vendor_ramdisk + recovery modules, they're duplicates of
// fuzz targets we're going to package anyway.
if !fuzzModule.Enabled() || fuzzModule.InRamdisk() || fuzzModule.InVendorRamdisk() || fuzzModule.InRecovery() {
return false
}
// Discard modules that are in an unavailable namespace.
if !fuzzModule.ExportedToMake() {
return false
}
return true
}
func (s *FuzzPackager) PackageArtifacts(ctx android.SingletonContext, module android.Module, fuzzModule FuzzPackagedModule, archDir android.OutputPath, builder *android.RuleBuilder) []FileToZip {
// Package the corpora into a zipfile.
var files []FileToZip
if fuzzModule.Corpus != nil {
corpusZip := archDir.Join(ctx, module.Name()+"_seed_corpus.zip")
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", corpusZip)
rspFile := corpusZip.ReplaceExtension(ctx, "rsp")
command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.Corpus)
files = append(files, FileToZip{corpusZip, ""})
}
// Package the data into a zipfile.
if fuzzModule.Data != nil {
dataZip := archDir.Join(ctx, module.Name()+"_data.zip")
command := builder.Command().BuiltTool("soong_zip").
FlagWithOutput("-o ", dataZip)
for _, f := range fuzzModule.Data {
intermediateDir := strings.TrimSuffix(f.String(), f.Rel())
command.FlagWithArg("-C ", intermediateDir)
command.FlagWithInput("-f ", f)
}
files = append(files, FileToZip{dataZip, ""})
}
// The dictionary.
if fuzzModule.Dictionary != nil {
files = append(files, FileToZip{fuzzModule.Dictionary, ""})
}
// Additional fuzz config.
if fuzzModule.Config != nil {
files = append(files, FileToZip{fuzzModule.Config, ""})
}
return files
}
func (s *FuzzPackager) BuildZipFile(ctx android.SingletonContext, module android.Module, fuzzModule FuzzPackagedModule, files []FileToZip, builder *android.RuleBuilder, archDir android.OutputPath, archString string, hostOrTargetString string, archOs ArchOs, archDirs map[ArchOs][]FileToZip) ([]FileToZip, bool) {
fuzzZip := archDir.Join(ctx, module.Name()+".zip")
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", fuzzZip)
for _, file := range files {
if file.DestinationPathPrefix != "" {
command.FlagWithArg("-P ", file.DestinationPathPrefix)
} else {
command.Flag("-P ''")
}
command.FlagWithInput("-f ", file.SourceFilePath)
}
builder.Build("create-"+fuzzZip.String(),
"Package "+module.Name()+" for "+archString+"-"+hostOrTargetString)
// Don't add modules to 'make haiku-rust' that are set to not be
// exported to the fuzzing infrastructure.
if config := fuzzModule.FuzzProperties.Fuzz_config; config != nil {
if strings.Contains(hostOrTargetString, "host") && !BoolDefault(config.Fuzz_on_haiku_host, true) {
return archDirs[archOs], false
} else if !BoolDefault(config.Fuzz_on_haiku_device, true) {
return archDirs[archOs], false
}
}
s.FuzzTargets[module.Name()] = true
archDirs[archOs] = append(archDirs[archOs], FileToZip{fuzzZip, ""})
return archDirs[archOs], true
}
func (s *FuzzPackager) CreateFuzzPackage(ctx android.SingletonContext, archDirs map[ArchOs][]FileToZip, lang Lang) {
var archOsList []ArchOs
for archOs := range archDirs {
archOsList = append(archOsList, archOs)
}
sort.Slice(archOsList, func(i, j int) bool { return archOsList[i].Dir < archOsList[j].Dir })
for _, archOs := range archOsList {
filesToZip := archDirs[archOs]
arch := archOs.Arch
hostOrTarget := archOs.HostOrTarget
builder := android.NewRuleBuilder(pctx, ctx)
zipFileName := "fuzz-" + hostOrTarget + "-" + arch + ".zip"
if lang == Rust {
zipFileName = "fuzz-rust-" + hostOrTarget + "-" + arch + ".zip"
}
outputFile := android.PathForOutput(ctx, zipFileName)
s.Packages = append(s.Packages, outputFile)
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", outputFile).
Flag("-L 0") // No need to try and re-compress the zipfiles.
for _, fileToZip := range filesToZip {
if fileToZip.DestinationPathPrefix != "" {
command.FlagWithArg("-P ", fileToZip.DestinationPathPrefix)
} else {
command.Flag("-P ''")
}
command.FlagWithInput("-f ", fileToZip.SourceFilePath)
}
builder.Build("create-fuzz-package-"+arch+"-"+hostOrTarget,
"Create fuzz target packages for "+arch+"-"+hostOrTarget)
}
}
func (s *FuzzPackager) PreallocateSlice(ctx android.MakeVarsContext, targets string) {
fuzzTargets := make([]string, 0, len(s.FuzzTargets))
for target, _ := range s.FuzzTargets {
fuzzTargets = append(fuzzTargets, target)
}
sort.Strings(fuzzTargets)
ctx.Strict(targets, strings.Join(fuzzTargets, " "))
}

View file

@ -205,24 +205,24 @@ func (fuzz *fuzzDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *andro
ctx.SubAndroidMk(entries, fuzz.binaryDecorator) ctx.SubAndroidMk(entries, fuzz.binaryDecorator)
var fuzzFiles []string var fuzzFiles []string
for _, d := range fuzz.corpus { for _, d := range fuzz.fuzzPackagedModule.Corpus {
fuzzFiles = append(fuzzFiles, fuzzFiles = append(fuzzFiles,
filepath.Dir(fuzz.corpusIntermediateDir.String())+":corpus/"+d.Base()) filepath.Dir(fuzz.fuzzPackagedModule.CorpusIntermediateDir.String())+":corpus/"+d.Base())
} }
for _, d := range fuzz.data { for _, d := range fuzz.fuzzPackagedModule.Data {
fuzzFiles = append(fuzzFiles, fuzzFiles = append(fuzzFiles,
filepath.Dir(fuzz.dataIntermediateDir.String())+":data/"+d.Rel()) filepath.Dir(fuzz.fuzzPackagedModule.DataIntermediateDir.String())+":data/"+d.Rel())
} }
if fuzz.dictionary != nil { if fuzz.fuzzPackagedModule.Dictionary != nil {
fuzzFiles = append(fuzzFiles, fuzzFiles = append(fuzzFiles,
filepath.Dir(fuzz.dictionary.String())+":"+fuzz.dictionary.Base()) filepath.Dir(fuzz.fuzzPackagedModule.Dictionary.String())+":"+fuzz.fuzzPackagedModule.Dictionary.Base())
} }
if fuzz.config != nil { if fuzz.fuzzPackagedModule.Config != nil {
fuzzFiles = append(fuzzFiles, fuzzFiles = append(fuzzFiles,
filepath.Dir(fuzz.config.String())+":config.json") filepath.Dir(fuzz.fuzzPackagedModule.Config.String())+":config.json")
} }
entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext,

View file

@ -32,13 +32,7 @@ func init() {
type fuzzDecorator struct { type fuzzDecorator struct {
*binaryDecorator *binaryDecorator
Properties cc.FuzzProperties fuzzPackagedModule cc.FuzzPackagedModule
dictionary android.Path
corpus android.Paths
corpusIntermediateDir android.Path
config android.Path
data android.Paths
dataIntermediateDir android.Path
} }
var _ compiler = (*binaryDecorator)(nil) var _ compiler = (*binaryDecorator)(nil)
@ -88,7 +82,7 @@ func (fuzzer *fuzzDecorator) compilerDeps(ctx DepsContext, deps Deps) Deps {
func (fuzzer *fuzzDecorator) compilerProps() []interface{} { func (fuzzer *fuzzDecorator) compilerProps() []interface{} {
return append(fuzzer.binaryDecorator.compilerProps(), return append(fuzzer.binaryDecorator.compilerProps(),
&fuzzer.Properties) &fuzzer.fuzzPackagedModule.FuzzProperties)
} }
func (fuzzer *fuzzDecorator) stdLinkage(ctx *depsContext) RustLinkage { func (fuzzer *fuzzDecorator) stdLinkage(ctx *depsContext) RustLinkage {
@ -102,32 +96,19 @@ func (fuzzer *fuzzDecorator) autoDep(ctx android.BottomUpMutatorContext) autoDep
// Responsible for generating GNU Make rules that package fuzz targets into // Responsible for generating GNU Make rules that package fuzz targets into
// their architecture & target/host specific zip file. // their architecture & target/host specific zip file.
type rustFuzzPackager struct { type rustFuzzPackager struct {
packages android.Paths cc.FuzzPackager
fuzzTargets map[string]bool
} }
func rustFuzzPackagingFactory() android.Singleton { func rustFuzzPackagingFactory() android.Singleton {
return &rustFuzzPackager{} return &rustFuzzPackager{}
} }
type fileToZip struct {
SourceFilePath android.Path
DestinationPathPrefix string
}
type archOs struct {
hostOrTarget string
arch string
dir string
}
func (s *rustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { func (s *rustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
// Map between each architecture + host/device combination. // Map between each architecture + host/device combination.
archDirs := make(map[archOs][]fileToZip) archDirs := make(map[cc.ArchOs][]cc.FileToZip)
// List of individual fuzz targets. // List of individual fuzz targets.
s.fuzzTargets = make(map[string]bool) s.FuzzTargets = make(map[string]bool)
ctx.VisitAllModules(func(module android.Module) { ctx.VisitAllModules(func(module android.Module) {
// Discard non-fuzz targets. // Discard non-fuzz targets.
@ -136,23 +117,15 @@ func (s *rustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
return return
} }
if ok := cc.IsValid(rustModule.FuzzModule); !ok || rustModule.Properties.PreventInstall {
return
}
fuzzModule, ok := rustModule.compiler.(*fuzzDecorator) fuzzModule, ok := rustModule.compiler.(*fuzzDecorator)
if !ok { if !ok {
return return
} }
// Discard ramdisk + vendor_ramdisk + recovery modules, they're duplicates of
// fuzz targets we're going to package anyway.
if !rustModule.Enabled() || rustModule.Properties.PreventInstall ||
rustModule.InRamdisk() || rustModule.InVendorRamdisk() || rustModule.InRecovery() {
return
}
// Discard modules that are in an unavailable namespace.
if !rustModule.ExportedToMake() {
return
}
hostOrTargetString := "target" hostOrTargetString := "target"
if rustModule.Host() { if rustModule.Host() {
hostOrTargetString = "host" hostOrTargetString = "host"
@ -160,126 +133,34 @@ func (s *rustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) {
archString := rustModule.Arch().ArchType.String() archString := rustModule.Arch().ArchType.String()
archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString) archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString)
archOs := archOs{hostOrTarget: hostOrTargetString, arch: archString, dir: archDir.String()} archOs := cc.ArchOs{HostOrTarget: hostOrTargetString, Arch: archString, Dir: archDir.String()}
var files []fileToZip var files []cc.FileToZip
builder := android.NewRuleBuilder(pctx, ctx) builder := android.NewRuleBuilder(pctx, ctx)
// Package the corpora into a zipfile. // Package the artifacts (data, corpus, config and dictionary into a zipfile.
if fuzzModule.corpus != nil { files = s.PackageArtifacts(ctx, module, fuzzModule.fuzzPackagedModule, archDir, builder)
corpusZip := archDir.Join(ctx, module.Name()+"_seed_corpus.zip")
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", corpusZip)
rspFile := corpusZip.ReplaceExtension(ctx, "rsp")
command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.corpus)
files = append(files, fileToZip{corpusZip, ""})
}
// Package the data into a zipfile.
if fuzzModule.data != nil {
dataZip := archDir.Join(ctx, module.Name()+"_data.zip")
command := builder.Command().BuiltTool("soong_zip").
FlagWithOutput("-o ", dataZip)
for _, f := range fuzzModule.data {
intermediateDir := strings.TrimSuffix(f.String(), f.Rel())
command.FlagWithArg("-C ", intermediateDir)
command.FlagWithInput("-f ", f)
}
files = append(files, fileToZip{dataZip, ""})
}
// The executable. // The executable.
files = append(files, fileToZip{rustModule.unstrippedOutputFile.Path(), ""}) files = append(files, cc.FileToZip{rustModule.unstrippedOutputFile.Path(), ""})
// The dictionary. archDirs[archOs], ok = s.BuildZipFile(ctx, module, fuzzModule.fuzzPackagedModule, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs)
if fuzzModule.dictionary != nil { if !ok {
files = append(files, fileToZip{fuzzModule.dictionary, ""}) return
} }
// Additional fuzz config.
if fuzzModule.config != nil {
files = append(files, fileToZip{fuzzModule.config, ""})
}
fuzzZip := archDir.Join(ctx, module.Name()+".zip")
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", fuzzZip)
for _, file := range files {
if file.DestinationPathPrefix != "" {
command.FlagWithArg("-P ", file.DestinationPathPrefix)
} else {
command.Flag("-P ''")
}
command.FlagWithInput("-f ", file.SourceFilePath)
}
builder.Build("create-"+fuzzZip.String(),
"Package "+module.Name()+" for "+archString+"-"+hostOrTargetString)
// Don't add modules to 'make haiku-rust' that are set to not be
// exported to the fuzzing infrastructure.
if config := fuzzModule.Properties.Fuzz_config; config != nil {
if rustModule.Host() && !BoolDefault(config.Fuzz_on_haiku_host, true) {
return
} else if !BoolDefault(config.Fuzz_on_haiku_device, true) {
return
}
}
s.fuzzTargets[module.Name()] = true
archDirs[archOs] = append(archDirs[archOs], fileToZip{fuzzZip, ""})
}) })
s.CreateFuzzPackage(ctx, archDirs, cc.Rust)
var archOsList []archOs
for archOs := range archDirs {
archOsList = append(archOsList, archOs)
}
sort.Slice(archOsList, func(i, j int) bool { return archOsList[i].dir < archOsList[j].dir })
for _, archOs := range archOsList {
filesToZip := archDirs[archOs]
arch := archOs.arch
hostOrTarget := archOs.hostOrTarget
builder := android.NewRuleBuilder(pctx, ctx)
outputFile := android.PathForOutput(ctx, "fuzz-rust-"+hostOrTarget+"-"+arch+".zip")
s.packages = append(s.packages, outputFile)
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", outputFile).
Flag("-L 0") // No need to try and re-compress the zipfiles.
for _, fileToZip := range filesToZip {
if fileToZip.DestinationPathPrefix != "" {
command.FlagWithArg("-P ", fileToZip.DestinationPathPrefix)
} else {
command.Flag("-P ''")
}
command.FlagWithInput("-f ", fileToZip.SourceFilePath)
}
builder.Build("create-fuzz-package-"+arch+"-"+hostOrTarget,
"Create fuzz target packages for "+arch+"-"+hostOrTarget)
}
} }
func (s *rustFuzzPackager) MakeVars(ctx android.MakeVarsContext) { func (s *rustFuzzPackager) MakeVars(ctx android.MakeVarsContext) {
packages := s.packages.Strings() packages := s.Packages.Strings()
sort.Strings(packages) sort.Strings(packages)
ctx.Strict("SOONG_RUST_FUZZ_PACKAGING_ARCH_MODULES", strings.Join(packages, " ")) ctx.Strict("SOONG_RUST_FUZZ_PACKAGING_ARCH_MODULES", strings.Join(packages, " "))
// Preallocate the slice of fuzz targets to minimise memory allocations. // Preallocate the slice of fuzz targets to minimize memory allocations.
fuzzTargets := make([]string, 0, len(s.fuzzTargets)) s.PreallocateSlice(ctx, "ALL_RUST_FUZZ_TARGETS")
for target, _ := range s.fuzzTargets {
fuzzTargets = append(fuzzTargets, target)
}
sort.Strings(fuzzTargets)
ctx.Strict("ALL_RUST_FUZZ_TARGETS", strings.Join(fuzzTargets, " "))
} }
func (fuzz *fuzzDecorator) install(ctx ModuleContext) { func (fuzz *fuzzDecorator) install(ctx ModuleContext) {
@ -289,13 +170,13 @@ func (fuzz *fuzzDecorator) install(ctx ModuleContext) {
"fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName()) "fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName())
fuzz.binaryDecorator.baseCompiler.install(ctx) fuzz.binaryDecorator.baseCompiler.install(ctx)
if fuzz.Properties.Corpus != nil { if fuzz.fuzzPackagedModule.FuzzProperties.Corpus != nil {
fuzz.corpus = android.PathsForModuleSrc(ctx, fuzz.Properties.Corpus) fuzz.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Corpus)
} }
if fuzz.Properties.Data != nil { if fuzz.fuzzPackagedModule.FuzzProperties.Data != nil {
fuzz.data = android.PathsForModuleSrc(ctx, fuzz.Properties.Data) fuzz.fuzzPackagedModule.Data = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Data)
} }
if fuzz.Properties.Dictionary != nil { if fuzz.fuzzPackagedModule.FuzzProperties.Dictionary != nil {
fuzz.dictionary = android.PathForModuleSrc(ctx, *fuzz.Properties.Dictionary) fuzz.fuzzPackagedModule.Dictionary = android.PathForModuleSrc(ctx, *fuzz.fuzzPackagedModule.FuzzProperties.Dictionary)
} }
} }

View file

@ -119,9 +119,7 @@ type BaseProperties struct {
} }
type Module struct { type Module struct {
android.ModuleBase cc.FuzzModule
android.DefaultableModuleBase
android.ApexModuleBase
VendorProperties cc.VendorProperties VendorProperties cc.VendorProperties