2020-12-10 23:19:18 +01:00
|
|
|
// Copyright 2020 Google Inc. All rights reserved.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package bazel
|
|
|
|
|
|
|
|
import (
|
2022-05-12 22:43:01 +02:00
|
|
|
"crypto/sha256"
|
2022-06-02 16:19:13 +02:00
|
|
|
"encoding/base64"
|
2020-12-10 23:19:18 +01:00
|
|
|
"encoding/json"
|
2020-12-23 18:02:11 +01:00
|
|
|
"fmt"
|
|
|
|
"path/filepath"
|
2022-05-12 22:43:01 +02:00
|
|
|
"reflect"
|
2021-11-04 23:58:12 +01:00
|
|
|
"regexp"
|
2022-05-12 22:43:01 +02:00
|
|
|
"sort"
|
2020-12-10 23:19:18 +01:00
|
|
|
"strings"
|
|
|
|
|
|
|
|
"github.com/google/blueprint/proptools"
|
|
|
|
)
|
|
|
|
|
2022-05-27 23:40:21 +02:00
|
|
|
type artifactId int
|
|
|
|
type depsetId int
|
|
|
|
type pathFragmentId int
|
|
|
|
|
2020-12-10 23:19:18 +01:00
|
|
|
// artifact contains relevant portions of Bazel's aquery proto, Artifact.
|
|
|
|
// Represents a single artifact, whether it's a source file or a derived output file.
|
|
|
|
type artifact struct {
|
2022-05-27 23:40:21 +02:00
|
|
|
Id artifactId
|
|
|
|
PathFragmentId pathFragmentId
|
2020-12-23 18:02:11 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
type pathFragment struct {
|
2022-05-27 23:40:21 +02:00
|
|
|
Id pathFragmentId
|
2020-12-23 18:02:11 +01:00
|
|
|
Label string
|
2022-05-27 23:40:21 +02:00
|
|
|
ParentId pathFragmentId
|
2020-12-10 23:19:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// KeyValuePair represents Bazel's aquery proto, KeyValuePair.
|
|
|
|
type KeyValuePair struct {
|
|
|
|
Key string
|
|
|
|
Value string
|
|
|
|
}
|
|
|
|
|
2022-04-26 04:35:15 +02:00
|
|
|
// AqueryDepset is a depset definition from Bazel's aquery response. This is
|
2022-05-12 22:43:01 +02:00
|
|
|
// akin to the `depSetOfFiles` in the response proto, except:
|
|
|
|
// * direct artifacts are enumerated by full path instead of by ID
|
2022-05-13 22:44:07 +02:00
|
|
|
// * it has a hash of the depset contents, instead of an int ID (for determinism)
|
2022-04-26 04:35:15 +02:00
|
|
|
// A depset is a data structure for efficient transitive handling of artifact
|
|
|
|
// paths. A single depset consists of one or more artifact paths and one or
|
|
|
|
// more "child" depsets.
|
|
|
|
type AqueryDepset struct {
|
2022-05-12 22:43:01 +02:00
|
|
|
ContentHash string
|
|
|
|
DirectArtifacts []string
|
|
|
|
TransitiveDepSetHashes []string
|
2022-04-26 04:35:15 +02:00
|
|
|
}
|
|
|
|
|
2020-12-10 23:19:18 +01:00
|
|
|
// depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles.
|
|
|
|
// Represents a data structure containing one or more files. Depsets in Bazel are an efficient
|
|
|
|
// data structure for storing large numbers of file paths.
|
|
|
|
type depSetOfFiles struct {
|
2022-05-27 23:40:21 +02:00
|
|
|
Id depsetId
|
|
|
|
DirectArtifactIds []artifactId
|
|
|
|
TransitiveDepSetIds []depsetId
|
2020-12-10 23:19:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// action contains relevant portions of Bazel's aquery proto, Action.
|
|
|
|
// Represents a single command line invocation in the Bazel build graph.
|
|
|
|
type action struct {
|
|
|
|
Arguments []string
|
|
|
|
EnvironmentVariables []KeyValuePair
|
2022-05-27 23:40:21 +02:00
|
|
|
InputDepSetIds []depsetId
|
2020-12-10 23:19:18 +01:00
|
|
|
Mnemonic string
|
2022-05-27 23:40:21 +02:00
|
|
|
OutputIds []artifactId
|
2021-11-04 23:58:12 +01:00
|
|
|
TemplateContent string
|
|
|
|
Substitutions []KeyValuePair
|
2020-12-10 23:19:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer.
|
|
|
|
// An aquery response from Bazel contains a single ActionGraphContainer proto.
|
|
|
|
type actionGraphContainer struct {
|
|
|
|
Artifacts []artifact
|
|
|
|
Actions []action
|
|
|
|
DepSetOfFiles []depSetOfFiles
|
2020-12-23 18:02:11 +01:00
|
|
|
PathFragments []pathFragment
|
2020-12-10 23:19:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// BuildStatement contains information to register a build statement corresponding (one to one)
|
|
|
|
// with a Bazel action from Bazel's action graph.
|
|
|
|
type BuildStatement struct {
|
2021-06-08 21:04:11 +02:00
|
|
|
Command string
|
|
|
|
Depfile *string
|
|
|
|
OutputPaths []string
|
|
|
|
SymlinkPaths []string
|
|
|
|
Env []KeyValuePair
|
|
|
|
Mnemonic string
|
2022-04-26 04:35:15 +02:00
|
|
|
|
|
|
|
// Inputs of this build statement, either as unexpanded depsets or expanded
|
|
|
|
// input paths. There should be no overlap between these fields; an input
|
|
|
|
// path should either be included as part of an unexpanded depset or a raw
|
|
|
|
// input path string, but not both.
|
2022-05-12 22:43:01 +02:00
|
|
|
InputDepsetHashes []string
|
|
|
|
InputPaths []string
|
2020-12-10 23:19:18 +01:00
|
|
|
}
|
|
|
|
|
2021-05-18 18:31:25 +02:00
|
|
|
// A helper type for aquery processing which facilitates retrieval of path IDs from their
|
|
|
|
// less readable Bazel structures (depset and path fragment).
|
|
|
|
type aqueryArtifactHandler struct {
|
2022-05-12 22:43:01 +02:00
|
|
|
// Maps depset id to AqueryDepset, a representation of depset which is
|
|
|
|
// post-processed for middleman artifact handling, unhandled artifact
|
|
|
|
// dropping, content hashing, etc.
|
2022-05-27 23:40:21 +02:00
|
|
|
depsetIdToAqueryDepset map[depsetId]AqueryDepset
|
2022-05-12 22:43:01 +02:00
|
|
|
// Maps content hash to AqueryDepset.
|
|
|
|
depsetHashToAqueryDepset map[string]AqueryDepset
|
|
|
|
|
2021-05-18 18:31:25 +02:00
|
|
|
// depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening
|
|
|
|
// may be an expensive operation.
|
2022-05-12 22:43:01 +02:00
|
|
|
depsetHashToArtifactPathsCache map[string][]string
|
2022-05-27 23:40:21 +02:00
|
|
|
// Maps artifact ids to fully expanded paths.
|
|
|
|
artifactIdToPath map[artifactId]string
|
2021-05-18 18:31:25 +02:00
|
|
|
}
|
2020-12-10 23:19:18 +01:00
|
|
|
|
2021-11-04 23:58:12 +01:00
|
|
|
// The tokens should be substituted with the value specified here, instead of the
|
|
|
|
// one returned in 'substitutions' of TemplateExpand action.
|
2022-05-27 23:40:21 +02:00
|
|
|
var templateActionOverriddenTokens = map[string]string{
|
2021-11-04 23:58:12 +01:00
|
|
|
// Uses "python3" for %python_binary% instead of the value returned by aquery
|
|
|
|
// which is "py3wrapper.sh". See removePy3wrapperScript.
|
|
|
|
"%python_binary%": "python3",
|
|
|
|
}
|
|
|
|
|
|
|
|
// This pattern matches the MANIFEST file created for a py_binary target.
|
|
|
|
var manifestFilePattern = regexp.MustCompile(".*/.+\\.runfiles/MANIFEST$")
|
|
|
|
|
|
|
|
// The file name of py3wrapper.sh, which is used by py_binary targets.
|
2022-05-12 22:43:01 +02:00
|
|
|
const py3wrapperFileName = "/py3wrapper.sh"
|
2021-11-04 23:58:12 +01:00
|
|
|
|
2022-05-27 23:40:21 +02:00
|
|
|
func indexBy[K comparable, V any](values []V, keyFn func(v V) K) map[K]V {
|
|
|
|
m := map[K]V{}
|
|
|
|
for _, v := range values {
|
|
|
|
m[keyFn(v)] = v
|
2020-12-23 18:02:11 +01:00
|
|
|
}
|
2022-05-27 23:40:21 +02:00
|
|
|
return m
|
|
|
|
}
|
|
|
|
|
|
|
|
func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler, error) {
|
|
|
|
pathFragments := indexBy(aqueryResult.PathFragments, func(pf pathFragment) pathFragmentId {
|
|
|
|
return pf.Id
|
|
|
|
})
|
2021-05-18 18:31:25 +02:00
|
|
|
|
2022-05-27 23:40:21 +02:00
|
|
|
artifactIdToPath := map[artifactId]string{}
|
2020-12-10 23:19:18 +01:00
|
|
|
for _, artifact := range aqueryResult.Artifacts {
|
2020-12-23 18:02:11 +01:00
|
|
|
artifactPath, err := expandPathFragment(artifact.PathFragmentId, pathFragments)
|
|
|
|
if err != nil {
|
2021-01-15 18:22:41 +01:00
|
|
|
return nil, err
|
2020-12-23 18:02:11 +01:00
|
|
|
}
|
|
|
|
artifactIdToPath[artifact.Id] = artifactPath
|
2020-12-10 23:19:18 +01:00
|
|
|
}
|
2021-01-19 17:36:50 +01:00
|
|
|
|
2022-05-12 22:43:01 +02:00
|
|
|
// Map middleman artifact ContentHash to input artifact depset ID.
|
2022-04-26 04:35:15 +02:00
|
|
|
// Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example,
|
|
|
|
// if we find a middleman action which has outputs [foo, bar], and output [baz_middleman], then,
|
|
|
|
// for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for
|
|
|
|
// that action instead.
|
2022-05-27 23:40:21 +02:00
|
|
|
middlemanIdToDepsetIds := map[artifactId][]depsetId{}
|
2020-12-10 23:19:18 +01:00
|
|
|
for _, actionEntry := range aqueryResult.Actions {
|
2021-02-22 22:13:50 +01:00
|
|
|
if actionEntry.Mnemonic == "Middleman" {
|
|
|
|
for _, outputId := range actionEntry.OutputIds {
|
2021-05-18 18:31:25 +02:00
|
|
|
middlemanIdToDepsetIds[outputId] = actionEntry.InputDepSetIds
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-04-26 04:35:15 +02:00
|
|
|
|
2022-05-27 23:40:21 +02:00
|
|
|
depsetIdToDepset := indexBy(aqueryResult.DepSetOfFiles, func(d depSetOfFiles) depsetId {
|
|
|
|
return d.Id
|
|
|
|
})
|
2022-05-12 22:43:01 +02:00
|
|
|
|
|
|
|
aqueryHandler := aqueryArtifactHandler{
|
2022-05-27 23:40:21 +02:00
|
|
|
depsetIdToAqueryDepset: map[depsetId]AqueryDepset{},
|
2022-05-12 22:43:01 +02:00
|
|
|
depsetHashToAqueryDepset: map[string]AqueryDepset{},
|
|
|
|
depsetHashToArtifactPathsCache: map[string][]string{},
|
|
|
|
artifactIdToPath: artifactIdToPath,
|
|
|
|
}
|
|
|
|
|
2022-04-26 04:35:15 +02:00
|
|
|
// Validate and adjust aqueryResult.DepSetOfFiles values.
|
|
|
|
for _, depset := range aqueryResult.DepSetOfFiles {
|
2022-05-12 22:43:01 +02:00
|
|
|
_, err := aqueryHandler.populateDepsetMaps(depset, middlemanIdToDepsetIds, depsetIdToDepset)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2022-04-26 04:35:15 +02:00
|
|
|
}
|
2022-05-12 22:43:01 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return &aqueryHandler, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Ensures that the handler's depsetIdToAqueryDepset map contains an entry for the given
|
|
|
|
// depset.
|
2022-05-27 23:40:21 +02:00
|
|
|
func (a *aqueryArtifactHandler) populateDepsetMaps(depset depSetOfFiles, middlemanIdToDepsetIds map[artifactId][]depsetId, depsetIdToDepset map[depsetId]depSetOfFiles) (AqueryDepset, error) {
|
2022-05-12 22:43:01 +02:00
|
|
|
if aqueryDepset, containsDepset := a.depsetIdToAqueryDepset[depset.Id]; containsDepset {
|
|
|
|
return aqueryDepset, nil
|
|
|
|
}
|
|
|
|
transitiveDepsetIds := depset.TransitiveDepSetIds
|
2022-05-27 23:40:21 +02:00
|
|
|
var directArtifactPaths []string
|
2022-05-12 22:43:01 +02:00
|
|
|
for _, artifactId := range depset.DirectArtifactIds {
|
|
|
|
path, pathExists := a.artifactIdToPath[artifactId]
|
|
|
|
if !pathExists {
|
|
|
|
return AqueryDepset{}, fmt.Errorf("undefined input artifactId %d", artifactId)
|
|
|
|
}
|
|
|
|
// Filter out any inputs which are universally dropped, and swap middleman
|
|
|
|
// artifacts with their corresponding depsets.
|
|
|
|
if depsetsToUse, isMiddleman := middlemanIdToDepsetIds[artifactId]; isMiddleman {
|
|
|
|
// Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts.
|
|
|
|
transitiveDepsetIds = append(transitiveDepsetIds, depsetsToUse...)
|
|
|
|
} else if strings.HasSuffix(path, py3wrapperFileName) || manifestFilePattern.MatchString(path) {
|
|
|
|
// Drop these artifacts.
|
|
|
|
// See go/python-binary-host-mixed-build for more details.
|
|
|
|
// 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of
|
|
|
|
// Bazel py_binary targets, so there is no Ninja build statements generated for creating it.
|
|
|
|
// 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets,
|
|
|
|
// but it doesn't contain sufficient information so no Ninja build statements are generated
|
|
|
|
// for creating it.
|
|
|
|
// So in mixed build mode, when these two are used as input of some Ninja build statement,
|
|
|
|
// since there is no build statement to create them, they should be removed from input paths.
|
|
|
|
// TODO(b/197135294): Clean up this custom runfiles handling logic when
|
|
|
|
// SourceSymlinkManifest and SymlinkTree actions are supported.
|
|
|
|
} else {
|
|
|
|
// TODO(b/216194240): Filter out bazel tools.
|
|
|
|
directArtifactPaths = append(directArtifactPaths, path)
|
2022-04-26 04:35:15 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-27 23:40:21 +02:00
|
|
|
var childDepsetHashes []string
|
2022-05-12 22:43:01 +02:00
|
|
|
for _, childDepsetId := range transitiveDepsetIds {
|
|
|
|
childDepset, exists := depsetIdToDepset[childDepsetId]
|
|
|
|
if !exists {
|
|
|
|
return AqueryDepset{}, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id)
|
|
|
|
}
|
|
|
|
childAqueryDepset, err := a.populateDepsetMaps(childDepset, middlemanIdToDepsetIds, depsetIdToDepset)
|
|
|
|
if err != nil {
|
|
|
|
return AqueryDepset{}, err
|
|
|
|
}
|
|
|
|
childDepsetHashes = append(childDepsetHashes, childAqueryDepset.ContentHash)
|
|
|
|
}
|
|
|
|
aqueryDepset := AqueryDepset{
|
|
|
|
ContentHash: depsetContentHash(directArtifactPaths, childDepsetHashes),
|
|
|
|
DirectArtifacts: directArtifactPaths,
|
|
|
|
TransitiveDepSetHashes: childDepsetHashes,
|
|
|
|
}
|
|
|
|
a.depsetIdToAqueryDepset[depset.Id] = aqueryDepset
|
|
|
|
a.depsetHashToAqueryDepset[aqueryDepset.ContentHash] = aqueryDepset
|
|
|
|
return aqueryDepset, nil
|
2021-05-18 18:31:25 +02:00
|
|
|
}
|
|
|
|
|
2022-04-26 04:35:15 +02:00
|
|
|
// getInputPaths flattens the depsets of the given IDs and returns all transitive
|
|
|
|
// input paths contained in these depsets.
|
|
|
|
// This is a potentially expensive operation, and should not be invoked except
|
|
|
|
// for actions which need specialized input handling.
|
2022-05-27 23:40:21 +02:00
|
|
|
func (a *aqueryArtifactHandler) getInputPaths(depsetIds []depsetId) ([]string, error) {
|
|
|
|
var inputPaths []string
|
2021-05-18 18:31:25 +02:00
|
|
|
|
|
|
|
for _, inputDepSetId := range depsetIds {
|
2022-05-12 22:43:01 +02:00
|
|
|
depset := a.depsetIdToAqueryDepset[inputDepSetId]
|
|
|
|
inputArtifacts, err := a.artifactPathsFromDepsetHash(depset.ContentHash)
|
2021-05-18 18:31:25 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-05-12 22:43:01 +02:00
|
|
|
for _, inputPath := range inputArtifacts {
|
2022-04-26 04:35:15 +02:00
|
|
|
inputPaths = append(inputPaths, inputPath)
|
2021-02-22 22:13:50 +01:00
|
|
|
}
|
|
|
|
}
|
2021-11-04 23:58:12 +01:00
|
|
|
|
2022-04-26 04:35:15 +02:00
|
|
|
return inputPaths, nil
|
2021-05-18 18:31:25 +02:00
|
|
|
}
|
|
|
|
|
2022-05-12 22:43:01 +02:00
|
|
|
func (a *aqueryArtifactHandler) artifactPathsFromDepsetHash(depsetHash string) ([]string, error) {
|
|
|
|
if result, exists := a.depsetHashToArtifactPathsCache[depsetHash]; exists {
|
2021-05-18 18:31:25 +02:00
|
|
|
return result, nil
|
|
|
|
}
|
2022-05-12 22:43:01 +02:00
|
|
|
if depset, exists := a.depsetHashToAqueryDepset[depsetHash]; exists {
|
|
|
|
result := depset.DirectArtifacts
|
|
|
|
for _, childHash := range depset.TransitiveDepSetHashes {
|
|
|
|
childArtifactIds, err := a.artifactPathsFromDepsetHash(childHash)
|
2021-05-18 18:31:25 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
result = append(result, childArtifactIds...)
|
|
|
|
}
|
2022-05-12 22:43:01 +02:00
|
|
|
a.depsetHashToArtifactPathsCache[depsetHash] = result
|
2021-05-18 18:31:25 +02:00
|
|
|
return result, nil
|
|
|
|
} else {
|
2022-06-02 16:19:13 +02:00
|
|
|
return nil, fmt.Errorf("undefined input depset hash %s", depsetHash)
|
2021-05-18 18:31:25 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-26 04:35:15 +02:00
|
|
|
// AqueryBuildStatements returns a slice of BuildStatements and a slice of AqueryDepset
|
2022-05-27 23:40:21 +02:00
|
|
|
// which should be registered (and output to a ninja file) to correspond with Bazel's
|
2022-04-26 04:35:15 +02:00
|
|
|
// action graph, as described by the given action graph json proto.
|
|
|
|
// BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets
|
|
|
|
// are one-to-one with Bazel's depSetOfFiles objects.
|
|
|
|
func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDepset, error) {
|
2021-05-18 18:31:25 +02:00
|
|
|
var aqueryResult actionGraphContainer
|
|
|
|
err := json.Unmarshal(aqueryJsonProto, &aqueryResult)
|
|
|
|
if err != nil {
|
2022-04-26 04:35:15 +02:00
|
|
|
return nil, nil, err
|
2021-05-18 18:31:25 +02:00
|
|
|
}
|
|
|
|
aqueryHandler, err := newAqueryHandler(aqueryResult)
|
|
|
|
if err != nil {
|
2022-04-26 04:35:15 +02:00
|
|
|
return nil, nil, err
|
2021-05-18 18:31:25 +02:00
|
|
|
}
|
2021-02-22 22:13:50 +01:00
|
|
|
|
2022-05-12 22:43:01 +02:00
|
|
|
var buildStatements []BuildStatement
|
|
|
|
|
2021-02-22 22:13:50 +01:00
|
|
|
for _, actionEntry := range aqueryResult.Actions {
|
|
|
|
if shouldSkipAction(actionEntry) {
|
|
|
|
continue
|
|
|
|
}
|
2021-06-08 21:04:11 +02:00
|
|
|
|
2022-04-26 04:35:15 +02:00
|
|
|
var buildStatement BuildStatement
|
2021-06-08 21:04:11 +02:00
|
|
|
if isSymlinkAction(actionEntry) {
|
2022-04-26 04:35:15 +02:00
|
|
|
buildStatement, err = aqueryHandler.symlinkActionBuildStatement(actionEntry)
|
2021-11-04 23:58:12 +01:00
|
|
|
} else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 {
|
2022-04-26 04:35:15 +02:00
|
|
|
buildStatement, err = aqueryHandler.templateExpandActionBuildStatement(actionEntry)
|
2021-11-04 23:58:12 +01:00
|
|
|
} else if isPythonZipperAction(actionEntry) {
|
2022-04-26 04:35:15 +02:00
|
|
|
buildStatement, err = aqueryHandler.pythonZipperActionBuildStatement(actionEntry, buildStatements)
|
2021-06-08 21:04:11 +02:00
|
|
|
} else if len(actionEntry.Arguments) < 1 {
|
2022-04-26 04:35:15 +02:00
|
|
|
return nil, nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic)
|
|
|
|
} else {
|
|
|
|
buildStatement, err = aqueryHandler.normalActionBuildStatement(actionEntry)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
2021-02-22 22:13:50 +01:00
|
|
|
}
|
2020-12-10 23:19:18 +01:00
|
|
|
buildStatements = append(buildStatements, buildStatement)
|
|
|
|
}
|
|
|
|
|
2022-05-12 22:43:01 +02:00
|
|
|
depsetsByHash := map[string]AqueryDepset{}
|
2022-05-27 23:40:21 +02:00
|
|
|
var depsets []AqueryDepset
|
2022-05-12 22:43:01 +02:00
|
|
|
for _, aqueryDepset := range aqueryHandler.depsetIdToAqueryDepset {
|
|
|
|
if prevEntry, hasKey := depsetsByHash[aqueryDepset.ContentHash]; hasKey {
|
|
|
|
// Two depsets collide on hash. Ensure that their contents are identical.
|
|
|
|
if !reflect.DeepEqual(aqueryDepset, prevEntry) {
|
|
|
|
return nil, nil, fmt.Errorf("Two different depsets have the same hash: %v, %v", prevEntry, aqueryDepset)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
depsetsByHash[aqueryDepset.ContentHash] = aqueryDepset
|
|
|
|
depsets = append(depsets, aqueryDepset)
|
2022-04-26 04:35:15 +02:00
|
|
|
}
|
|
|
|
}
|
2022-05-12 22:43:01 +02:00
|
|
|
|
|
|
|
// Build Statements and depsets must be sorted by their content hash to
|
|
|
|
// preserve determinism between builds (this will result in consistent ninja file
|
|
|
|
// output). Note they are not sorted by their original IDs nor their Bazel ordering,
|
|
|
|
// as Bazel gives nondeterministic ordering / identifiers in aquery responses.
|
|
|
|
sort.Slice(buildStatements, func(i, j int) bool {
|
|
|
|
// For build statements, compare output lists. In Bazel, each output file
|
|
|
|
// may only have one action which generates it, so this will provide
|
|
|
|
// a deterministic ordering.
|
|
|
|
outputs_i := buildStatements[i].OutputPaths
|
|
|
|
outputs_j := buildStatements[j].OutputPaths
|
|
|
|
if len(outputs_i) != len(outputs_j) {
|
|
|
|
return len(outputs_i) < len(outputs_j)
|
|
|
|
}
|
|
|
|
if len(outputs_i) == 0 {
|
|
|
|
// No outputs for these actions, so compare commands.
|
|
|
|
return buildStatements[i].Command < buildStatements[j].Command
|
|
|
|
}
|
|
|
|
// There may be multiple outputs, but the output ordering is deterministic.
|
|
|
|
return outputs_i[0] < outputs_j[0]
|
|
|
|
})
|
|
|
|
sort.Slice(depsets, func(i, j int) bool {
|
|
|
|
return depsets[i].ContentHash < depsets[j].ContentHash
|
|
|
|
})
|
2022-04-26 04:35:15 +02:00
|
|
|
return buildStatements, depsets, nil
|
|
|
|
}
|
|
|
|
|
2022-05-12 22:43:01 +02:00
|
|
|
// depsetContentHash computes and returns a SHA256 checksum of the contents of
|
|
|
|
// the given depset. This content hash may serve as the depset's identifier.
|
|
|
|
// Using a content hash for an identifier is superior for determinism. (For example,
|
|
|
|
// using an integer identifier which depends on the order in which the depsets are
|
|
|
|
// created would result in nondeterministic depset IDs.)
|
|
|
|
func depsetContentHash(directPaths []string, transitiveDepsetHashes []string) string {
|
|
|
|
h := sha256.New()
|
|
|
|
// Use newline as delimiter, as paths cannot contain newline.
|
|
|
|
h.Write([]byte(strings.Join(directPaths, "\n")))
|
2022-06-02 16:19:13 +02:00
|
|
|
h.Write([]byte(strings.Join(transitiveDepsetHashes, "")))
|
|
|
|
fullHash := base64.RawURLEncoding.EncodeToString(h.Sum(nil))
|
2022-05-12 22:43:01 +02:00
|
|
|
return fullHash
|
|
|
|
}
|
|
|
|
|
2022-05-27 23:40:21 +02:00
|
|
|
func (a *aqueryArtifactHandler) depsetContentHashes(inputDepsetIds []depsetId) ([]string, error) {
|
|
|
|
var hashes []string
|
2022-04-26 04:35:15 +02:00
|
|
|
for _, depsetId := range inputDepsetIds {
|
2022-05-27 16:45:00 +02:00
|
|
|
if aqueryDepset, exists := a.depsetIdToAqueryDepset[depsetId]; !exists {
|
2022-04-26 04:35:15 +02:00
|
|
|
return nil, fmt.Errorf("undefined input depsetId %d", depsetId)
|
2022-05-12 22:43:01 +02:00
|
|
|
} else {
|
|
|
|
hashes = append(hashes, aqueryDepset.ContentHash)
|
2022-04-26 04:35:15 +02:00
|
|
|
}
|
|
|
|
}
|
2022-05-12 22:43:01 +02:00
|
|
|
return hashes, nil
|
2022-04-26 04:35:15 +02:00
|
|
|
}
|
|
|
|
|
2022-05-27 16:45:00 +02:00
|
|
|
func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry action) (BuildStatement, error) {
|
2022-04-26 04:35:15 +02:00
|
|
|
command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ")
|
2022-05-27 16:45:00 +02:00
|
|
|
inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds)
|
2022-04-26 04:35:15 +02:00
|
|
|
if err != nil {
|
|
|
|
return BuildStatement{}, err
|
|
|
|
}
|
2022-05-27 16:45:00 +02:00
|
|
|
outputPaths, depfile, err := a.getOutputPaths(actionEntry)
|
2022-04-26 04:35:15 +02:00
|
|
|
if err != nil {
|
|
|
|
return BuildStatement{}, err
|
|
|
|
}
|
|
|
|
|
|
|
|
buildStatement := BuildStatement{
|
2022-05-12 22:43:01 +02:00
|
|
|
Command: command,
|
|
|
|
Depfile: depfile,
|
|
|
|
OutputPaths: outputPaths,
|
|
|
|
InputDepsetHashes: inputDepsetHashes,
|
|
|
|
Env: actionEntry.EnvironmentVariables,
|
|
|
|
Mnemonic: actionEntry.Mnemonic,
|
2022-04-26 04:35:15 +02:00
|
|
|
}
|
|
|
|
return buildStatement, nil
|
|
|
|
}
|
|
|
|
|
2022-05-27 16:45:00 +02:00
|
|
|
func (a *aqueryArtifactHandler) pythonZipperActionBuildStatement(actionEntry action, prevBuildStatements []BuildStatement) (BuildStatement, error) {
|
|
|
|
inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds)
|
2022-04-26 04:35:15 +02:00
|
|
|
if err != nil {
|
|
|
|
return BuildStatement{}, err
|
|
|
|
}
|
2022-05-27 16:45:00 +02:00
|
|
|
outputPaths, depfile, err := a.getOutputPaths(actionEntry)
|
2022-04-26 04:35:15 +02:00
|
|
|
if err != nil {
|
|
|
|
return BuildStatement{}, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(inputPaths) < 1 || len(outputPaths) != 1 {
|
|
|
|
return BuildStatement{}, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths)
|
|
|
|
}
|
|
|
|
command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ")
|
|
|
|
inputPaths, command = removePy3wrapperScript(inputPaths, command)
|
|
|
|
command = addCommandForPyBinaryRunfilesDir(command, inputPaths[0], outputPaths[0])
|
|
|
|
// Add the python zip file as input of the corresponding python binary stub script in Ninja build statements.
|
|
|
|
// In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input,
|
|
|
|
// which is not sufficient without the python zip file from which runfiles directory is created for py_binary.
|
|
|
|
//
|
|
|
|
// The following logic relies on that Bazel aquery output returns actions in the order that
|
|
|
|
// PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions
|
|
|
|
// in that order, the following logic might not find the build statement generated for Python binary
|
|
|
|
// stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output.
|
|
|
|
// See go/python-binary-host-mixed-build for more details.
|
|
|
|
pythonZipFilePath := outputPaths[0]
|
|
|
|
pyBinaryFound := false
|
2022-05-27 23:40:21 +02:00
|
|
|
for i := range prevBuildStatements {
|
2022-04-26 04:35:15 +02:00
|
|
|
if len(prevBuildStatements[i].OutputPaths) == 1 && prevBuildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath {
|
|
|
|
prevBuildStatements[i].InputPaths = append(prevBuildStatements[i].InputPaths, pythonZipFilePath)
|
|
|
|
pyBinaryFound = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !pyBinaryFound {
|
|
|
|
return BuildStatement{}, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths)
|
|
|
|
}
|
|
|
|
|
|
|
|
buildStatement := BuildStatement{
|
|
|
|
Command: command,
|
|
|
|
Depfile: depfile,
|
|
|
|
OutputPaths: outputPaths,
|
|
|
|
InputPaths: inputPaths,
|
|
|
|
Env: actionEntry.EnvironmentVariables,
|
|
|
|
Mnemonic: actionEntry.Mnemonic,
|
|
|
|
}
|
|
|
|
return buildStatement, nil
|
|
|
|
}
|
|
|
|
|
2022-05-27 16:45:00 +02:00
|
|
|
func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry action) (BuildStatement, error) {
|
|
|
|
outputPaths, depfile, err := a.getOutputPaths(actionEntry)
|
2022-04-26 04:35:15 +02:00
|
|
|
if err != nil {
|
|
|
|
return BuildStatement{}, err
|
|
|
|
}
|
|
|
|
if len(outputPaths) != 1 {
|
|
|
|
return BuildStatement{}, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths)
|
|
|
|
}
|
|
|
|
expandedTemplateContent := expandTemplateContent(actionEntry)
|
|
|
|
// The expandedTemplateContent is escaped for being used in double quotes and shell unescape,
|
|
|
|
// and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might
|
|
|
|
// change \n to space and mess up the format of Python programs.
|
|
|
|
// sed is used to convert \\n back to \n before saving to output file.
|
|
|
|
// See go/python-binary-host-mixed-build for more details.
|
|
|
|
command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`,
|
|
|
|
escapeCommandlineArgument(expandedTemplateContent), outputPaths[0])
|
2022-05-27 16:45:00 +02:00
|
|
|
inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds)
|
2022-04-26 04:35:15 +02:00
|
|
|
if err != nil {
|
|
|
|
return BuildStatement{}, err
|
|
|
|
}
|
|
|
|
|
|
|
|
buildStatement := BuildStatement{
|
2022-05-12 22:43:01 +02:00
|
|
|
Command: command,
|
|
|
|
Depfile: depfile,
|
|
|
|
OutputPaths: outputPaths,
|
|
|
|
InputDepsetHashes: inputDepsetHashes,
|
|
|
|
Env: actionEntry.EnvironmentVariables,
|
|
|
|
Mnemonic: actionEntry.Mnemonic,
|
2022-04-26 04:35:15 +02:00
|
|
|
}
|
|
|
|
return buildStatement, nil
|
|
|
|
}
|
|
|
|
|
2022-05-27 16:45:00 +02:00
|
|
|
func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry action) (BuildStatement, error) {
|
|
|
|
outputPaths, depfile, err := a.getOutputPaths(actionEntry)
|
2022-04-26 04:35:15 +02:00
|
|
|
if err != nil {
|
|
|
|
return BuildStatement{}, err
|
|
|
|
}
|
|
|
|
|
2022-05-27 16:45:00 +02:00
|
|
|
inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds)
|
2022-04-26 04:35:15 +02:00
|
|
|
if err != nil {
|
|
|
|
return BuildStatement{}, err
|
|
|
|
}
|
|
|
|
if len(inputPaths) != 1 || len(outputPaths) != 1 {
|
|
|
|
return BuildStatement{}, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths)
|
|
|
|
}
|
|
|
|
out := outputPaths[0]
|
|
|
|
outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out))
|
|
|
|
out = proptools.ShellEscapeIncludingSpaces(out)
|
|
|
|
in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0]))
|
|
|
|
// Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`).
|
|
|
|
command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in)
|
|
|
|
symlinkPaths := outputPaths[:]
|
|
|
|
|
|
|
|
buildStatement := BuildStatement{
|
|
|
|
Command: command,
|
|
|
|
Depfile: depfile,
|
|
|
|
OutputPaths: outputPaths,
|
|
|
|
InputPaths: inputPaths,
|
|
|
|
Env: actionEntry.EnvironmentVariables,
|
|
|
|
Mnemonic: actionEntry.Mnemonic,
|
|
|
|
SymlinkPaths: symlinkPaths,
|
|
|
|
}
|
|
|
|
return buildStatement, nil
|
|
|
|
}
|
|
|
|
|
2022-05-27 16:45:00 +02:00
|
|
|
func (a *aqueryArtifactHandler) getOutputPaths(actionEntry action) (outputPaths []string, depfile *string, err error) {
|
2022-04-26 04:35:15 +02:00
|
|
|
for _, outputId := range actionEntry.OutputIds {
|
2022-05-27 16:45:00 +02:00
|
|
|
outputPath, exists := a.artifactIdToPath[outputId]
|
2022-04-26 04:35:15 +02:00
|
|
|
if !exists {
|
|
|
|
err = fmt.Errorf("undefined outputId %d", outputId)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
ext := filepath.Ext(outputPath)
|
|
|
|
if ext == ".d" {
|
|
|
|
if depfile != nil {
|
|
|
|
err = fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath)
|
|
|
|
return
|
|
|
|
} else {
|
|
|
|
depfile = &outputPath
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
outputPaths = append(outputPaths, outputPath)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return
|
2020-12-10 23:19:18 +01:00
|
|
|
}
|
2020-12-23 18:02:11 +01:00
|
|
|
|
2021-11-04 23:58:12 +01:00
|
|
|
// expandTemplateContent substitutes the tokens in a template.
|
|
|
|
func expandTemplateContent(actionEntry action) string {
|
|
|
|
replacerString := []string{}
|
|
|
|
for _, pair := range actionEntry.Substitutions {
|
|
|
|
value := pair.Value
|
2022-05-27 23:40:21 +02:00
|
|
|
if val, ok := templateActionOverriddenTokens[pair.Key]; ok {
|
2021-11-04 23:58:12 +01:00
|
|
|
value = val
|
|
|
|
}
|
|
|
|
replacerString = append(replacerString, pair.Key, value)
|
|
|
|
}
|
|
|
|
replacer := strings.NewReplacer(replacerString...)
|
|
|
|
return replacer.Replace(actionEntry.TemplateContent)
|
|
|
|
}
|
|
|
|
|
|
|
|
func escapeCommandlineArgument(str string) string {
|
|
|
|
// \->\\, $->\$, `->\`, "->\", \n->\\n, '->'"'"'
|
|
|
|
replacer := strings.NewReplacer(
|
|
|
|
`\`, `\\`,
|
|
|
|
`$`, `\$`,
|
|
|
|
"`", "\\`",
|
|
|
|
`"`, `\"`,
|
|
|
|
"\n", "\\n",
|
|
|
|
`'`, `'"'"'`,
|
|
|
|
)
|
|
|
|
return replacer.Replace(str)
|
|
|
|
}
|
|
|
|
|
|
|
|
// removePy3wrapperScript removes py3wrapper.sh from the input paths and command of the action of
|
|
|
|
// creating python zip file in mixed build mode. py3wrapper.sh is returned as input by aquery but
|
|
|
|
// there is no action returned by aquery for creating it. So in mixed build "python3" is used
|
|
|
|
// as the PYTHON_BINARY in python binary stub script, and py3wrapper.sh is not needed and should be
|
|
|
|
// removed from input paths and command of creating python zip file.
|
|
|
|
// See go/python-binary-host-mixed-build for more details.
|
|
|
|
// TODO(b/205879240) remove this after py3wrapper.sh could be created in the mixed build mode.
|
2022-04-26 04:35:15 +02:00
|
|
|
func removePy3wrapperScript(inputPaths []string, command string) (newInputPaths []string, newCommand string) {
|
2021-11-04 23:58:12 +01:00
|
|
|
// Remove from inputs
|
|
|
|
filteredInputPaths := []string{}
|
2022-04-26 04:35:15 +02:00
|
|
|
for _, path := range inputPaths {
|
2021-11-04 23:58:12 +01:00
|
|
|
if !strings.HasSuffix(path, py3wrapperFileName) {
|
|
|
|
filteredInputPaths = append(filteredInputPaths, path)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
newInputPaths = filteredInputPaths
|
|
|
|
|
|
|
|
// Remove from command line
|
|
|
|
var re = regexp.MustCompile(`\S*` + py3wrapperFileName)
|
2022-04-26 04:35:15 +02:00
|
|
|
newCommand = re.ReplaceAllString(command, "")
|
2021-11-04 23:58:12 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// addCommandForPyBinaryRunfilesDir adds commands creating python binary runfiles directory.
|
|
|
|
// runfiles directory is created by using MANIFEST file and MANIFEST file is the output of
|
|
|
|
// SourceSymlinkManifest action is in aquery output of Bazel py_binary targets,
|
|
|
|
// but since SourceSymlinkManifest doesn't contain sufficient information
|
|
|
|
// so MANIFEST file could not be created, which also blocks the creation of runfiles directory.
|
|
|
|
// See go/python-binary-host-mixed-build for more details.
|
|
|
|
// TODO(b/197135294) create runfiles directory from MANIFEST file once it can be created from SourceSymlinkManifest action.
|
2022-04-26 04:35:15 +02:00
|
|
|
func addCommandForPyBinaryRunfilesDir(oldCommand string, zipperCommandPath, zipFilePath string) string {
|
2021-11-04 23:58:12 +01:00
|
|
|
// Unzip the zip file, zipFilePath looks like <python_binary>.zip
|
|
|
|
runfilesDirName := zipFilePath[0:len(zipFilePath)-4] + ".runfiles"
|
|
|
|
command := fmt.Sprintf("%s x %s -d %s", zipperCommandPath, zipFilePath, runfilesDirName)
|
|
|
|
// Create a symbolic link in <python_binary>.runfiles/, which is the expected structure
|
|
|
|
// when running the python binary stub script.
|
|
|
|
command += fmt.Sprintf(" && ln -sf runfiles/__main__ %s", runfilesDirName)
|
2022-04-26 04:35:15 +02:00
|
|
|
return oldCommand + " && " + command
|
2021-11-04 23:58:12 +01:00
|
|
|
}
|
|
|
|
|
2021-06-08 21:04:11 +02:00
|
|
|
func isSymlinkAction(a action) bool {
|
2022-05-13 22:55:35 +02:00
|
|
|
return a.Mnemonic == "Symlink" || a.Mnemonic == "SolibSymlink" || a.Mnemonic == "ExecutableSymlink"
|
2021-06-08 21:04:11 +02:00
|
|
|
}
|
|
|
|
|
2021-11-04 23:58:12 +01:00
|
|
|
func isTemplateExpandAction(a action) bool {
|
|
|
|
return a.Mnemonic == "TemplateExpand"
|
|
|
|
}
|
|
|
|
|
|
|
|
func isPythonZipperAction(a action) bool {
|
|
|
|
return a.Mnemonic == "PythonZipper"
|
|
|
|
}
|
|
|
|
|
2021-02-22 22:13:50 +01:00
|
|
|
func shouldSkipAction(a action) bool {
|
2021-06-08 21:04:11 +02:00
|
|
|
// TODO(b/180945121): Handle complex symlink actions.
|
|
|
|
if a.Mnemonic == "SymlinkTree" || a.Mnemonic == "SourceSymlinkManifest" {
|
2021-02-22 22:13:50 +01:00
|
|
|
return true
|
|
|
|
}
|
2021-05-18 18:31:25 +02:00
|
|
|
// Middleman actions are not handled like other actions; they are handled separately as a
|
|
|
|
// preparatory step so that their inputs may be relayed to actions depending on middleman
|
|
|
|
// artifacts.
|
2021-02-22 22:13:50 +01:00
|
|
|
if a.Mnemonic == "Middleman" {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
// Skip "Fail" actions, which are placeholder actions designed to always fail.
|
|
|
|
if a.Mnemonic == "Fail" {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
// TODO(b/180946980): Handle FileWrite. The aquery proto currently contains no information
|
|
|
|
// about the contents that are written.
|
|
|
|
if a.Mnemonic == "FileWrite" {
|
|
|
|
return true
|
|
|
|
}
|
2022-05-18 00:13:28 +02:00
|
|
|
if a.Mnemonic == "BaselineCoverage" {
|
|
|
|
return true
|
|
|
|
}
|
2021-02-22 22:13:50 +01:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2022-05-27 23:40:21 +02:00
|
|
|
func expandPathFragment(id pathFragmentId, pathFragmentsMap map[pathFragmentId]pathFragment) (string, error) {
|
|
|
|
var labels []string
|
2020-12-23 18:02:11 +01:00
|
|
|
currId := id
|
|
|
|
// Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node.
|
|
|
|
for currId > 0 {
|
|
|
|
currFragment, ok := pathFragmentsMap[currId]
|
|
|
|
if !ok {
|
2021-01-15 18:22:41 +01:00
|
|
|
return "", fmt.Errorf("undefined path fragment id %d", currId)
|
2020-12-23 18:02:11 +01:00
|
|
|
}
|
|
|
|
labels = append([]string{currFragment.Label}, labels...)
|
2021-06-08 21:04:11 +02:00
|
|
|
if currId == currFragment.ParentId {
|
|
|
|
return "", fmt.Errorf("Fragment cannot refer to itself as parent %#v", currFragment)
|
|
|
|
}
|
2020-12-23 18:02:11 +01:00
|
|
|
currId = currFragment.ParentId
|
|
|
|
}
|
|
|
|
return filepath.Join(labels...), nil
|
|
|
|
}
|