Copy NodeProperties instead of importing into command.proto (bazelbui…

…ld#503) * Copy definition of NodeProperties instead of importing into command.proto * Fix linter issues * Update command.proto to add reasoning for avoiding import * Update command.proto * Fix typos
mrahs · Sep 5, 2023 · bb66956 · bb66956
1 parent 2caea49
commit bb66956
Show file tree

Hide file tree

Showing 9 changed files with 485 additions and 259 deletions.
diff --git a/go/api/command/BUILD.bazel b/go/api/command/BUILD.bazel
@@ -7,8 +7,8 @@ proto_library(
     srcs = ["command.proto"],
     visibility = ["//visibility:public"],
     deps = [
-      "@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto",
       "@com_google_protobuf//:timestamp_proto",
+      "@com_google_protobuf//:wrappers_proto",
     ],
 )
 
@@ -17,7 +17,6 @@ go_proto_library(
     importpath = "github.com/bazelbuild/remote-apis-sdks/go/api/command",
     proto = ":command_proto",
     visibility = ["//visibility:public"],
-    deps = ["@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto"],
 )
 
 go_library(

diff --git a/go/api/command/command.pb.go b/go/api/command/command.pb.go
diff --git a/go/api/command/command.proto b/go/api/command/command.proto
@@ -2,8 +2,8 @@ syntax = "proto3";
 
 package cmd;
 
-import "build/bazel/remote/execution/v2/remote_execution.proto";
 import "google/protobuf/timestamp.proto";
+import "google/protobuf/wrappers.proto";
 
 // A command to execute locally or remotely.
 message Command {
@@ -126,7 +126,33 @@ message InputSpec {
   SymlinkBehaviorType.Value symlink_behavior = 6;
 
   // Node properties of inputs.
-  map<string,build.bazel.remote.execution.v2.NodeProperties> input_node_properties = 7;
+  map<string,NodeProperties> input_node_properties = 7;
+}
+
+// A copy of NodeProperties from https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto
+// to avoid importing it as a proto dependency.
+// Importing the messages causes resource exhaustion on high parallelism builds.
+message NodeProperties {
+  // A list of string-based
+  // [NodeProperties][build.bazel.remote.execution.v2.NodeProperty].
+  repeated NodeProperty properties = 1;
+
+  // The file's last modification timestamp.
+  google.protobuf.Timestamp mtime = 2;
+
+  // The UNIX file mode, e.g., 0755.
+  google.protobuf.UInt32Value unix_mode = 3;
+}
+
+// A copy of NodeProperty from https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto
+// to avoid importing it as a proto dependency.
+// Importing the messages causes resource exhaustion on high parallelism builds.
+message NodeProperty {
+    // The property name.
+    string name = 1;
+
+    // The property value.
+    string value = 2;
 }
 
 message OutputSpec {

diff --git a/go/pkg/client/BUILD.bazel b/go/pkg/client/BUILD.bazel
@@ -16,6 +16,7 @@ go_library(
     importpath = "github.com/bazelbuild/remote-apis-sdks/go/pkg/client",
     visibility = ["//visibility:public"],
     deps = [
+        "//go/api/command",
         "//go/pkg/actas",
         "//go/pkg/balancer",
         "//go/pkg/balancer/proto",

diff --git a/go/pkg/client/tree.go b/go/pkg/client/tree.go
@@ -10,6 +10,7 @@ import (
 	"sort"
 	"strings"
 
+	cpb "github.com/bazelbuild/remote-apis-sdks/go/api/command"
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/command"
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/digest"
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/filemetadata"
@@ -41,7 +42,7 @@ type fileSysNode struct {
 	file                 *fileNode
 	emptyDirectoryMarker bool
 	symlink              *symlinkNode
-	nodeProperties       *repb.NodeProperties
+	nodeProperties       *cpb.NodeProperties
 }
 
 // TreeStats contains various stats/metadata of the constructed Merkle tree.
@@ -166,7 +167,7 @@ func getExecRootRelPaths(absPath, execRoot, workingDir, remoteWorkingDir string)
 
 // loadFiles reads all files specified by the given InputSpec (descending into subdirectories
 // recursively), and loads their contents into the provided map.
-func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*command.InputExclusion, filesToProcess []string, fs map[string]*fileSysNode, cache filemetadata.Cache, opts *TreeSymlinkOpts, nodeProperties map[string]*repb.NodeProperties) error {
+func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*command.InputExclusion, filesToProcess []string, fs map[string]*fileSysNode, cache filemetadata.Cache, opts *TreeSymlinkOpts, nodeProperties map[string]*cpb.NodeProperties) error {
 	if opts == nil {
 		opts = DefaultTreeSymlinkOpts()
 	}
@@ -411,7 +412,7 @@ func packageTree(t *treeNode, stats *TreeStats, prefix string, tree map[string]d
 		// A node can have exactly one of file/symlink/emptyDirectoryMarker.
 		if n.file != nil {
 			dg := n.file.ue.Digest
-			dir.Files = append(dir.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable, NodeProperties: n.nodeProperties})
+			dir.Files = append(dir.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable, NodeProperties: command.NodePropertiesToAPI(n.nodeProperties)})
 			blobs[dg] = n.file.ue
 			stats.InputFiles++
 			stats.TotalInputBytes += dg.Size
@@ -421,7 +422,7 @@ func packageTree(t *treeNode, stats *TreeStats, prefix string, tree map[string]d
 			continue
 		}
 		if n.symlink != nil {
-			dir.Symlinks = append(dir.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target, NodeProperties: n.nodeProperties})
+			dir.Symlinks = append(dir.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target, NodeProperties: command.NodePropertiesToAPI(n.nodeProperties)})
 			stats.InputSymlinks++
 		}
 	}
@@ -570,12 +571,12 @@ func packageDirectories(t *treeNode) (root *repb.Directory, files map[digest.Dig
 		// A node can have exactly one of file/symlink/emptyDirectoryMarker.
 		if n.file != nil {
 			dg := n.file.ue.Digest
-			root.Files = append(root.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable, NodeProperties: n.nodeProperties})
+			root.Files = append(root.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable, NodeProperties: command.NodePropertiesToAPI(n.nodeProperties)})
 			files[dg] = n.file.ue
 			continue
 		}
 		if n.symlink != nil {
-			root.Symlinks = append(root.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target, NodeProperties: n.nodeProperties})
+			root.Symlinks = append(root.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target, NodeProperties: command.NodePropertiesToAPI(n.nodeProperties)})
 		}
 	}
 	sort.Slice(root.Files, func(i, j int) bool { return root.Files[i].Name < root.Files[j].Name })
@@ -587,7 +588,7 @@ func packageDirectories(t *treeNode) (root *repb.Directory, files map[digest.Dig
 // ComputeOutputsToUpload transforms the provided local output paths into uploadable Chunkers.
 // The paths have to be relative to execRoot.
 // It also populates the remote ActionResult, packaging output directories as trees where required.
-func (c *Client) ComputeOutputsToUpload(execRoot, workingDir string, paths []string, cache filemetadata.Cache, sb command.SymlinkBehaviorType, nodeProperties map[string]*repb.NodeProperties) (map[digest.Digest]*uploadinfo.Entry, *repb.ActionResult, error) {
+func (c *Client) ComputeOutputsToUpload(execRoot, workingDir string, paths []string, cache filemetadata.Cache, sb command.SymlinkBehaviorType, nodeProperties map[string]*cpb.NodeProperties) (map[digest.Digest]*uploadinfo.Entry, *repb.ActionResult, error) {
 	outs := make(map[digest.Digest]*uploadinfo.Entry)
 	resPb := &repb.ActionResult{}
 	for _, path := range paths {
@@ -610,7 +611,7 @@ func (c *Client) ComputeOutputsToUpload(execRoot, workingDir string, paths []str
 			// A regular file.
 			ue := uploadinfo.EntryFromFile(meta.Digest, absPath)
 			outs[meta.Digest] = ue
-			resPb.OutputFiles = append(resPb.OutputFiles, &repb.OutputFile{Path: normPath, Digest: meta.Digest.ToProto(), IsExecutable: meta.IsExecutable, NodeProperties: nodeProperties[normPath]})
+			resPb.OutputFiles = append(resPb.OutputFiles, &repb.OutputFile{Path: normPath, Digest: meta.Digest.ToProto(), IsExecutable: meta.IsExecutable, NodeProperties: command.NodePropertiesToAPI(nodeProperties[normPath])})
 			continue
 		}
 		// A directory.