diff --git a/go/pkg/fakes/server.go b/go/pkg/fakes/server.go index 989fcea6a..147bd1c1f 100644 --- a/go/pkg/fakes/server.go +++ b/go/pkg/fakes/server.go @@ -28,7 +28,7 @@ import ( repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" bsgrpc "google.golang.org/genproto/googleapis/bytestream" bspb "google.golang.org/genproto/googleapis/bytestream" - anypb "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/anypb" dpb "google.golang.org/protobuf/types/known/durationpb" tspb "google.golang.org/protobuf/types/known/timestamppb" ) @@ -307,6 +307,26 @@ func (f *InputFile) apply(ac *repb.ActionResult, s *Server, execRoot string) err return nil } +// InputSymlink (Path -> Target) to be made available to the fake action. +type InputSymlink struct { + Path string // newname + TargetContent string + Target string // oldname +} + +// Apply puts the target file in the fake CAS and create a symlink in OS. +func (ins *InputSymlink) apply(ac *repb.ActionResult, s *Server, execRoot string) error { + inf := InputFile{Path: ins.Target, Contents: ins.TargetContent} + if err := inf.apply(ac, s, execRoot); err != nil { + return err + } + // create a symlink from old name (target) to new name (path) + if err := os.Symlink(filepath.Join(execRoot, ins.Target), filepath.Join(execRoot, ins.Path)); err != nil { + return fmt.Errorf("error creating symlink: %w", err) + } + return nil +} + // OutputFile is to be added as an output of the fake action. type OutputFile struct { Path string diff --git a/go/pkg/tool/tool.go b/go/pkg/tool/tool.go index 2e202f416..754d397a3 100644 --- a/go/pkg/tool/tool.go +++ b/go/pkg/tool/tool.go @@ -706,7 +706,7 @@ func (c *Client) formatAction(ctx context.Context, actionProto *repb.Action, res } showActionRes.WriteString("\nInputs\n======\n") log.Infof("Fetching input tree from input root digest..") - inpTree, _, err := c.getInputTree(ctx, actionProto.GetInputRootDigest()) + inpTree, _, _, err := c.getInputTree(ctx, actionProto.GetInputRootDigest()) if err != nil { showActionRes.WriteString("Failed to fetch input tree:\n") showActionRes.WriteString(err.Error()) @@ -773,7 +773,7 @@ func (c *Client) getOutputs(ctx context.Context, actionRes *repb.ActionResult) ( return "", err } - outputs, _, err := c.flattenTree(ctx, outDirTree) + outputs, _, _, err := c.flattenTree(ctx, outDirTree) if err != nil { return "", err } @@ -784,41 +784,41 @@ func (c *Client) getOutputs(ctx context.Context, actionRes *repb.ActionResult) ( return res.String(), nil } -func (c *Client) getInputTree(ctx context.Context, root *repb.Digest) (string, []string, error) { +func (c *Client) getInputTree(ctx context.Context, root *repb.Digest) (string, map[string]string, map[string]string, error) { var res bytes.Buffer dg, err := digest.NewFromProto(root) if err != nil { - return "", nil, err + return "", nil, nil, fmt.Errorf("failed generate Digest object from proto: %v", err) } res.WriteString(fmt.Sprintf("[Root directory digest: %v]", dg)) dirs, err := c.GrpcClient.GetDirectoryTree(ctx, root) if err != nil { - return "", nil, err + return "", nil, nil, fmt.Errorf("failed to get dir tree: %v", err) } if len(dirs) == 0 { - return "", nil, fmt.Errorf("Empty directories returned by GetTree for %v", dg) + return "", nil, nil, fmt.Errorf("empty directories returned by GetTree for %v", dg) } t := &repb.Tree{ Root: dirs[0], Children: dirs, } - inputs, paths, err := c.flattenTree(ctx, t) + inputs, paths, symlinks, err := c.flattenTree(ctx, t) if err != nil { - return "", nil, err + return "", nil, nil, err } res.WriteString("\n") res.WriteString(inputs) - return res.String(), paths, nil + return res.String(), paths, symlinks, nil } -func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []string, error) { +func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, map[string]string, map[string]string, error) { var res bytes.Buffer outputs, err := c.GrpcClient.FlattenTree(t, "") if err != nil { - return "", nil, err + return "", nil, nil, fmt.Errorf("failed falt tree: %v", err) } // Sort the values by path. paths := make([]string, 0, len(outputs)) @@ -830,8 +830,11 @@ func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []strin paths = append(paths, path) } sort.Strings(paths) + pathToDgs := make(map[string]string, len(paths)) + symToDgs := map[string]string{} for _, path := range paths { output := outputs[path] + dg := output.Digest var np string if output.NodeProperties != nil { np = fmt.Sprintf(" [Node properties: %v]", prototext.MarshalOptions{Multiline: false}.Format(output.NodeProperties)) @@ -840,11 +843,18 @@ func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []strin res.WriteString(fmt.Sprintf("%v: [Directory digest: %v]%s\n", path, output.Digest, np)) } else if output.SymlinkTarget != "" { res.WriteString(fmt.Sprintf("%v: [Symlink digest: %v, Symlink Target: %v]%s\n", path, output.Digest, output.SymlinkTarget, np)) + path = path + "->" + output.SymlinkTarget + if o, ok := outputs[output.SymlinkTarget]; ok { + dg = o.Digest + } + symToDgs[path] = fmt.Sprintf("%v", dg) + continue } else { res.WriteString(fmt.Sprintf("%v: [File digest: %v]%s\n", path, output.Digest, np)) } + pathToDgs[path] = fmt.Sprintf("%v", dg) } - return res.String(), paths, nil + return res.String(), pathToDgs, symToDgs, nil } func (c *Client) getActionResult(ctx context.Context, actionDigest string) (*repb.ActionResult, error) { @@ -862,3 +872,88 @@ func (c *Client) getActionResult(ctx context.Context, actionDigest string) (*rep } return resPb, nil } + +// FlatActionIO is a collection of input root Digest, input paths +// (with value as digests), and output files/dirs (with value as digests) of an +// action. For a symlink, the key is `->`, the value is the dg of +// the target file. +type FlatActionIO struct { + RootDg string + InputPaths map[string]string + InputPathSymlinks map[string]string + OutputFiles map[string]string + OutputDirs map[string]string + OutputFileSymlinks map[string]string + OutputDirSymlinks map[string]string +} + +// FlattenActionIO returns the Inputs and Outputs of an action Digest. +func (c *Client) FlattenActionIO(ctx context.Context, actionDigest string) (*FlatActionIO, error) { + acDg, err := digest.NewFromString(actionDigest) + if err != nil { + return nil, fmt.Errorf("error creating action digest: %w", err) + } + actionProto := &repb.Action{} + + if _, err := c.GrpcClient.ReadProto(ctx, acDg, actionProto); err != nil { + return nil, fmt.Errorf("error reading from proto: %w", err) + } + rootDg, err := digest.NewFromProto(actionProto.GetInputRootDigest()) + if err != nil { + return nil, fmt.Errorf("error getting input root digest: %w", err) + } + _, inputPaths, inputSymlinks, err := c.getInputTree(ctx, actionProto.GetInputRootDigest()) + if err != nil { + return nil, err + } + // If getActionResult failed (say, error in checking action cache: + // http://shortn/_dMVjMYPOZE), leave the output as empty, log the warning + // message, and proceed with the inputs' information. + resPb, err := c.getActionResult(ctx, actionDigest) + if err != nil { + log.Warningf("Error in getting action result for digest %v: %v\n", actionDigest, err) + } + + outputFiles := map[string]string{} + outputDirs := map[string]string{} + outputFileSymlinks := map[string]string{} + outputDirSymlinks := map[string]string{} + for _, f := range resPb.GetOutputFiles() { + if f != nil { + dg, err := digest.NewFromProto(f.GetDigest()) + if err != nil { + log.Errorf("error creating Digest from proto %v: %w", f.GetDigest(), err) + } + outputFiles[f.GetPath()] = fmt.Sprintf("%v", dg) + } + } + for _, d := range resPb.GetOutputDirectories() { + if d != nil { + dg, err := digest.NewFromProto(d.GetTreeDigest()) + if err != nil { + log.Errorf("error creating Digest from proto %v: %w", d.GetTreeDigest(), err) + } + outputDirs[d.GetPath()] = fmt.Sprintf("%v", dg) + } + } + for _, fs := range resPb.GetOutputFileSymlinks() { + if fs != nil { + outputFileSymlinks[fs.GetPath()+"->"+fs.GetTarget()] = outputFiles[fs.GetTarget()] + } + } + for _, ds := range resPb.GetOutputDirectorySymlinks() { + if ds != nil { + outputDirSymlinks[ds.GetPath()+"->"+ds.GetTarget()] = outputDirs[ds.GetTarget()] + } + } + + return &FlatActionIO{ + RootDg: fmt.Sprintf("%v", rootDg), + InputPaths: inputPaths, + InputPathSymlinks: inputSymlinks, + OutputFiles: outputFiles, + OutputDirs: outputDirs, + OutputFileSymlinks: outputFileSymlinks, + OutputDirSymlinks: outputDirSymlinks, + }, nil +} diff --git a/go/pkg/tool/tool_test.go b/go/pkg/tool/tool_test.go index 8ada5adf2..35b3cfb3b 100644 --- a/go/pkg/tool/tool_test.go +++ b/go/pkg/tool/tool_test.go @@ -166,7 +166,7 @@ func TestTool_DownloadAction(t *testing.T) { OutputFiles: []string{"a/b/out"}, Platform: &repb.Platform{ Properties: []*repb.Platform_Property{ - &repb.Platform_Property{ + { Name: "container-image", Value: "foo", }, @@ -444,3 +444,62 @@ func TestTool_UploadBlob(t *testing.T) { t.Fatalf("Expected 1 write for blob '%v', got %v", dg.String(), cas.BlobWrites(dg)) } } + +func TestTool_FlattenActionIO(t *testing.T) { + e, cleanup := fakes.NewTestEnv(t) + defer cleanup() + cmd := &command.Command{ + Args: []string{"tool"}, + ExecRoot: e.ExecRoot, + InputSpec: &command.InputSpec{ + Inputs: []string{"foo.c", "bar.c"}, + SymlinkBehavior: command.PreserveSymlink, + }, + } + opt := command.DefaultExecutionOptions() + _, acDg, _, _ := e.Set( + cmd, + opt, + &command.Result{Status: command.CacheHitResultStatus}, + &fakes.InputFile{Path: "foo.c", Contents: "foo"}, + &fakes.InputSymlink{Path: "bar.c", TargetContent: "bar", Target: "previous_dir/old_target_file"}, + &fakes.OutputFile{Path: "a/b/out", Contents: "foo"}, + &fakes.OutputSymlink{Path: "a/b/sl", Target: "a/b/out"}, + ) + toolClient := &Client{GrpcClient: e.Client.GrpcClient} + tmpDir := t.TempDir() + io, err := toolClient.FlattenActionIO(context.Background(), acDg.String()) + if err != nil { + t.Fatalf("DownloadActionResult(%v,%v) failed: %v", acDg.String(), tmpDir, err) + } + + getInputRootDg := io.RootDg + wantInputRootDg := "ea520b417ef2887249b4ea2d24ff64f5d7d6c419eb1a0ce2067c39ece5c37b56/206" + if diff := cmp.Diff(wantInputRootDg, getInputRootDg); diff != "" { + t.Errorf("FlattenActionIO returned diff in Inputs' list: (-want +got)\n%s", diff) + } + + getInputPaths := io.InputPaths + wantInputsPaths := map[string]string{"foo.c": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3", + "previous_dir/old_target_file": "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9/3"} + if diff := cmp.Diff(wantInputsPaths, getInputPaths); diff != "" { + t.Errorf("FlattenActionIO returned diff in InputPaths: (-want +got)\n%s", diff) + } + getInputPathSymlinks := io.InputPathSymlinks + wantInputPathSymlinks := map[string]string{"bar.c->previous_dir/old_target_file": "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9/3"} + if diff := cmp.Diff(wantInputPathSymlinks, getInputPathSymlinks); diff != "" { + t.Errorf("FlattenActionIO returned diff in InputPathSymlinks: (-want +got)\n%s", diff) + } + + getOutputFiles := io.OutputFiles + getOutputFileSymlinks := io.OutputFileSymlinks + + wantOutputFiles := map[string]string{"a/b/out": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3"} + wantOutputFileSymlinks := map[string]string{"a/b/sl->a/b/out": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3"} + if diff := cmp.Diff(wantOutputFiles, getOutputFiles); diff != "" { + t.Errorf("FlattenActionIO returned diff in Output Files: (-want +got)\n%s", diff) + } + if diff := cmp.Diff(wantOutputFileSymlinks, getOutputFileSymlinks); diff != "" { + t.Errorf("FlattenActionIO returned diff in Output File Symlinks: (-want +got)\n%s", diff) + } +}