From f8e7488db11ed24a2ade52e844a249eb552d0f22 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Tue, 6 Aug 2024 15:52:54 +0200 Subject: [PATCH 1/6] Override JobsAPI.GetRun --- service/jobs/ext_api.go | 30 ++++++++++++++++++++++++++++++ service/jobs/model.go | 11 +++++++++++ 2 files changed, 41 insertions(+) create mode 100644 service/jobs/ext_api.go diff --git a/service/jobs/ext_api.go b/service/jobs/ext_api.go new file mode 100644 index 000000000..5c64d813d --- /dev/null +++ b/service/jobs/ext_api.go @@ -0,0 +1,30 @@ +package jobs + +import "context" + +func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, error) { + run, err := a.jobsImpl.GetRun(ctx, request) + if err != nil { + return nil, err + } + + isPaginatingIterations := run.Iterations != nil && len(run.Iterations) > 0 + + for len(run.NextPageToken) != 0 { + request.PageToken = run.NextPageToken + nextRun, err := a.jobsImpl.GetRun(ctx, request) + if err != nil { + return nil, err + } + + if isPaginatingIterations { + run.Iterations = append(run.Iterations, nextRun.Iterations...) + } else { + run.Tasks = append(run.Tasks, nextRun.Tasks...) + } + run.NextPageToken = nextRun.NextPageToken + } + + run.PrevPageToken = "" + return run, nil +} diff --git a/service/jobs/model.go b/service/jobs/model.go index 029722a4d..e7651e656 100755 --- a/service/jobs/model.go +++ b/service/jobs/model.go @@ -753,6 +753,10 @@ type GetRunRequest struct { IncludeHistory bool `json:"-" url:"include_history,omitempty"` // Whether to include resolved parameter values in the response. IncludeResolvedValues bool `json:"-" url:"include_resolved_values,omitempty"` + // To list the next page or the previous page of job tasks, set this field + // to the value of the `next_page_token` or `prev_page_token` returned in + // the GetJob response. + PageToken string `json:"-" url:"page_token,omitempty"` // The canonical identifier of the run for which to retrieve the metadata. // This field is required. RunId int64 `json:"-" url:"run_id"` @@ -2153,6 +2157,9 @@ type Run struct { // Note: dbt and SQL File tasks support only version-controlled sources. If // dbt or SQL File tasks are used, `git_source` must be defined on the job. GitSource *GitSource `json:"git_source,omitempty"` + // Only populated by for-each iterations. The parent for-each task is + // located in tasks array. + Iterations []RunTask `json:"iterations,omitempty"` // A list of job cluster specifications that can be shared and reused by // tasks of this job. Libraries cannot be declared in a shared job cluster. // You must declare dependent libraries in task settings. @@ -2161,6 +2168,8 @@ type Run struct { JobId int64 `json:"job_id,omitempty"` // Job-level parameters used in the run JobParameters []JobParameter `json:"job_parameters,omitempty"` + // A token that can be used to list the next page of sub-resources. + NextPageToken string `json:"next_page_token,omitempty"` // A unique identifier for this job run. This is set to the same value as // `run_id`. NumberInJob int64 `json:"number_in_job,omitempty"` @@ -2169,6 +2178,8 @@ type Run struct { OriginalAttemptRunId int64 `json:"original_attempt_run_id,omitempty"` // The parameters used for this run. OverridingParameters *RunParameters `json:"overriding_parameters,omitempty"` + // A token that can be used to list the previous page of sub-resources. + PrevPageToken string `json:"prev_page_token,omitempty"` // The time in milliseconds that the run has spent in the queue. QueueDuration int64 `json:"queue_duration,omitempty"` // The repair history of the run. From cb79825676c6419f42fcc4886fe5b32f5188d14d Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Tue, 6 Aug 2024 18:13:56 +0200 Subject: [PATCH 2/6] Add some tests --- service/jobs/ext_api_test.go | 217 +++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 service/jobs/ext_api_test.go diff --git a/service/jobs/ext_api_test.go b/service/jobs/ext_api_test.go new file mode 100644 index 000000000..166ad34fd --- /dev/null +++ b/service/jobs/ext_api_test.go @@ -0,0 +1,217 @@ +package jobs + +import ( + "context" + "github.com/databricks/databricks-sdk-go/qa" + "testing" + + "github.com/stretchr/testify/assert" +) + +func commonFixtureWithStatusResponse() qa.HTTPFixtures { + return []qa.HTTPFixture{ + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?run_id=514594995218126", + Response: Run{ + Iterations: []RunTask{}, + Tasks: []RunTask{ + { + RunId: 123, + TaskKey: "task1", + }, + { + RunId: 1234, + TaskKey: "task2", + }, + }, + NextPageToken: "", + }, + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?run_id=111222333", + Response: Run{ + Iterations: []RunTask{}, + Tasks: []RunTask{ + { + RunId: 123, + }, + { + RunId: 1234, + }, + }, + JobClusters: []JobCluster{ + { + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", + }, + }, + NextPageToken: "token1", + }, + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?page_token=token1&run_id=111222333", + Response: Run{ + Iterations: []RunTask{}, + Tasks: []RunTask{ + { + RunId: 222, + }, + { + RunId: 333, + }, + }, + JobClusters: []JobCluster{ + { + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", + }, + }, + PrevPageToken: "token1-reverse", + }, + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?run_id=4444", + Response: Run{ + Iterations: []RunTask{ + { + RunId: 123, + }, + { + RunId: 1234, + }, + }, + Tasks: []RunTask{ + { + RunId: 999, + }, + }, + NextPageToken: "token1", + }, + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?page_token=token1&run_id=4444", + Response: Run{ + Iterations: []RunTask{ + { + RunId: 222, + }, + { + RunId: 333, + }, + }, + Tasks: []RunTask{ + { + RunId: 999, + }, + }, + PrevPageToken: "token1-reverse", + }, + }, + } +} + +func TestGetRun(t *testing.T) { + ctx := context.Background() + + t.Run("successful run with no pagination", func(t *testing.T) { + client, server := commonFixtureWithStatusResponse().Client(t) + defer server.Close() + + mockJobsImpl := &jobsImpl{ + client: client, + } + api := &JobsAPI{jobsImpl: *mockJobsImpl} + + request := GetRunRequest{RunId: 514594995218126} + run, err := api.GetRun(ctx, request) + + assert.NoError(t, err) + assert.EqualValues(t, 123, run.Tasks[0].RunId) + assert.EqualValues(t, 1234, run.Tasks[1].RunId) + }) + + t.Run("successful run with two tasks pages", func(t *testing.T) { + client, server := commonFixtureWithStatusResponse().Client(t) + defer server.Close() + + mockJobsImpl := &jobsImpl{ + client: client, + } + api := &JobsAPI{jobsImpl: *mockJobsImpl} + + request := GetRunRequest{RunId: 111222333} + run, err := api.GetRun(ctx, request) + + assert.NoError(t, err) + assert.Equal(t, 4, len(run.Tasks)) + assert.Empty(t, run.Iterations) + assert.Empty(t, run.NextPageToken) + assert.Empty(t, run.PrevPageToken) + expected := []RunTask{ + {RunId: 123, ForceSendFields: []string{"RunId", "TaskKey"}}, + {RunId: 1234, ForceSendFields: []string{"RunId", "TaskKey"}}, + {RunId: 222, ForceSendFields: []string{"RunId", "TaskKey"}}, + {RunId: 333, ForceSendFields: []string{"RunId", "TaskKey"}}, + } + assert.Equal(t, expected, run.Tasks) + }) + + t.Run("successful run with two iterations pages", func(t *testing.T) { + client, server := commonFixtureWithStatusResponse().Client(t) + defer server.Close() + + mockJobsImpl := &jobsImpl{ + client: client, + } + api := &JobsAPI{jobsImpl: *mockJobsImpl} + + request := GetRunRequest{RunId: 4444} + run, err := api.GetRun(ctx, request) + + assert.NoError(t, err) + assert.Equal(t, 4, len(run.Iterations)) + assert.Equal(t, 1, len(run.Tasks)) + assert.Empty(t, run.NextPageToken) + assert.Empty(t, run.PrevPageToken) + expected := []RunTask{ + {RunId: 123, ForceSendFields: []string{"RunId", "TaskKey"}}, + {RunId: 1234, ForceSendFields: []string{"RunId", "TaskKey"}}, + {RunId: 222, ForceSendFields: []string{"RunId", "TaskKey"}}, + {RunId: 333, ForceSendFields: []string{"RunId", "TaskKey"}}, + } + assert.Equal(t, expected, run.Iterations) + assert.EqualValues(t, 999, run.Tasks[0].RunId) + }) + + t.Run("clusters array is not increased when paginated", func(t *testing.T) { + client, server := commonFixtureWithStatusResponse().Client(t) + defer server.Close() + + mockJobsImpl := &jobsImpl{ + client: client, + } + api := &JobsAPI{jobsImpl: *mockJobsImpl} + + request := GetRunRequest{RunId: 111222333} + run, err := api.GetRun(ctx, request) + + assert.NoError(t, err) + assert.Equal(t, 2, len(run.JobClusters)) + assert.Equal(t, "cluster1", run.JobClusters[0].JobClusterKey) + assert.Equal(t, "cluster2", run.JobClusters[1].JobClusterKey) + }) +} From 02dc28d41af65ba99bc727e4bcce619388c3b9e4 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Tue, 13 Aug 2024 10:02:27 +0200 Subject: [PATCH 3/6] Apply fmt --- service/jobs/ext_api_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/jobs/ext_api_test.go b/service/jobs/ext_api_test.go index 166ad34fd..585267048 100644 --- a/service/jobs/ext_api_test.go +++ b/service/jobs/ext_api_test.go @@ -2,9 +2,10 @@ package jobs import ( "context" - "github.com/databricks/databricks-sdk-go/qa" "testing" + "github.com/databricks/databricks-sdk-go/qa" + "github.com/stretchr/testify/assert" ) From c938a2d190f7b5226c169e00a498dffce93e06a3 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Tue, 13 Aug 2024 17:56:20 +0200 Subject: [PATCH 4/6] Add test stubs for 2.1 calls --- service/jobs/ext_api_test.go | 253 +++++++++++++++++++++++------------ 1 file changed, 167 insertions(+), 86 deletions(-) diff --git a/service/jobs/ext_api_test.go b/service/jobs/ext_api_test.go index 585267048..a5cd93584 100644 --- a/service/jobs/ext_api_test.go +++ b/service/jobs/ext_api_test.go @@ -10,119 +10,200 @@ import ( ) func commonFixtureWithStatusResponse() qa.HTTPFixtures { - return []qa.HTTPFixture{ - { - Method: "GET", - ReuseRequest: true, - Resource: "/api/2.2/jobs/runs/get?run_id=514594995218126", - Response: Run{ - Iterations: []RunTask{}, - Tasks: []RunTask{ - { - RunId: 123, - TaskKey: "task1", - }, - { - RunId: 1234, - TaskKey: "task2", + return append( + // API 2.2 stubs + []qa.HTTPFixture{ + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?run_id=514594995218126", + Response: Run{ + Iterations: []RunTask{}, + Tasks: []RunTask{ + { + RunId: 123, + TaskKey: "task1", + }, + { + RunId: 1234, + TaskKey: "task2", + }, }, + NextPageToken: "", }, - NextPageToken: "", }, - }, - { - Method: "GET", - ReuseRequest: true, - Resource: "/api/2.2/jobs/runs/get?run_id=111222333", - Response: Run{ - Iterations: []RunTask{}, - Tasks: []RunTask{ - { - RunId: 123, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?run_id=111222333", + Response: Run{ + Iterations: []RunTask{}, + Tasks: []RunTask{ + { + RunId: 123, + }, + { + RunId: 1234, + }, }, - { - RunId: 1234, + JobClusters: []JobCluster{ + { + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", + }, }, + NextPageToken: "token1", }, - JobClusters: []JobCluster{ - { - JobClusterKey: "cluster1", + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?page_token=token1&run_id=111222333", + Response: Run{ + Iterations: []RunTask{}, + Tasks: []RunTask{ + { + RunId: 222, + }, + { + RunId: 333, + }, }, - { - JobClusterKey: "cluster2", + JobClusters: []JobCluster{ + { + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", + }, }, + PrevPageToken: "token1-reverse", }, - NextPageToken: "token1", }, - }, - { - Method: "GET", - ReuseRequest: true, - Resource: "/api/2.2/jobs/runs/get?page_token=token1&run_id=111222333", - Response: Run{ - Iterations: []RunTask{}, - Tasks: []RunTask{ - { - RunId: 222, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?run_id=4444", + Response: Run{ + Iterations: []RunTask{ + { + RunId: 123, + }, + { + RunId: 1234, + }, }, - { - RunId: 333, + Tasks: []RunTask{ + { + RunId: 999, + }, }, + NextPageToken: "token1", }, - JobClusters: []JobCluster{ - { - JobClusterKey: "cluster1", + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?page_token=token1&run_id=4444", + Response: Run{ + Iterations: []RunTask{ + { + RunId: 222, + }, + { + RunId: 333, + }, }, - { - JobClusterKey: "cluster2", + Tasks: []RunTask{ + { + RunId: 999, + }, }, + PrevPageToken: "token1-reverse", }, - PrevPageToken: "token1-reverse", }, }, - { - Method: "GET", - ReuseRequest: true, - Resource: "/api/2.2/jobs/runs/get?run_id=4444", - Response: Run{ - Iterations: []RunTask{ - { - RunId: 123, - }, - { - RunId: 1234, + //API 2.1 stubs + []qa.HTTPFixture{ + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.1/jobs/runs/get?run_id=514594995218126", + Response: Run{ + Iterations: []RunTask{}, + Tasks: []RunTask{ + { + RunId: 123, + TaskKey: "task1", + }, + { + RunId: 1234, + TaskKey: "task2", + }, }, }, - Tasks: []RunTask{ - { - RunId: 999, - }, - }, - NextPageToken: "token1", }, - }, - { - Method: "GET", - ReuseRequest: true, - Resource: "/api/2.2/jobs/runs/get?page_token=token1&run_id=4444", - Response: Run{ - Iterations: []RunTask{ - { - RunId: 222, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.1/jobs/runs/get?run_id=111222333", + Response: Run{ + Iterations: []RunTask{}, + Tasks: []RunTask{ + { + RunId: 123, + }, + { + RunId: 1234, + }, + { + RunId: 222, + }, + { + RunId: 333, + }, }, - { - RunId: 333, + JobClusters: []JobCluster{ + { + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", + }, }, }, - Tasks: []RunTask{ - { - RunId: 999, + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.1/jobs/runs/get?run_id=4444", + Response: Run{ + Iterations: []RunTask{ + { + RunId: 123, + }, + { + RunId: 1234, + }, + { + RunId: 222, + }, + { + RunId: 333, + }, + }, + Tasks: []RunTask{ + { + RunId: 999, + }, }, }, - PrevPageToken: "token1-reverse", }, - }, - } + }..., + ) + } func TestGetRun(t *testing.T) { From 486741304af517168ffb2c45b97d3b4978a663c0 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Fri, 16 Aug 2024 13:35:56 +0200 Subject: [PATCH 5/6] Add comments, remove unused line, reformat tests --- service/jobs/ext_api.go | 12 +- service/jobs/ext_api_test.go | 269 ++++++++++++++++++++++------------- 2 files changed, 182 insertions(+), 99 deletions(-) diff --git a/service/jobs/ext_api.go b/service/jobs/ext_api.go index 5c64d813d..6887b8b40 100644 --- a/service/jobs/ext_api.go +++ b/service/jobs/ext_api.go @@ -2,16 +2,21 @@ package jobs import "context" +// GetRun retrieves a run based on the provided request. +// It handles pagination if the run contains multiple iterations or tasks. func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, error) { run, err := a.jobsImpl.GetRun(ctx, request) if err != nil { return nil, err } + // When querying a Job run, a page token is returned when there are more than 100 tasks. No iterations are defined for a Job run. Therefore, the next page in the response only includes the next page of tasks. + // When querying a ForEach task run, a page token is returned when there are more than 100 iterations. Only a single task is returned, corresponding to the ForEach task itself. Therefore, the client only reads the iterations from the next page and not the tasks. isPaginatingIterations := run.Iterations != nil && len(run.Iterations) > 0 - for len(run.NextPageToken) != 0 { - request.PageToken = run.NextPageToken + pageToken := run.NextPageToken + for pageToken != "" { + request.PageToken = pageToken nextRun, err := a.jobsImpl.GetRun(ctx, request) if err != nil { return nil, err @@ -22,9 +27,10 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro } else { run.Tasks = append(run.Tasks, nextRun.Tasks...) } - run.NextPageToken = nextRun.NextPageToken + pageToken = nextRun.NextPageToken } + run.NextPageToken = "" run.PrevPageToken = "" return run, nil } diff --git a/service/jobs/ext_api_test.go b/service/jobs/ext_api_test.go index a5cd93584..f370b0673 100644 --- a/service/jobs/ext_api_test.go +++ b/service/jobs/ext_api_test.go @@ -9,14 +9,33 @@ import ( "github.com/stretchr/testify/assert" ) -func commonFixtureWithStatusResponse() qa.HTTPFixtures { - return append( - // API 2.2 stubs - []qa.HTTPFixture{ +func TestGetRun(t *testing.T) { + ctx := context.Background() + + t.Run("run with no pagination", func(t *testing.T) { + var requestMocks qa.HTTPFixtures = []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.2/jobs/runs/get?run_id=514594995218126", + Response: Run{ + Iterations: []RunTask{}, + Tasks: []RunTask{ + { + RunId: 123, + TaskKey: "task1", + }, + { + RunId: 1234, + TaskKey: "task2", + }, + }, + NextPageToken: "", + }, + }, { Method: "GET", ReuseRequest: true, - Resource: "/api/2.2/jobs/runs/get?run_id=514594995218126", + Resource: "/api/2.1/jobs/runs/get?run_id=514594995218126", Response: Run{ Iterations: []RunTask{}, Tasks: []RunTask{ @@ -29,9 +48,29 @@ func commonFixtureWithStatusResponse() qa.HTTPFixtures { TaskKey: "task2", }, }, - NextPageToken: "", }, }, + } + client, server := requestMocks.Client(t) + defer server.Close() + + mockJobsImpl := &jobsImpl{ + client: client, + } + api := &JobsAPI{jobsImpl: *mockJobsImpl} + + request := GetRunRequest{RunId: 514594995218126} + run, err := api.GetRun(ctx, request) + + assert.NoError(t, err) + assert.Equal(t, 2, len(run.Tasks)) + assert.Empty(t, run.Iterations) + assert.EqualValues(t, 123, run.Tasks[0].RunId) + assert.EqualValues(t, 1234, run.Tasks[1].RunId) + }) + + t.Run("run with two tasks pages", func(t *testing.T) { + var requestMocks qa.HTTPFixtures = []qa.HTTPFixture{ { Method: "GET", ReuseRequest: true, @@ -85,64 +124,109 @@ func commonFixtureWithStatusResponse() qa.HTTPFixtures { { Method: "GET", ReuseRequest: true, - Resource: "/api/2.2/jobs/runs/get?run_id=4444", + Resource: "/api/2.1/jobs/runs/get?run_id=111222333", Response: Run{ - Iterations: []RunTask{ + Iterations: []RunTask{}, + Tasks: []RunTask{ { RunId: 123, }, { RunId: 1234, }, + { + RunId: 222, + }, + { + RunId: 333, + }, }, - Tasks: []RunTask{ + JobClusters: []JobCluster{ { - RunId: 999, + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", }, }, - NextPageToken: "token1", }, }, + } + client, server := requestMocks.Client(t) + defer server.Close() + + mockJobsImpl := &jobsImpl{ + client: client, + } + api := &JobsAPI{jobsImpl: *mockJobsImpl} + + request := GetRunRequest{RunId: 111222333} + run, err := api.GetRun(ctx, request) + + assert.NoError(t, err) + assert.Equal(t, 4, len(run.Tasks)) + assert.Empty(t, run.Iterations) + assert.Empty(t, run.NextPageToken) + assert.Empty(t, run.PrevPageToken) + expected := []RunTask{ + {RunId: 123, ForceSendFields: []string{"RunId", "TaskKey"}}, + {RunId: 1234, ForceSendFields: []string{"RunId", "TaskKey"}}, + {RunId: 222, ForceSendFields: []string{"RunId", "TaskKey"}}, + {RunId: 333, ForceSendFields: []string{"RunId", "TaskKey"}}, + } + assert.Equal(t, expected, run.Tasks) + }) + + t.Run("clusters array is not increased when paginated", func(t *testing.T) { + var requestMocks qa.HTTPFixtures = []qa.HTTPFixture{ { Method: "GET", ReuseRequest: true, - Resource: "/api/2.2/jobs/runs/get?page_token=token1&run_id=4444", + Resource: "/api/2.2/jobs/runs/get?run_id=111222333", Response: Run{ - Iterations: []RunTask{ + Iterations: []RunTask{}, + Tasks: []RunTask{ { - RunId: 222, + RunId: 123, }, { - RunId: 333, + RunId: 1234, }, }, - Tasks: []RunTask{ + JobClusters: []JobCluster{ { - RunId: 999, + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", }, }, - PrevPageToken: "token1-reverse", + NextPageToken: "token1", }, }, - }, - //API 2.1 stubs - []qa.HTTPFixture{ { Method: "GET", ReuseRequest: true, - Resource: "/api/2.1/jobs/runs/get?run_id=514594995218126", + Resource: "/api/2.2/jobs/runs/get?page_token=token1&run_id=111222333", Response: Run{ Iterations: []RunTask{}, Tasks: []RunTask{ { - RunId: 123, - TaskKey: "task1", + RunId: 222, }, { - RunId: 1234, - TaskKey: "task2", + RunId: 333, + }, + }, + JobClusters: []JobCluster{ + { + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", }, }, + PrevPageToken: "token1-reverse", }, }, { @@ -175,6 +259,68 @@ func commonFixtureWithStatusResponse() qa.HTTPFixtures { }, }, }, + } + client, server := requestMocks.Client(t) + defer server.Close() + + mockJobsImpl := &jobsImpl{ + client: client, + } + api := &JobsAPI{jobsImpl: *mockJobsImpl} + + request := GetRunRequest{RunId: 111222333} + run, err := api.GetRun(ctx, request) + + assert.NoError(t, err) + assert.Equal(t, 2, len(run.JobClusters)) + assert.Equal(t, "cluster1", run.JobClusters[0].JobClusterKey) + assert.Equal(t, "cluster2", run.JobClusters[1].JobClusterKey) + }) + + t.Run("run with two iterations pages", func(t *testing.T) { + var requestMocks qa.HTTPFixtures = []qa.HTTPFixture{ + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?run_id=4444", + Response: Run{ + Iterations: []RunTask{ + { + RunId: 123, + }, + { + RunId: 1234, + }, + }, + Tasks: []RunTask{ + { + RunId: 999, + }, + }, + NextPageToken: "token1", + }, + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.2/jobs/runs/get?page_token=token1&run_id=4444", + Response: Run{ + Iterations: []RunTask{ + { + RunId: 222, + }, + { + RunId: 333, + }, + }, + Tasks: []RunTask{ + { + RunId: 999, + }, + }, + PrevPageToken: "token1-reverse", + }, + }, { Method: "GET", ReuseRequest: true, @@ -201,59 +347,8 @@ func commonFixtureWithStatusResponse() qa.HTTPFixtures { }, }, }, - }..., - ) - -} - -func TestGetRun(t *testing.T) { - ctx := context.Background() - - t.Run("successful run with no pagination", func(t *testing.T) { - client, server := commonFixtureWithStatusResponse().Client(t) - defer server.Close() - - mockJobsImpl := &jobsImpl{ - client: client, } - api := &JobsAPI{jobsImpl: *mockJobsImpl} - - request := GetRunRequest{RunId: 514594995218126} - run, err := api.GetRun(ctx, request) - - assert.NoError(t, err) - assert.EqualValues(t, 123, run.Tasks[0].RunId) - assert.EqualValues(t, 1234, run.Tasks[1].RunId) - }) - - t.Run("successful run with two tasks pages", func(t *testing.T) { - client, server := commonFixtureWithStatusResponse().Client(t) - defer server.Close() - - mockJobsImpl := &jobsImpl{ - client: client, - } - api := &JobsAPI{jobsImpl: *mockJobsImpl} - - request := GetRunRequest{RunId: 111222333} - run, err := api.GetRun(ctx, request) - - assert.NoError(t, err) - assert.Equal(t, 4, len(run.Tasks)) - assert.Empty(t, run.Iterations) - assert.Empty(t, run.NextPageToken) - assert.Empty(t, run.PrevPageToken) - expected := []RunTask{ - {RunId: 123, ForceSendFields: []string{"RunId", "TaskKey"}}, - {RunId: 1234, ForceSendFields: []string{"RunId", "TaskKey"}}, - {RunId: 222, ForceSendFields: []string{"RunId", "TaskKey"}}, - {RunId: 333, ForceSendFields: []string{"RunId", "TaskKey"}}, - } - assert.Equal(t, expected, run.Tasks) - }) - - t.Run("successful run with two iterations pages", func(t *testing.T) { - client, server := commonFixtureWithStatusResponse().Client(t) + client, server := requestMocks.Client(t) defer server.Close() mockJobsImpl := &jobsImpl{ @@ -278,22 +373,4 @@ func TestGetRun(t *testing.T) { assert.Equal(t, expected, run.Iterations) assert.EqualValues(t, 999, run.Tasks[0].RunId) }) - - t.Run("clusters array is not increased when paginated", func(t *testing.T) { - client, server := commonFixtureWithStatusResponse().Client(t) - defer server.Close() - - mockJobsImpl := &jobsImpl{ - client: client, - } - api := &JobsAPI{jobsImpl: *mockJobsImpl} - - request := GetRunRequest{RunId: 111222333} - run, err := api.GetRun(ctx, request) - - assert.NoError(t, err) - assert.Equal(t, 2, len(run.JobClusters)) - assert.Equal(t, "cluster1", run.JobClusters[0].JobClusterKey) - assert.Equal(t, "cluster2", run.JobClusters[1].JobClusterKey) - }) } From 9f8482d25d01f8c4c71d1a1faf8e0a5ea67bfb1d Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Mon, 4 Nov 2024 17:17:36 +0100 Subject: [PATCH 6/6] Remove prePageToken that does not exist in proto anymore --- service/jobs/ext_api.go | 2 -- service/jobs/ext_api_test.go | 5 ----- 2 files changed, 7 deletions(-) diff --git a/service/jobs/ext_api.go b/service/jobs/ext_api.go index 6887b8b40..42417150e 100644 --- a/service/jobs/ext_api.go +++ b/service/jobs/ext_api.go @@ -30,7 +30,5 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro pageToken = nextRun.NextPageToken } - run.NextPageToken = "" - run.PrevPageToken = "" return run, nil } diff --git a/service/jobs/ext_api_test.go b/service/jobs/ext_api_test.go index f370b0673..e13a437f8 100644 --- a/service/jobs/ext_api_test.go +++ b/service/jobs/ext_api_test.go @@ -118,7 +118,6 @@ func TestGetRun(t *testing.T) { JobClusterKey: "cluster2", }, }, - PrevPageToken: "token1-reverse", }, }, { @@ -167,7 +166,6 @@ func TestGetRun(t *testing.T) { assert.Equal(t, 4, len(run.Tasks)) assert.Empty(t, run.Iterations) assert.Empty(t, run.NextPageToken) - assert.Empty(t, run.PrevPageToken) expected := []RunTask{ {RunId: 123, ForceSendFields: []string{"RunId", "TaskKey"}}, {RunId: 1234, ForceSendFields: []string{"RunId", "TaskKey"}}, @@ -226,7 +224,6 @@ func TestGetRun(t *testing.T) { JobClusterKey: "cluster2", }, }, - PrevPageToken: "token1-reverse", }, }, { @@ -318,7 +315,6 @@ func TestGetRun(t *testing.T) { RunId: 999, }, }, - PrevPageToken: "token1-reverse", }, }, { @@ -363,7 +359,6 @@ func TestGetRun(t *testing.T) { assert.Equal(t, 4, len(run.Iterations)) assert.Equal(t, 1, len(run.Tasks)) assert.Empty(t, run.NextPageToken) - assert.Empty(t, run.PrevPageToken) expected := []RunTask{ {RunId: 123, ForceSendFields: []string{"RunId", "TaskKey"}}, {RunId: 1234, ForceSendFields: []string{"RunId", "TaskKey"}},