diff --git a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/avro/QueryEvents.avsc b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/avro/QueryEvents.avsc index 49a3614..3e727d8 100644 --- a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/avro/QueryEvents.avsc +++ b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/avro/QueryEvents.avsc @@ -248,6 +248,14 @@ ], "default": null }, + { + "name": "YarnApplicationId", + "type": [ + "null", + "string" + ], + "default": null + }, { "name": "DatabasesRead", "type": [ @@ -275,14 +283,6 @@ "string" ], "default": null - }, - { - "name": "ApplicationData", - "type": [ - "null", - "string" - ], - "default": null } ] } \ No newline at end of file diff --git a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/ApplicationIdRetriever.java b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/ApplicationIdRetriever.java index a653f40..00d3d33 100644 --- a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/ApplicationIdRetriever.java +++ b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/ApplicationIdRetriever.java @@ -17,10 +17,7 @@ package com.google.cloud.bigquery.dwhassessment.hooks.logger; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; +import java.util.Optional; import java.util.stream.Stream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; @@ -39,17 +36,17 @@ public class ApplicationIdRetriever { private static final Logger LOG = LoggerFactory.getLogger(ApplicationIdRetriever.class); - public static List determineApplicationIds( + public static Optional determineApplicationId( HiveConf conf, ExecutionMode executionMode) { switch (executionMode) { case MR: - return determineMapReduceApplicationIds(); + return determineMapReduceApplicationId(); case TEZ: - return determineTezApplicationIds(); + return determineTezApplicationId(); case LLAP: - return determineLlapApplicationIds(conf, executionMode); + return determineLlapApplicationId(conf, executionMode); default: - return new ArrayList<>(); + return Optional.empty(); } } @@ -58,28 +55,27 @@ public static List determineApplicationIds( * application always have only one queue – if queue changes in the session, new application is * created. */ - private static List determineTezApplicationIds() { + private static Optional determineTezApplicationId() { SessionState sessionState = SessionState.get(); if (sessionState != null) { TezSessionState tezSessionState = sessionState.getTezSession(); if (tezSessionState != null) { TezClient tezClient = tezSessionState.getSession(); if (tezClient != null) { - ApplicationId applicationId = tezClient.getAppMasterApplicationId(); - return applicationId != null ? Collections.singletonList(tezClient.getAppMasterApplicationId()) : new ArrayList<>(); + return Optional.ofNullable(tezClient.getAppMasterApplicationId()); } } } LOG.info("Failed to retrieve Application ID from Tez session"); - return new ArrayList<>(); + return Optional.empty(); } /** * Retrieves Application ID from the first MapReduce job as multiple MapReduce jobs created for a * single query are submitted to the same queue. */ - private static List determineMapReduceApplicationIds() { + private static Optional determineMapReduceApplicationId() { return SessionState.get().getMapRedStats().values().stream() .map(MapRedStats::getJobId) .flatMap( @@ -93,14 +89,15 @@ private static List determineMapReduceApplicationIds() { jobId); return Stream.empty(); } - }).collect(Collectors.toList()); + }) + .findFirst(); } /** * Retrieve Application ID for Llap daemon. They are long-living YARN applications, using the same * queue, so it should be relatively static. */ - public static List determineLlapApplicationIds( + public static Optional determineLlapApplicationId( HiveConf conf, ExecutionMode mode) { // Note: for now, LLAP is only supported in Tez tasks. Will never come to MR; others may // be added here, although this is only necessary to have extra debug information. @@ -110,7 +107,7 @@ public static List determineLlapApplicationIds( String hosts = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS); if (hosts != null && !hosts.isEmpty()) { try { - return Collections.singletonList(LlapRegistryService.getClient(conf).getApplicationId()); + return Optional.of(LlapRegistryService.getClient(conf).getApplicationId()); } catch (IOException e) { LOG.error("Error trying to get llap instance. Hosts: {}", hosts, e); } @@ -119,6 +116,6 @@ public static List determineLlapApplicationIds( } } - return new ArrayList<>(); + return Optional.empty(); } } diff --git a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventRecordConstructor.java b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventRecordConstructor.java index 35af781..0d3ec08 100644 --- a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventRecordConstructor.java +++ b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventRecordConstructor.java @@ -25,13 +25,9 @@ import static org.apache.hadoop.hive.ql.hooks.Entity.Type.TABLE; import com.google.cloud.bigquery.dwhassessment.hooks.logger.utils.TasksRetriever; -import com.google.common.collect.ImmutableMap; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.net.InetAddress; import java.net.UnknownHostException; import java.time.Clock; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -53,7 +49,6 @@ import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.Counters.Group; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.tez.common.counters.CounterGroup; import org.apache.tez.common.counters.TezCounter; import org.apache.tez.common.counters.TezCounters; @@ -147,38 +142,19 @@ private GenericRecord getPostHookEvent(HookContext hookContext, EventStatus stat .set("PerfObject", dumpPerfData(hookContext.getPerfLogger())) .set("OperationId", hookContext.getOperationId()); - List > listOfApplicationData = new ArrayList<>(); - ApplicationIdRetriever.determineApplicationIds(conf, getExecutionMode(plan)) - .forEach(applicationId -> { - ImmutableMap.Builder applicationDataMap = ImmutableMap.builder(); - - applicationDataMap.put("YarnApplicationId", applicationId.toString()); - yarnApplicationRetriever.retrieve(conf, applicationId) + ApplicationIdRetriever.determineApplicationId(conf, getExecutionMode(plan)) + .ifPresent( + applicationId -> { + recordBuilder.set("YarnApplicationId", applicationId.toString()); + yarnApplicationRetriever + .retrieve(conf, applicationId) .ifPresent( applicationReport -> { - recordBuilder - .set("HiveHostName", applicationReport.getHost()) - .set("Queue", applicationReport.getQueue()); - applicationDataMap.put("HiveHostName", applicationReport.getHost()) - .put("Queue", applicationReport.getQueue()) - .put("YarnProcess", String.valueOf(applicationReport.getProgress())) - .put("YarnApplicationType", applicationReport.getApplicationType()) - .put("YarnApplicationState", applicationReport.getYarnApplicationState().toString()) - .put("YarnDiagnostics", applicationReport.getDiagnostics()) - .put("YarnCurrentApplicationAttemptId", applicationReport.getCurrentApplicationAttemptId().toString()) - .put("YarnUser", applicationReport.getUser()) - .put("YarnStartTime", String.valueOf(applicationReport.getStartTime())) - .put("YarnFinishTime", String.valueOf(applicationReport.getFinishTime())) - .put("YarnFinalApplicationStatus", applicationReport.getFinalApplicationStatus().toString()); - retrieveApplicationResourceUsageReport(applicationReport.getApplicationResourceUsageReport(), applicationDataMap); - } - ); - listOfApplicationData.add(applicationDataMap.build()); + recordBuilder.set("HiveHostName", applicationReport.getHost()); + recordBuilder.set("Queue", applicationReport.getQueue()); + }); }); - generateYarnApplicationJson(listOfApplicationData) - .ifPresent(applications -> recordBuilder.set("ApplicationData", applications)); - dumpTezCounters(plan) .map(Optional::of) .orElseGet(EventRecordConstructor::dumpMapReduceCounters) @@ -187,90 +163,6 @@ private GenericRecord getPostHookEvent(HookContext hookContext, EventStatus stat return recordBuilder.build(); } - private void retrieveApplicationResourceUsageReport( - ApplicationResourceUsageReport applicationResourceUsageReport, - ImmutableMap.Builder applicationDataMap - ) { - ImmutableMap reportMethods = - ImmutableMap.builder() - .put("getNumUsedContainers", "YarnReportNumUsedContainers") - .put("getNumReservedContainers", "YarnReportNumReservedContainers") - .put("getMemorySeconds", "YarnReportMemorySeconds") - .put("getVcoreSeconds", "YarnReportVcoreSeconds") - .put("getQueueUsagePercentage", "YarnReportQueueUsagePercentage") - .put("getClusterUsagePercentage", "YarnReportClusterUsagePercentage") - .put("getPreemptedMemorySeconds", "YarnReportPreemptedMemorySeconds") - .put("getPreemptedVcoreSeconds", "YarnReportPreemptedVcoreSeconds") - .build(); - - reportMethods.forEach( - (methodName, fieldName) -> - callMethodWithReflection( - methodName, fieldName, applicationDataMap, applicationResourceUsageReport)); - - applicationDataMap.put( - "YarnReportUsedResources", - String.valueOf(applicationResourceUsageReport.getUsedResources().toString())); - callMethodWithReflection( - "getMemorySize", - "YarnReportUsedResourcesMemory", - applicationDataMap, - applicationResourceUsageReport.getUsedResources()); - callMethodWithReflection( - "getVirtualCores", - "YarnReportUsedResourcesVcore", - applicationDataMap, - applicationResourceUsageReport.getUsedResources()); - - applicationDataMap.put( - "YarnReportReservedResources", - String.valueOf(applicationResourceUsageReport.getReservedResources().toString())); - callMethodWithReflection( - "getMemorySize", - "YarnReportReservedResourcesMemory", - applicationDataMap, - applicationResourceUsageReport.getReservedResources()); - callMethodWithReflection( - "getVirtualCores", - "YarnReportReservedResourcesVcore", - applicationDataMap, - applicationResourceUsageReport.getReservedResources()); - - applicationDataMap.put( - "YarnReportNeededResources", - String.valueOf(applicationResourceUsageReport.getNeededResources().toString())); - callMethodWithReflection( - "getMemorySize", - "YarnReportNeededResourcesMemory", - applicationDataMap, - applicationResourceUsageReport.getNeededResources()); - callMethodWithReflection( - "getVirtualCores", - "YarnReportNeededResourcesVcore", - applicationDataMap, - applicationResourceUsageReport.getNeededResources()); - - } - - private void callMethodWithReflection( - String methodName, String fieldNameInAvro, ImmutableMap.Builder applicationDataMap, T object) { - try { - Method method = Class.forName(object.getClass().getName()).getMethod(methodName); - applicationDataMap.put(fieldNameInAvro, String.valueOf(method.invoke(object))); - } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { - LOG.warn("Failed to invoke method '{}'", methodName); - } catch (ClassNotFoundException e) { - LOG.warn( - "Failed to find class {} to invoke method '{}'", object.getClass().getName(), methodName); - } - } - - private static Optional generateYarnApplicationJson(List > listOfApplicationData) { - JSONArray jsonArray = new JSONArray(); - listOfApplicationData.stream().map(JSONObject::new).forEach(jsonArray::put); - return jsonArray.length() > 0 ? Optional.of(jsonArray.toString()) : Optional.empty(); - } - /** * Retrieves YARN queue name where query is supposed to land. It might be different in reality, * depending on YARN configuration. diff --git a/src/javatests/com/google/cloud/bigquery/dwhassessment/hooks/logger/ApplicationIdRetrieverTest.java b/src/javatests/com/google/cloud/bigquery/dwhassessment/hooks/logger/ApplicationIdRetrieverTest.java index 6b59d99..6bcdeb3 100644 --- a/src/javatests/com/google/cloud/bigquery/dwhassessment/hooks/logger/ApplicationIdRetrieverTest.java +++ b/src/javatests/com/google/cloud/bigquery/dwhassessment/hooks/logger/ApplicationIdRetrieverTest.java @@ -19,11 +19,11 @@ import static com.google.cloud.bigquery.dwhassessment.hooks.testing.TestUtils.createDefaultSessionState; import static com.google.cloud.bigquery.dwhassessment.hooks.testing.TestUtils.createMapRedStats; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.ArrayList; -import java.util.List; +import java.util.Optional; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; import org.apache.hadoop.hive.ql.session.SessionState; @@ -45,69 +45,66 @@ public void setup() { } @Test - public void determineLlapApplicationIds_unsetHostsConfigValue() { + public void determineLlapApplicationId_unsetHostsConfigValue() { // Act - List applicationId = - ApplicationIdRetriever.determineLlapApplicationIds(conf, ExecutionMode.LLAP); + Optional applicationId = + ApplicationIdRetriever.determineLlapApplicationId(conf, ExecutionMode.LLAP); // Assert assertThat(applicationId).isEmpty(); } @Test - public void determineApplicationIds_unsupportedExecutionMode() { + public void determineApplicationId_unsupportedExecutionMode() { // Act - List applicationId = - ApplicationIdRetriever.determineApplicationIds(conf, ExecutionMode.DDL); + Optional applicationId = + ApplicationIdRetriever.determineApplicationId(conf, ExecutionMode.DDL); // Assert assertThat(applicationId).isEmpty(); } @Test - public void determineApplicationIds_MapReduce_success() { + public void determineApplicationId_MapReduce_success() { SessionState sessionState = createDefaultSessionState(conf); sessionState.getMapRedStats().put("Stage-1", createMapRedStats("job_1685098059769_1951")); sessionState.getMapRedStats().put("Stage-2", createMapRedStats("job_1685098059769_1949")); - List expectedList = new ArrayList<>(); - expectedList.add(ApplicationId.newInstance(/* clusterTimestamp= */ 1685098059769L, /* id= */ 1951)); - expectedList.add(ApplicationId.newInstance(/* clusterTimestamp= */ 1685098059769L, /* id= */ 1949)); // Act - List applicationIds = - ApplicationIdRetriever.determineApplicationIds(conf, ExecutionMode.MR); + ApplicationId applicationId = + ApplicationIdRetriever.determineApplicationId(conf, ExecutionMode.MR).get(); // Assert - assertThat(applicationIds).isEqualTo(expectedList); + assertThat(applicationId.toString()).isEqualTo("application_1685098059769_1951"); } @Test - public void determineApplicationIds_MapReduce_malformedJobId() { + public void determineApplicationId_MapReduce_malformedJobId() { SessionState sessionState = createDefaultSessionState(conf); sessionState.getMapRedStats().put("Stage-1", createMapRedStats("malformed_job_id")); // Act - List applicationId = - ApplicationIdRetriever.determineApplicationIds(conf, ExecutionMode.MR); + Optional applicationId = + ApplicationIdRetriever.determineApplicationId(conf, ExecutionMode.MR); // Assert assertThat(applicationId).isEmpty(); } @Test - public void determineApplicationIds_TezWithEmptyState() { + public void determineApplicationId_TezWithEmptyState() { // Act - List applicationId = - ApplicationIdRetriever.determineApplicationIds(conf, ExecutionMode.TEZ); + Optional applicationId = + ApplicationIdRetriever.determineApplicationId(conf, ExecutionMode.TEZ); // Assert assertThat(applicationId).isEmpty(); } @Test - public void determineApplicationIds_TezWithAppIdInSessionState() { + public void determineApplicationId_TezWithAppIdInSessionState() { ApplicationId expectedApplicationId = ApplicationId.newInstance(/* clusterTimestamp= */ 123, /* id= */ 456); SessionState sessionState = createDefaultSessionState(conf); @@ -118,10 +115,10 @@ public void determineApplicationIds_TezWithAppIdInSessionState() { sessionState.setTezSession(tezSessionState); // Act - List applicationId = - ApplicationIdRetriever.determineApplicationIds(conf, ExecutionMode.TEZ); + Optional applicationId = + ApplicationIdRetriever.determineApplicationId(conf, ExecutionMode.TEZ); // Assert - assertThat(applicationId).containsExactly(expectedApplicationId); + assertThat(applicationId).hasValue(expectedApplicationId); } } diff --git a/src/javatests/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventRecordConstructorTest.java b/src/javatests/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventRecordConstructorTest.java index 293719d..9e4c71b 100644 --- a/src/javatests/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventRecordConstructorTest.java +++ b/src/javatests/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventRecordConstructorTest.java @@ -20,7 +20,6 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth8.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -31,8 +30,6 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import java.util.Optional; import java.util.Set; import org.apache.avro.generic.GenericRecord; @@ -56,14 +53,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.Counters.Group; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.Token; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.Records; import org.apache.tez.common.counters.CounterGroup; import org.apache.tez.common.counters.TezCounters; @@ -207,49 +197,21 @@ public void postExecHook_success() { assertThat(record).hasValue(TestUtils.createPostExecRecord(EventStatus.SUCCESS)); } - @Test - public void postExecHook_withNoYarnApplicationData() { - hookContext.setHookType(HookType.POST_EXEC_HOOK); - queryPlan.setRootTasks(new ArrayList<>(ImmutableList.of())); - - // Act - GenericRecord record = eventRecordConstructor.constructEvent(hookContext).get(); - - // Assert - assertThat(record.get("ApplicationData")).isNull(); - assertThat(record.get("HiveHostName")).isNull(); - assertThat(record.get("Queue")).isNull(); - } @Test public void postExecHook_recordsYarnApplicationDataWhenPossible() { hookContext.setHookType(HookType.POST_EXEC_HOOK); - queryPlan.setRootTasks(new ArrayList<>(ImmutableList.of(new ExecDriver(), new ExecDriver()))); - Map a = ImmutableMap.builder() - .put("Stage-1", createMapRedStats("job_1685098059769_1000")) - .put("Stage-2", createMapRedStats("job_1685098059769_2000")).build(); - state.getMapRedStats().putAll(a); - ApplicationId applicationId1 = ApplicationId.newInstance(1685098059769L, 1000); - ApplicationId applicationId2 = ApplicationId.newInstance(1685098059769L, 2000); - ApplicationReport report1 = constructApplicationReport(applicationId1); - ApplicationReport report2 = constructApplicationReport(applicationId2); - String expectedJson = - "[{\"YarnApplicationId\":\"application_1685098059769_1000\",\"HiveHostName\":\"test_host\",\"Queue\":\"test_queue\",\"YarnProcess\":\"10000.6\",\"YarnApplicationType\":\"MR\",\"YarnApplicationState\":\"RUNNING\",\"YarnDiagnostics\":\"test_diagnostics\",\"YarnCurrentApplicationAttemptId\":\"appattempt_1685098059769_1000_000001\",\"YarnUser\":\"test_user\",\"YarnStartTime\":\"100000\",\"YarnFinishTime\":\"200000\",\"YarnFinalApplicationStatus\":\"UNDEFINED\",\"YarnReportNumUsedContainers\":\"3\",\"YarnReportNumReservedContainers\":\"4\",\"YarnReportMemorySeconds\":\"100\",\"YarnReportVcoreSeconds\":\"400\",\"YarnReportQueueUsagePercentage\":\"90.5\",\"YarnReportClusterUsagePercentage\":\"64.5\",\"YarnReportPreemptedMemorySeconds\":\"200\",\"YarnReportPreemptedVcoreSeconds\":\"300\",\"YarnReportUsedResources\":\"\",\"YarnReportUsedResourcesMemory\":\"800\",\"YarnReportUsedResourcesVcore\":\"5\",\"YarnReportReservedResources\":\"\",\"YarnReportReservedResourcesMemory\":\"600\",\"YarnReportReservedResourcesVcore\":\"8\",\"YarnReportNeededResources\":\"\",\"YarnReportNeededResourcesMemory\":\"700\",\"YarnReportNeededResourcesVcore\":\"4\"}," - + "{\"YarnApplicationId\":\"application_1685098059769_2000\",\"HiveHostName\":\"test_host\",\"Queue\":\"test_queue\",\"YarnProcess\":\"10000.6\",\"YarnApplicationType\":\"MR\",\"YarnApplicationState\":\"RUNNING\",\"YarnDiagnostics\":\"test_diagnostics\",\"YarnCurrentApplicationAttemptId\":\"appattempt_1685098059769_2000_000001\",\"YarnUser\":\"test_user\",\"YarnStartTime\":\"100000\",\"YarnFinishTime\":\"200000\",\"YarnFinalApplicationStatus\":\"UNDEFINED\",\"YarnReportNumUsedContainers\":\"3\",\"YarnReportNumReservedContainers\":\"4\",\"YarnReportMemorySeconds\":\"100\",\"YarnReportVcoreSeconds\":\"400\",\"YarnReportQueueUsagePercentage\":\"90.5\",\"YarnReportClusterUsagePercentage\":\"64.5\",\"YarnReportPreemptedMemorySeconds\":\"200\",\"YarnReportPreemptedVcoreSeconds\":\"300\",\"YarnReportUsedResources\":\"\",\"YarnReportUsedResourcesMemory\":\"800\",\"YarnReportUsedResourcesVcore\":\"5\",\"YarnReportReservedResources\":\"\",\"YarnReportReservedResourcesMemory\":\"600\",\"YarnReportReservedResourcesVcore\":\"8\",\"YarnReportNeededResources\":\"\",\"YarnReportNeededResourcesMemory\":\"700\",\"YarnReportNeededResourcesVcore\":\"4\"}]"; - - when(yarnApplicationRetrieverMock.retrieve(any(), eq(applicationId1))).thenReturn(Optional.of(report1)); - when(yarnApplicationRetrieverMock.retrieve(any(), eq(applicationId2))).thenReturn(Optional.of(report2)); + queryPlan.setRootTasks(new ArrayList<>(ImmutableList.of(new ExecDriver()))); + state.getMapRedStats().put("Stage-1", createMapRedStats("job_1685098059769_1951")); + ApplicationReport report = Records.newRecord(ApplicationReport.class); + report.setQueue("test_queue"); + report.setHost("test_host"); + when(yarnApplicationRetrieverMock.retrieve(any(), any())).thenReturn(Optional.of(report)); // Act GenericRecord record = eventRecordConstructor.constructEvent(hookContext).get(); // Assert - assertThat(record.get("ApplicationData")).isEqualTo(expectedJson); + assertThat(record.get("YarnApplicationId")).isEqualTo("application_1685098059769_1951"); assertThat(record.get("HiveHostName")).isEqualTo("test_host"); assertThat(record.get("Queue")).isEqualTo("test_queue"); } @@ -265,39 +227,6 @@ public void onFailureHook_success() { assertThat(record).hasValue(TestUtils.createPostExecRecord(EventStatus.FAIL)); } - private ApplicationReport constructApplicationReport(ApplicationId applicationId) { - ApplicationReport report = Records.newRecord(ApplicationReport.class); - ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1); - report.setQueue("test_queue"); - report.setHost("test_host"); - report.setProgress(10000.6f); - report.setApplicationType("MR"); - report.setYarnApplicationState(YarnApplicationState.RUNNING); - report.setDiagnostics("test_diagnostics"); - report.setCurrentApplicationAttemptId(applicationAttemptId); - report.setUser("test_user"); - report.setStartTime(100000L); - report.setFinishTime(200000L); - report.setFinalApplicationStatus(FinalApplicationStatus.UNDEFINED); - ApplicationResourceUsageReport applicationResourceUsageReport = Records.newRecord(ApplicationResourceUsageReport.class); - applicationResourceUsageReport.setMemorySeconds(100L); - applicationResourceUsageReport.setVcoreSeconds(400L); - applicationResourceUsageReport.setClusterUsagePercentage(64.5f); - applicationResourceUsageReport.setQueueUsagePercentage(90.5f); - applicationResourceUsageReport.setPreemptedMemorySeconds(200L); - applicationResourceUsageReport.setPreemptedVcoreSeconds(300L); - applicationResourceUsageReport.setNumUsedContainers(3); - applicationResourceUsageReport.setNumReservedContainers(4); - Resource reservedResource = Resource.newInstance(600, 8); - Resource neededResources = Resource.newInstance(700, 4); - Resource usedResources = Resource.newInstance(800, 5); - applicationResourceUsageReport.setReservedResources(reservedResource); - applicationResourceUsageReport.setNeededResources(neededResources); - applicationResourceUsageReport.setUsedResources(usedResources); - report.setApplicationResourceUsageReport(applicationResourceUsageReport); - return report; - } - @DataPoints("PostHookTypes") public static final ImmutableList POST_HOOK_TYPES = ImmutableList.of(HookType.ON_FAILURE_HOOK, HookType.POST_EXEC_HOOK);