From dd29ddcf4ac7363a2a6af0008b30b44ea4976c29 Mon Sep 17 00:00:00 2001 From: Roman Fursov Date: Thu, 22 Jun 2023 12:32:36 +0200 Subject: [PATCH] Increase default rollover period (#41) --- .../hooks/logger/DatePartitionedRecordsWriterFactory.java | 3 ++- .../bigquery/dwhassessment/hooks/logger/EventLogger.java | 5 ++--- .../dwhassessment/hooks/logger/LoggingHookConstants.java | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/DatePartitionedRecordsWriterFactory.java b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/DatePartitionedRecordsWriterFactory.java index e362b22..35c0983 100644 --- a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/DatePartitionedRecordsWriterFactory.java +++ b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/DatePartitionedRecordsWriterFactory.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigquery.dwhassessment.hooks.logger; +import static com.google.cloud.bigquery.dwhassessment.hooks.logger.LoggingHookConstants.QUERY_EVENTS_FILE_PREFIX; import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE_TIME; import static java.time.temporal.ChronoField.HOUR_OF_DAY; @@ -204,7 +205,7 @@ private LocalDate getCurrentDate() { } private String constructFileName() { - return "dwhassessment_" + return QUERY_EVENTS_FILE_PREFIX + LOG_TIME_FORMAT.format(clock.instant().atOffset(ZoneOffset.UTC)) + "_" + loggerId diff --git a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventLogger.java b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventLogger.java index f5c9796..ee40377 100644 --- a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventLogger.java +++ b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/EventLogger.java @@ -20,6 +20,8 @@ import static com.google.cloud.bigquery.dwhassessment.hooks.logger.LoggerVarsConfig.HIVE_QUERY_EVENTS_QUEUE_CAPACITY; import static com.google.cloud.bigquery.dwhassessment.hooks.logger.LoggerVarsConfig.HIVE_QUERY_EVENTS_ROLLOVER_ELIGIBILITY_CHECK_INTERVAL; import static com.google.cloud.bigquery.dwhassessment.hooks.logger.LoggerVarsConfig.HIVE_QUERY_EVENTS_ROLLOVER_INTERVAL; +import static com.google.cloud.bigquery.dwhassessment.hooks.logger.LoggingHookConstants.DEFAULT_ROLLOVER_ELIGIBILITY_CHECK_INTERVAL_DURATION; +import static com.google.cloud.bigquery.dwhassessment.hooks.logger.LoggingHookConstants.DEFAULT_ROLLOVER_INTERVAL_DURATION; import static com.google.cloud.bigquery.dwhassessment.hooks.logger.LoggingHookConstants.QUERY_EVENT_SCHEMA; import static java.util.concurrent.TimeUnit.SECONDS; @@ -55,9 +57,6 @@ public class EventLogger { private static final int MAX_RETRIES = 2; private static final Duration SHUTDOWN_WAIT_TIME = Duration.ofSeconds(5); private static final int QUERY_EVENTS_QUEUE_DEFAULT_SIZE = 64; - private static final Duration DEFAULT_ROLLOVER_ELIGIBILITY_CHECK_INTERVAL_DURATION = - Duration.ofMinutes(10); - private static final Duration DEFAULT_ROLLOVER_INTERVAL_DURATION = Duration.ofMinutes(10); private final DatePartitionedRecordsWriterFactory recordsWriterFactory; private final EventRecordConstructor eventRecordConstructor; diff --git a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/LoggingHookConstants.java b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/LoggingHookConstants.java index 81d9e1b..11f0b58 100644 --- a/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/LoggingHookConstants.java +++ b/src/java/com/google/cloud/bigquery/dwhassessment/hooks/logger/LoggingHookConstants.java @@ -17,6 +17,7 @@ package com.google.cloud.bigquery.dwhassessment.hooks.logger; import com.google.cloud.bigquery.dwhassessment.hooks.avro.AvroSchemaLoader; +import java.time.Duration; import org.apache.avro.Schema; public final class LoggingHookConstants { @@ -24,6 +25,12 @@ public final class LoggingHookConstants { public static final Schema QUERY_EVENT_SCHEMA = AvroSchemaLoader.loadSchema("QueryEvents.avsc"); public static final String HOOK_VERSION = "1.0"; + + public static final String QUERY_EVENTS_FILE_PREFIX = "dwhassessment_"; + + public static final Duration DEFAULT_ROLLOVER_ELIGIBILITY_CHECK_INTERVAL_DURATION = + Duration.ofMinutes(10); + public static final Duration DEFAULT_ROLLOVER_INTERVAL_DURATION = Duration.ofHours(1); private LoggingHookConstants() {} }