Make fileSize optional

acvictor · May 31, 2024 · 24bd93c · 24bd93c
1 parent d45a0f4
commit 24bd93c
Show file tree

Hide file tree

Showing 7 changed files with 10 additions and 13 deletions.
diff --git a/velox/connectors/hive/FileHandle.cpp b/velox/connectors/hive/FileHandle.cpp
@@ -52,9 +52,7 @@ std::unique_ptr<FileHandle> FileHandleGenerator::operator()(
     fileHandle = std::make_unique<FileHandle>();
     filesystems::FileOptions options;
     if (properties) {
-      options.fileSize = properties->fileSize == -1
-          ? std::nullopt
-          : std::optional<int64_t>{properties->fileSize};
+      options.fileSize = properties->fileSize;
     }
     fileHandle->file = filesystems::getFileSystem(filename, properties_)
                            ->openFileForRead(filename, options);

diff --git a/velox/connectors/hive/FileProperties.h b/velox/connectors/hive/FileProperties.h
@@ -30,8 +30,8 @@
 namespace facebook::velox {
 
 struct FileProperties {
-  int64_t fileSize{-1};
-  int64_t modificationTime{-1};
+  std::optional<int64_t> fileSize;
+  std::optional<int64_t> modificationTime;
 };
 
 } // namespace facebook::velox
diff --git a/velox/connectors/hive/iceberg/IcebergSplit.cpp b/velox/connectors/hive/iceberg/IcebergSplit.cpp
@@ -26,7 +26,7 @@ HiveIcebergSplit::HiveIcebergSplit(
     dwio::common::FileFormat _fileFormat,
     uint64_t _start,
     uint64_t _length,
-    FileProperties _properties,
+    std::optional<FileProperties> _properties,
     const std::unordered_map<std::string, std::optional<std::string>>&
         _partitionKeys,
     std::optional<int32_t> _tableBucketNumber,
@@ -57,7 +57,7 @@ HiveIcebergSplit::HiveIcebergSplit(
     dwio::common::FileFormat _fileFormat,
     uint64_t _start,
     uint64_t _length,
-    FileProperties _properties,
+    std::optional<FileProperties> _properties,
     const std::unordered_map<std::string, std::optional<std::string>>&
         _partitionKeys,
     std::optional<int32_t> _tableBucketNumber,

diff --git a/velox/connectors/hive/iceberg/IcebergSplit.h b/velox/connectors/hive/iceberg/IcebergSplit.h
@@ -32,7 +32,7 @@ struct HiveIcebergSplit : public connector::hive::HiveConnectorSplit {
       dwio::common::FileFormat _fileFormat,
       uint64_t _start = 0,
       uint64_t _length = std::numeric_limits<uint64_t>::max(),
-      FileProperties fileProperties = {},
+      std::optional<FileProperties> fileProperties = std::nullopt,
       const std::unordered_map<std::string, std::optional<std::string>>&
           _partitionKeys = {},
       std::optional<int32_t> _tableBucketNumber = std::nullopt,
@@ -47,7 +47,7 @@ struct HiveIcebergSplit : public connector::hive::HiveConnectorSplit {
       dwio::common::FileFormat _fileFormat,
       uint64_t _start = 0,
       uint64_t _length = std::numeric_limits<uint64_t>::max(),
-      FileProperties fileProperties = {},
+      std::optional<FileProperties> fileProperties = std::nullopt,
       const std::unordered_map<std::string, std::optional<std::string>>&
           _partitionKeys = {},
       std::optional<int32_t> _tableBucketNumber = std::nullopt,

diff --git a/velox/connectors/hive/iceberg/tests/IcebergReadTest.cpp b/velox/connectors/hive/iceberg/tests/IcebergReadTest.cpp
@@ -163,7 +163,7 @@ class HiveIcebergTest : public HiveConnectorTestBase {
         fileFomat_,
         0,
         fileSize,
-        FileProperties(),
+        std::nullopt,
         partitionKeys,
         std::nullopt,
         customSplitInfo,

diff --git a/velox/connectors/hive/tests/HiveConnectorUtilTest.cpp b/velox/connectors/hive/tests/HiveConnectorUtilTest.cpp
@@ -74,7 +74,7 @@ TEST_F(HiveConnectorUtilTest, configureReaderOptions) {
         fileFormat,
         0UL,
         std::numeric_limits<uint64_t>::max(),
-        facebook::velox::FileProperties(),
+        std::nullopt,
         partitionKeys,
         std::nullopt,
         customSplitInfo,

diff --git a/velox/exec/tests/utils/HiveConnectorTestBase.h b/velox/exec/tests/utils/HiveConnectorTestBase.h
@@ -284,7 +284,7 @@ class HiveConnectorSplitBuilder {
         fileFormat_,
         start_,
         length_,
-        properties_,
+        std::nullopt,
         partitionKeys_,
         tableBucketNumber_,
         customSplitInfo,
@@ -299,7 +299,6 @@ class HiveConnectorSplitBuilder {
   dwio::common::FileFormat fileFormat_{dwio::common::FileFormat::DWRF};
   uint64_t start_{0};
   uint64_t length_{std::numeric_limits<uint64_t>::max()};
-  facebook::velox::FileProperties properties_;
   std::unordered_map<std::string, std::optional<std::string>> partitionKeys_;
   std::optional<int32_t> tableBucketNumber_;
   std::unordered_map<std::string, std::string> customSplitInfo_ = {};