From 0cc0529955ac034b1392723e8547642915fd7a96 Mon Sep 17 00:00:00 2001 From: ZhouJinsong Date: Mon, 6 Jan 2025 10:11:58 +0800 Subject: [PATCH] [AMORO-3391] Fix minor optimizing repeated issue (#3392) * Fix minor optimizing repeated issue * Fix segmentShouldRewritePos issue --- .../plan/CommonPartitionEvaluator.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java index a69704978a..fe7fc42713 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java @@ -235,16 +235,27 @@ public boolean fileShouldRewrite(DataFile dataFile, List> deletes public boolean segmentShouldRewritePos(DataFile dataFile, List> deletes) { Preconditions.checkArgument(!isFragmentFile(dataFile), "Unsupported fragment file."); - long posDeleteFileCount = - deletes.stream().filter(delete -> delete.content() == FileContent.POSITION_DELETES).count(); - if (posDeleteFileCount == 1) { - return !TableFileUtil.isOptimizingPosDeleteFile( - dataFile.path().toString(), deletes.get(0).path().toString()); - } else if (posDeleteFileCount > 1) { + long equalDeleteFileCount = 0; + long posDeleteFileCount = 0; + + for (ContentFile delete : deletes) { + if (delete.content() == FileContent.EQUALITY_DELETES) { + equalDeleteFileCount++; + } else if (delete.content() == FileContent.POSITION_DELETES) { + posDeleteFileCount++; + } + } + if (posDeleteFileCount > 1) { combinePosSegmentFileCount++; return true; + } else if (equalDeleteFileCount > 0) { + return true; + } else if (posDeleteFileCount == 1) { + return !TableFileUtil.isOptimizingPosDeleteFile( + dataFile.path().toString(), deletes.get(0).path().toString()); + } else { + return false; } - return deletes.stream().anyMatch(delete -> delete.content() == FileContent.EQUALITY_DELETES); } protected boolean isFullOptimizing() {