Fix S3MockTest [HZ-1734] (#22779)

Fixes #22731 - Switched test s3 client to use pathstyle urls ([Default behaviour has changed in AWS SDK 2.18](adobe/S3Mock#880)) - Made S3 tests more isolated - Cleaned up dependency and remove home-made S3MockContainer - Update S3Mock to newer version 2.4.14. Starting from 2.4.16 it fails when using `GetBucketLocation` operation
hazelcast · Nov 14, 2022 · 56e4ef0 · 56e4ef0
1 parent 03f7a72
commit 56e4ef0
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 113 deletions.
diff --git a/checkstyle/checkstyle_jet.xml b/checkstyle/checkstyle_jet.xml
@@ -290,7 +290,8 @@
         <module name="RegexpSinglelineJava">
             <property name="format" value="(import static java\.util\.UUID\.randomUUID|UUID\.randomUUID\(\))"/>
             <property name="ignoreComments" value="true"/>
-            <property name="message" value="Please use UuidUtil.buildRandomUUID() instead of UUID.randomUUID()!"/>
+            <property name="message"
+                      value="Please use UuidUtil.newUnsecureUUID() or UuidUtil.newSecureUUID() instead of UUID.randomUUID()"/>
         </module>
     </module>
 </module>
diff --git a/extensions/s3/pom.xml b/extensions/s3/pom.xml
@@ -76,24 +76,18 @@
         </dependency>
 
         <!-- TEST -->
-        <dependency>
-            <groupId>com.adobe.testing</groupId>
-            <artifactId>s3mock</artifactId>
-            <version>2.4.14</version>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.springframework.boot</groupId>
-                    <artifactId>spring-boot-starter-logging</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>
             <version>${testcontainers.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.adobe.testing</groupId>
+            <artifactId>s3mock-testcontainers</artifactId>
+            <!-- Remember to update tag in S3MockContainer constructor as well -->
+            <version>2.8.0</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-slf4j-impl</artifactId>

diff --git a/extensions/s3/src/test/java/com/hazelcast/jet/s3/S3MockContainer.java b/extensions/s3/src/test/java/com/hazelcast/jet/s3/S3MockContainer.java
diff --git a/extensions/s3/src/test/java/com/hazelcast/jet/s3/S3MockTest.java b/extensions/s3/src/test/java/com/hazelcast/jet/s3/S3MockTest.java
@@ -16,7 +16,9 @@
 
 package com.hazelcast.jet.s3;
 
+import com.adobe.testing.s3mock.testcontainers.S3MockContainer;
 import com.hazelcast.function.SupplierEx;
+import com.hazelcast.internal.util.UuidUtil;
 import com.hazelcast.jet.pipeline.Pipeline;
 import com.hazelcast.jet.pipeline.test.TestSources;
 import com.hazelcast.jet.s3.S3Sinks.S3SinkContext;
@@ -33,12 +35,16 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
 import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
 import java.util.stream.Collectors;
 
 import static com.hazelcast.test.DockerTestUtil.assumeDockerEnabled;
@@ -51,29 +57,31 @@
 @RunWith(HazelcastSerialClassRunner.class)
 @Category({QuickTest.class, ParallelJVMTest.class, IgnoreInJenkinsOnWindows.class})
 public class S3MockTest extends S3TestBase {
+    private static final int LINE_COUNT = 100;
 
     private static S3MockContainer s3MockContainer;
 
+    private static S3Client s3Client;
+
     private static final ILogger logger = Logger.getLogger(S3MockTest.class);
-    private static final String SOURCE_BUCKET = "source-bucket";
-    private static final String SOURCE_BUCKET_2 = "source-bucket-2";
-    private static final String SOURCE_BUCKET_EMPTY = "source-bucket-empty";
-    private static final String SINK_BUCKET = "sink-bucket";
-    private static final String SINK_BUCKET_OVERWRITE = "sink-bucket-overwrite";
-    private static final String SINK_BUCKET_NONASCII = "sink-bucket-nonascii";
+    private final UUID ID = UuidUtil.newUnsecureUUID();
+    private final String SOURCE_BUCKET = "source-bucket-" + ID;
+    private final String SOURCE_BUCKET_2 = "source-bucket-2-" + ID;
+    private final String SOURCE_BUCKET_EMPTY = "source-bucket-empty-" + ID;
+    private final String SINK_BUCKET = "sink-bucket-" + ID;
+    private final String SINK_BUCKET_OVERWRITE = "sink-bucket-overwrite-" + ID;
+    private final String SINK_BUCKET_NONASCII = "sink-bucket-nonascii-" + ID;
 
-    private static final int LINE_COUNT = 100;
 
-    private static S3Client s3Client;
 
 
     @BeforeClass
     public static void setupS3() {
         assumeDockerEnabled();
-        s3MockContainer = new S3MockContainer();
+        s3MockContainer = new S3MockContainer("2.4.14");
         s3MockContainer.start();
         s3MockContainer.followOutput(outputFrame -> logger.info(outputFrame.getUtf8String().trim()));
-        s3Client = s3MockContainer.client();
+        s3Client = s3Client(s3MockContainer.getHttpEndpoint());
     }
 
     @AfterClass
@@ -92,12 +100,6 @@ public static void teardown() {
     @Before
     public void setup() {
         S3SinkContext.maximumPartNumber = 1;
-        deleteBucket(s3Client, SOURCE_BUCKET);
-        deleteBucket(s3Client, SOURCE_BUCKET_2);
-        deleteBucket(s3Client, SOURCE_BUCKET_EMPTY);
-        deleteBucket(s3Client, SINK_BUCKET);
-        deleteBucket(s3Client, SINK_BUCKET_OVERWRITE);
-        deleteBucket(s3Client, SINK_BUCKET_NONASCII);
     }
 
     @After
@@ -247,7 +249,7 @@ public void when_sinkWithNonAsciiSymbolInFile() {
     }
 
     SupplierEx<S3Client> clientSupplier() {
-        return () -> S3MockContainer.client(s3MockContainer.endpointURL());
+        return () -> s3Client(s3MockContainer.getHttpEndpoint());
     }
 
     private void generateAndUploadObjects(String bucketName, String prefix, int objectCount, int lineCount) {
@@ -267,4 +269,14 @@ private void generateAndUploadObjects(String bucketName, String prefix, int obje
             builder.setLength(0);
         }
     }
+
+    private static S3Client s3Client(String endpointURL) {
+        return S3Client
+                .builder()
+                .credentialsProvider(AnonymousCredentialsProvider.create())
+                .region(Region.US_EAST_1)
+                .endpointOverride(URI.create(endpointURL))
+                .forcePathStyle(true)
+                .build();
+    }
 }