-
Notifications
You must be signed in to change notification settings - Fork 30
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Modify Jenkins Full E2E Integ Test to perform Transformations #1182
Merged
lewijacn
merged 21 commits into
opensearch-project:main
from
lewijacn:add-transform-to-integ
Jan 3, 2025
Merged
Changes from 19 commits
Commits
Show all changes
21 commits
Select commit
Hold shift + click to select a range
569a292
Add basic name change transform to verify
lewijacn 413136f
Revert ES5 change and attempt RFS transform
lewijacn e171f0f
Update to base64
lewijacn 90cee67
Update to add comma
lewijacn d0d91a3
Update pass unique id and use proper arg
lewijacn 79fe4bd
Modify RFS transformation
lewijacn 79b7af0
Update transform
lewijacn 01a7d7c
Modify transformations for RFS and Replayer
lewijacn b2cc0e1
Remove Replayer transformation for now
lewijacn fae7986
Try replayer transformation
lewijacn a6f51ef
Update checks
lewijacn 77c9577
Merge remote-tracking branch 'origin/main' into add-transform-to-integ
lewijacn d3f0d21
Handle linting fixes
lewijacn 552d7f7
Fix doc count
lewijacn c5fba6c
Minor naming and cleanup
lewijacn b189282
Testing refactoring and addressing PR comments
lewijacn ecf71dc
Merge remote-tracking branch 'origin/main' into add-transform-to-integ
lewijacn 0e2f453
Handle changes from main and refactor further
lewijacn 34c967a
Merge remote-tracking branch 'origin/main' into add-transform-to-integ
lewijacn 9bc8714
Merge remote-tracking branch 'origin/main' into add-transform-to-integ
lewijacn 279e76c
nit: fix typo
lewijacn File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
215 changes: 215 additions & 0 deletions
215
...gration/src/test/java/org/opensearch/migrations/bulkload/CustomRfsTransformationTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,215 @@ | ||
package org.opensearch.migrations.bulkload; | ||
|
||
import java.nio.file.Files; | ||
import java.util.Arrays; | ||
import java.util.HashMap; | ||
import java.util.Map; | ||
import java.util.concurrent.CompletableFuture; | ||
import java.util.function.Consumer; | ||
import java.util.function.Function; | ||
import java.util.stream.Stream; | ||
|
||
import org.opensearch.migrations.CreateSnapshot; | ||
import org.opensearch.migrations.bulkload.common.RestClient; | ||
import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; | ||
import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; | ||
import org.opensearch.migrations.bulkload.http.ClusterOperations; | ||
import org.opensearch.migrations.bulkload.http.SearchClusterRequests; | ||
import org.opensearch.migrations.reindexer.tracing.DocumentMigrationTestContext; | ||
import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; | ||
|
||
import lombok.SneakyThrows; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.hamcrest.MatcherAssert; | ||
import org.hamcrest.Matchers; | ||
import org.json.JSONArray; | ||
import org.json.JSONObject; | ||
import org.junit.jupiter.api.Assertions; | ||
import org.junit.jupiter.api.Tag; | ||
import org.junit.jupiter.api.Test; | ||
import org.testcontainers.containers.Network; | ||
|
||
|
||
@Slf4j | ||
@Tag("isolatedTest") | ||
public class CustomRfsTransformationTest extends SourceTestBase { | ||
|
||
public static final String TARGET_DOCKER_HOSTNAME = "target"; | ||
public static final String SNAPSHOT_NAME = "test_snapshot"; | ||
|
||
@Test | ||
public void testCustomTransformationProducesDesiredTargetClusterState() { | ||
String nameTransformation = createIndexNameTransformation("geonames", "geonames_transformed"); | ||
var expectedSourceMap = new HashMap<String, Integer>(); | ||
expectedSourceMap.put("geonames", 1); | ||
var expectedTargetMap = new HashMap<String, Integer>(); | ||
expectedTargetMap.put("geonames_transformed", 1); | ||
String[] transformationArgs = { | ||
"--doc-transformer-config", | ||
nameTransformation, | ||
}; | ||
int totalSourceShards = 1; | ||
Consumer<ClusterOperations> loadDataIntoSource = cluster -> { | ||
// Number of default shards is different across different versions on ES/OS. | ||
// So we explicitly set it. | ||
String body = String.format( | ||
"{" + | ||
" \"settings\": {" + | ||
" \"index\": {" + | ||
" \"number_of_shards\": %d," + | ||
" \"number_of_replicas\": 0" + | ||
" }" + | ||
" }" + | ||
"}", | ||
totalSourceShards | ||
); | ||
cluster.createIndex("geonames", body); | ||
cluster.createDocument("geonames", "111", "{\"author\":\"Tobias Funke\", \"category\": \"cooking\"}"); | ||
}; | ||
runTestProcess( | ||
transformationArgs, | ||
expectedSourceMap, | ||
expectedTargetMap, | ||
loadDataIntoSource, | ||
totalSourceShards, | ||
SourceTestBase::runProcessAgainstTarget | ||
); | ||
} | ||
|
||
@SneakyThrows | ||
private void runTestProcess( | ||
String[] transformationArgs, | ||
Map<String, Integer> expectedSourceDocs, | ||
Map<String, Integer> expectedTargetDocs, | ||
Consumer<ClusterOperations> preloadDataOperations, | ||
Integer numberOfShards, | ||
Function<String[], Integer> processRunner) | ||
{ | ||
final var testSnapshotContext = SnapshotTestContext.factory().noOtelTracking(); | ||
|
||
var tempDirSnapshot = Files.createTempDirectory("opensearchMigrationReindexFromSnapshot_test_snapshot"); | ||
var tempDirLucene = Files.createTempDirectory("opensearchMigrationReindexFromSnapshot_test_lucene"); | ||
|
||
try ( | ||
var esSourceContainer = new SearchClusterContainer(SearchClusterContainer.ES_V7_10_2) | ||
.withAccessToHost(true); | ||
var network = Network.newNetwork(); | ||
var osTargetContainer = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0) | ||
.withAccessToHost(true) | ||
.withNetwork(network) | ||
.withNetworkAliases(TARGET_DOCKER_HOSTNAME); | ||
) { | ||
CompletableFuture.allOf( | ||
CompletableFuture.runAsync(esSourceContainer::start), | ||
CompletableFuture.runAsync(osTargetContainer::start) | ||
).join(); | ||
|
||
var sourceClusterOperations = new ClusterOperations(esSourceContainer.getUrl()); | ||
preloadDataOperations.accept(sourceClusterOperations); | ||
|
||
// Create the snapshot from the source cluster | ||
var args = new CreateSnapshot.Args(); | ||
args.snapshotName = SNAPSHOT_NAME; | ||
args.fileSystemRepoPath = SearchClusterContainer.CLUSTER_SNAPSHOT_DIR; | ||
args.sourceArgs.host = esSourceContainer.getUrl(); | ||
|
||
var snapshotCreator = new CreateSnapshot(args, testSnapshotContext.createSnapshotCreateContext()); | ||
snapshotCreator.run(); | ||
esSourceContainer.copySnapshotData(tempDirSnapshot.toString()); | ||
|
||
String[] processArgs = { | ||
"--snapshot-name", | ||
SNAPSHOT_NAME, | ||
"--snapshot-local-dir", | ||
tempDirSnapshot.toString(), | ||
"--lucene-dir", | ||
tempDirLucene.toString(), | ||
"--target-host", | ||
osTargetContainer.getUrl(), | ||
"--documents-per-bulk-request", | ||
"5", | ||
"--max-connections", | ||
"4", | ||
"--source-version", | ||
"ES_7_10" | ||
}; | ||
String[] completeArgs = Stream.concat(Arrays.stream(processArgs), Arrays.stream(transformationArgs)).toArray(String[]::new); | ||
|
||
// Perform RFS process for each shard | ||
for(int i = 0; i < numberOfShards; i++) { | ||
int exitCode = processRunner.apply(completeArgs); | ||
log.atInfo().setMessage("Process exited with code: {}").addArgument(exitCode).log(); | ||
} | ||
|
||
// Assert doc count on the source and target cluster match expected | ||
validateFinalClusterDocs( | ||
esSourceContainer, | ||
osTargetContainer, | ||
DocumentMigrationTestContext.factory().noOtelTracking(), | ||
expectedSourceDocs, | ||
expectedTargetDocs | ||
); | ||
} finally { | ||
deleteTree(tempDirSnapshot); | ||
} | ||
} | ||
|
||
// Create a simple Jolt transform which matches documents of a given index name in a snpahost and changes that | ||
// index name to a desired index name when migrated to the target cluster | ||
private static String createIndexNameTransformation(String existingIndexName, String newIndexName) { | ||
JSONArray rootArray = new JSONArray(); | ||
JSONObject firstObject = new JSONObject(); | ||
JSONArray jsonConditionalTransformerProvider = new JSONArray(); | ||
|
||
// JsonJMESPathPredicateProvider object | ||
JSONObject jsonJMESPathPredicateProvider = new JSONObject(); | ||
jsonJMESPathPredicateProvider.put("script", String.format("index._index == '%s'", existingIndexName)); | ||
JSONObject jsonJMESPathPredicateWrapper = new JSONObject(); | ||
jsonJMESPathPredicateWrapper.put("JsonJMESPathPredicateProvider", jsonJMESPathPredicateProvider); | ||
jsonConditionalTransformerProvider.put(jsonJMESPathPredicateWrapper); | ||
|
||
JSONArray transformerList = new JSONArray(); | ||
|
||
// First JsonJoltTransformerProvider | ||
JSONObject firstJoltTransformer = new JSONObject(); | ||
JSONObject firstJoltScript = new JSONObject(); | ||
firstJoltScript.put("operation", "modify-overwrite-beta"); | ||
firstJoltScript.put("spec", new JSONObject().put("index", new JSONObject().put("\\_index", newIndexName))); | ||
firstJoltTransformer.put("JsonJoltTransformerProvider", new JSONObject().put("script", firstJoltScript)); | ||
transformerList.put(firstJoltTransformer); | ||
|
||
jsonConditionalTransformerProvider.put(transformerList); | ||
firstObject.put("JsonConditionalTransformerProvider", jsonConditionalTransformerProvider); | ||
rootArray.put(firstObject); | ||
return rootArray.toString(); | ||
} | ||
|
||
private static void validateFinalClusterDocs( | ||
SearchClusterContainer esSourceContainer, | ||
SearchClusterContainer osTargetContainer, | ||
DocumentMigrationTestContext context, | ||
Map<String, Integer> expectedSourceDocs, | ||
Map<String, Integer> expectedTargetDocs | ||
) { | ||
var targetClient = new RestClient(ConnectionContextTestParams.builder() | ||
.host(osTargetContainer.getUrl()) | ||
.build() | ||
.toConnectionContext() | ||
); | ||
var sourceClient = new RestClient(ConnectionContextTestParams.builder() | ||
.host(esSourceContainer.getUrl()) | ||
.build() | ||
.toConnectionContext() | ||
); | ||
|
||
var requests = new SearchClusterRequests(context); | ||
var sourceMap = requests.getMapOfIndexAndDocCount(sourceClient); | ||
var refreshResponse = targetClient.get("_refresh", context.createUnboundRequestContext()); | ||
Assertions.assertEquals(200, refreshResponse.statusCode); | ||
var targetMap = requests.getMapOfIndexAndDocCount(targetClient); | ||
|
||
MatcherAssert.assertThat(sourceMap, Matchers.equalTo(expectedSourceDocs)); | ||
MatcherAssert.assertThat(targetMap, Matchers.equalTo(expectedTargetDocs)); | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Non-blocking nit - typo (
snpahost
)There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed now 👍