From d9828438a799428982f7fdcd4da7048d929bf602 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Wed, 2 Oct 2024 13:29:58 -0700 Subject: [PATCH] Add support for publishing maven-metadata.xml Maven repositories normally have a maven-metadata.xml file that indicate to the Maven system what versions are available and which is to be considered the latest version. ```xml com.mycompany.app my-app 1.0 1.0 1.0 20200731090423 ``` At Confluent, we use AWS Code Artifactory which does not mark a Maven package as "published" unless a new maven-metadata.xml is uploaded indicating so. * Add support for reading existing maven-metadata.xml * Add support for adding the new version to the metadata object * Add support to upload the file for http & file protocols * Add small test cases to validate SerDe for Maven metadata object from XML Co-authored-by: Vince Rose Co-authored-by: Na Lou --- MODULE.bazel | 2 + private/rules/java_export.bzl | 6 + private/rules/maven_publish.bzl | 7 +- .../bazelbuild/rules_jvm_external/maven/BUILD | 25 +++ .../maven/MavenPublisher.java | 182 +++++++++++++++++- .../maven/MavenPublisherTest.java | 73 +++++++ repositories.bzl | 2 + rules_jvm_external_deps_install.json | 28 +-- 8 files changed, 308 insertions(+), 17 deletions(-) create mode 100644 private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/MavenPublisherTest.java diff --git a/MODULE.bazel b/MODULE.bazel index 5b834dd51..6329f7871 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -66,6 +66,7 @@ maven.install( "org.apache.maven:maven-core:%s" % _MAVEN_VERSION, "org.apache.maven:maven-model:%s" % _MAVEN_VERSION, "org.apache.maven:maven-model-builder:%s" % _MAVEN_VERSION, + "org.apache.maven:maven-repository-metadata:%s" % _MAVEN_VERSION, "org.apache.maven:maven-settings:%s" % _MAVEN_VERSION, "org.apache.maven:maven-settings-builder:%s" % _MAVEN_VERSION, "org.apache.maven:maven-resolver-provider:%s" % _MAVEN_VERSION, @@ -86,6 +87,7 @@ maven.install( "software.amazon.awssdk:s3:2.26.12", "org.bouncycastle:bcprov-jdk15on:1.68", "org.bouncycastle:bcpg-jdk15on:1.68", + "com.google.http-client:google-http-client:1.45.0", ], fail_if_repin_required = True, fetch_sources = True, diff --git a/private/rules/java_export.bzl b/private/rules/java_export.bzl index d42acf71e..119d71bb9 100644 --- a/private/rules/java_export.bzl +++ b/private/rules/java_export.bzl @@ -15,6 +15,7 @@ def java_export( tags = [], testonly = None, classifier_artifacts = {}, + publish_maven_metadata = False, **kwargs): """Extends `java_library` to allow maven artifacts to be uploaded. @@ -77,6 +78,7 @@ def java_export( `tags = ["no-javadoc"]`). visibility: The visibility of the target kwargs: These are passed to [`java_library`](https://bazel.build/reference/be/java#java_library), + publish_maven_metadata: Whether to publish a maven-metadata.xml and so may contain any valid parameter for that rule. """ @@ -111,6 +113,7 @@ def java_export( testonly = testonly, javadocopts = javadocopts, classifier_artifacts = classifier_artifacts, + publish_maven_metadata = publish_maven_metadata, doc_deps = doc_deps, doc_url = doc_url, doc_resources = doc_resources, @@ -134,6 +137,7 @@ def maven_export( doc_deps = [], doc_url = "", doc_resources = [], + publish_maven_metadata = False, toolchains = None): """ All arguments are the same as java_export with the addition of: @@ -194,6 +198,7 @@ def maven_export( `tags = ["no-javadoc"]`). doc_resources: Resources to be included in the javadoc jar. visibility: The visibility of the target + publish_maven_metadata: Whether to publish a maven-metadata.xml kwargs: These are passed to [`java_library`](https://bazel.build/reference/be/java#java_library), and so may contain any valid parameter for that rule. """ @@ -293,6 +298,7 @@ def maven_export( tags = tags, testonly = testonly, toolchains = toolchains, + publish_maven_metadata = publish_maven_metadata, ) # We may want to aggregate several `java_export` targets into a single Maven BOM POM diff --git a/private/rules/maven_publish.bzl b/private/rules/maven_publish.bzl index bb6879518..28f045a15 100644 --- a/private/rules/maven_publish.bzl +++ b/private/rules/maven_publish.bzl @@ -18,7 +18,7 @@ export USE_IN_MEMORY_PGP_KEYS="${{USE_IN_MEMORY_PGP_KEYS:-{use_in_memory_pgp_key export PGP_SIGNING_KEY="${{PGP_SIGNING_KEY:-{pgp_signing_key}}}" export PGP_SIGNING_PWD="${{PGP_SIGNING_PWD:-{pgp_signing_pwd}}}" echo Uploading "{coordinates}" to "${{MAVEN_REPO}}" -{uploader} "{coordinates}" '{pom}' '{artifact}' '{classifier_artifacts}' $@ +{uploader} "{coordinates}" '{pom}' '{artifact}' '{publish_maven_metadata}' '{classifier_artifacts}' $@ """ def _escape_arg(str): @@ -68,6 +68,7 @@ def _maven_publish_impl(ctx): pom = ctx.file.pom.short_path, artifact = artifacts_short_path, classifier_artifacts = ",".join(["{}={}".format(classifier, file.short_path) for (classifier, file) in classifier_artifacts_dict.items()]), + publish_maven_metadata = ctx.attr.publish_maven_metadata, ), ) @@ -125,6 +126,10 @@ When signing with GPG, the current default key is used. allow_single_file = True, ), "classifier_artifacts": attr.label_keyed_string_dict(allow_files = True), + "publish_maven_metadata": attr.bool( + default = False, + doc = "Whether to publish a maven-metadata.xml to the Maven repository", + ), "_uploader": attr.label( executable = True, cfg = "exec", diff --git a/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/BUILD b/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/BUILD index 744265774..772f44b3b 100644 --- a/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/BUILD +++ b/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/BUILD @@ -42,6 +42,14 @@ java_binary( "org.bouncycastle:bcpg-jdk15on", repository_name = "rules_jvm_external_deps", ), + artifact( + "com.google.http-client:google-http-client", + repository_name = "rules_jvm_external_deps", + ), + artifact( + "org.apache.maven:maven-repository-metadata", + repository_name = "rules_jvm_external_deps", + ), ], ) @@ -57,3 +65,20 @@ java_binary( ), ], ) + +java_test( + name = "MavenPublisherTest", + srcs = [ + "MavenPublisherTest.java", + ], + deps = [ + artifact( + "org.apache.maven:maven-repository-metadata", + repository_name = "rules_jvm_external_deps", + ), + artifact( + "junit:junit", + repository_name = "regression_testing_coursier", + ), + ] +) \ No newline at end of file diff --git a/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/MavenPublisher.java b/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/MavenPublisher.java index 3b8e36165..30bf1325e 100644 --- a/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/MavenPublisher.java +++ b/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/MavenPublisher.java @@ -31,6 +31,7 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.common.base.Splitter; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -57,8 +58,33 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; import java.util.logging.Logger; + +import com.google.common.io.CharStreams; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpResponseException; +import com.google.api.client.http.GenericUrl; + +import java.io.UncheckedIOException; + +import org.apache.maven.artifact.repository.metadata.Metadata; +import org.apache.maven.artifact.repository.metadata.Versioning; +import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader; +import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Writer; + +import java.nio.charset.StandardCharsets; +import java.net.URISyntaxException; +import java.util.Optional; +import java.io.StringReader; +import java.io.ByteArrayOutputStream; +import java.util.stream.Collectors; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.io.InputStreamReader; public class MavenPublisher { @@ -108,8 +134,10 @@ public static void main(String[] args) futures.add(upload(repo, credentials, coords, "." + ext, mainArtifact, signingMetadata)); } - if (args.length > 3 && !args[3].isEmpty()) { - List extraArtifactTuples = Splitter.onPattern(",").splitToList(args[3]); + boolean publishMavenMetadata = Boolean.valueOf(args[3]); + + if (args.length > 4 && !args[4].isEmpty()) { + List extraArtifactTuples = Splitter.onPattern(",").splitToList(args[4]); for (String artifactTuple : extraArtifactTuples) { String[] splits = artifactTuple.split("="); String classifier = splits[0]; @@ -128,6 +156,16 @@ public static void main(String[] args) CompletableFuture all = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + + // uploading the maven-metadata.xml signals to cut over to the new version, so it must be at the end. + if (publishMavenMetadata) { + all = all.thenCompose(Void -> uploadMavenMetadata( + repo, + credentials, + coords + )); + } + all.get(30, MINUTES); } finally { EXECUTOR.shutdown(); @@ -150,6 +188,92 @@ private static boolean isSchemeSupported(String repo) { return false; } + /** + * Download the pre-existing maven-metadata.xml file if it exists. + * If no such file exists, create a default Metadata with the Coordinates provided. + */ + private static CompletableFuture downloadExistingMavenMetadata( + String repo, + Credentials credentials, + Coordinates coords + ) { + String mavenMetadataUrl = + String.format( + "%s/%s/%s/maven-metadata.xml", + repo.replaceAll("/$", ""), + coords.groupId.replace('.', '/'), + coords.artifactId); + + return download(mavenMetadataUrl, credentials).thenApply( optionalFileContents -> { + try { + if (optionalFileContents.isEmpty()) { + // no file so just upload a new one + // we must bootstrap + Metadata metadata = new Metadata(); + metadata.setGroupId(coords.groupId); + metadata.setArtifactId(coords.artifactId); + metadata.setVersioning(new Versioning()); + return metadata; + } + return new MetadataXpp3Reader().read(new StringReader(optionalFileContents.get()), false); + } catch (Exception e) { + throw new RuntimeException(e); + } + + }); + } + + /** + * Upload the new maven-metadata.xml with the new version included in the version list & + * set the latest and release tags in the Metadata XML object. + * This function will first download the pre-existing metadata-xml and augment. + * If no maven-metadata.xml exists, a new one will be hydrated. + */ + private static CompletableFuture uploadMavenMetadata( + String repo, + Credentials credentials, + Coordinates coords + ) { + + String mavenMetadataUrl = + String.format( + "%s/%s/%s/maven-metadata.xml", + repo.replaceAll("/$", ""), + coords.groupId.replace('.', '/'), + coords.artifactId); + return downloadExistingMavenMetadata(repo, credentials, coords).thenCompose( + metadata -> { + try { + + // There is a chance versioning is null; handle it by creating the empty object. + Versioning versioning = Optional.ofNullable(metadata.getVersioning()).orElse(new Versioning()); + versioning.setLatest(coords.version); + versioning.setRelease(coords.version); + // This may be needed for SNAPSHOT support + String timestamp = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); + versioning.setLastUpdated("20200731090423"); + versioning.getVersions() + .add(coords.version); + // Let's handle adding multiple versions many times by turning it back to a set + versioning.setVersions( + versioning.getVersions() + .stream() + .distinct() + .collect(Collectors.toList()) + ); + metadata.setVersioning(versioning); + + Path newMavenMetadataXml = Files.createTempFile("maven-metadata", ".xml"); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + new MetadataXpp3Writer().write(os, metadata); + Files.write(newMavenMetadataXml, os.toByteArray()); + return upload(mavenMetadataUrl, credentials, newMavenMetadataXml); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + private static CompletableFuture upload( String repo, Credentials credentials, @@ -261,6 +385,60 @@ private static String toHexS(String fmt, String algorithm, byte[] toHash) { } } + /** + * Attempts to download the file at the given targetUrl. + * Valid protocols are: http(s) & file at the moment. + */ + private static CompletableFuture> download( + String targetUrl, Credentials credentials + ) { + if (targetUrl.startsWith("http")) { + return CompletableFuture.supplyAsync( () -> { + try { + HttpTransport httpTransport = new NetHttpTransport(); + HttpRequest request = httpTransport.createRequestFactory(initRequest -> { + Map> authHeaders = credentials.getRequestMetadata(); + // Add the authorization headers to the HTTP request + if (authHeaders != null) { + for (Map.Entry> entry : authHeaders.entrySet()) { + initRequest.getHeaders().put(entry.getKey(), entry.getValue()); + } + } + }).buildGetRequest(new GenericUrl(targetUrl)); + HttpResponse response = request.execute(); + return Optional.of( + CharStreams.toString(new InputStreamReader(response.getContent(), StandardCharsets.UTF_8)) + ); + } + catch (HttpResponseException e) { + if (e.getStatusCode() == 404) { + return Optional.empty(); + } else { + throw new UncheckedIOException(e); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } else if (targetUrl.startsWith("file://")) { + return CompletableFuture.supplyAsync( () -> { + try { + Path path = Paths.get(new URL(targetUrl).toURI()); + if (!Files.exists(path)) { + return Optional.empty(); + } + return Optional.of(com.google.common.io.Files.asCharSource(path.toFile(), StandardCharsets.UTF_8).read()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + }); + } else { + throw new IllegalArgumentException("Unsupported protocol for download."); + } + } + private static CompletableFuture upload( String targetUrl, Credentials credentials, Path toUpload) { Callable callable; diff --git a/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/MavenPublisherTest.java b/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/MavenPublisherTest.java new file mode 100644 index 000000000..531cf4b44 --- /dev/null +++ b/private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/MavenPublisherTest.java @@ -0,0 +1,73 @@ +package com.github.bazelbuild.rules_jvm_external.maven; + +import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Writer; +import org.junit.Test; +import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader; +import org.apache.maven.artifact.repository.metadata.Metadata; +import org.apache.maven.artifact.repository.metadata.Versioning; +import java.io.InputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import static org.junit.Assert.*; + +public class MavenPublisherTest { + + @Test + public void testDeserialization() throws Exception { + String xml = "\n" + + " com.mycompany.app\n" + + " my-app\n" + + " \n" + + " 1.0\n" + + " 1.0\n" + + " \n" + + " 1.0\n" + + " \n" + + " 20200731090423\n" + + " \n" + + ""; + Metadata metadata = new MetadataXpp3Reader().read( + new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)), false); + + assertEquals(metadata.getGroupId(), "com.mycompany.app"); + assertEquals(metadata.getArtifactId(), "my-app"); + + Versioning versioning = metadata.getVersioning(); + assertEquals(versioning.getLatest(), "1.0"); + assertEquals(versioning.getLastUpdated(), "20200731090423"); + } + + @Test + public void testSerialization() throws Exception { + String expected = "\n" + + "\n" + + " com.mycompany.app\n" + + " my-app\n" + + " \n" + + " 1.0\n" + + " 1.0\n" + + " \n" + + " 1.0\n" + + " \n" + + " 20200731090423\n" + + " \n" + + "\n"; + Metadata metadata = new Metadata(); + metadata.setGroupId("com.mycompany.app"); + metadata.setArtifactId("my-app"); + Versioning versioning = new Versioning(); + versioning.setLatest("1.0"); + versioning.setRelease("1.0"); + versioning.setLastUpdated("20200731090423"); + versioning.addVersion("1.0"); + metadata.setVersioning(versioning); + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + new MetadataXpp3Writer().write(os, metadata); + String actual = new String(os.toByteArray(), StandardCharsets.UTF_8); + assertEquals(actual, expected); + } + + +} diff --git a/repositories.bzl b/repositories.bzl index a70725a1c..16dc96c40 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -79,6 +79,7 @@ def rules_jvm_external_deps( "org.apache.maven:maven-core:%s" % _MAVEN_VERSION, "org.apache.maven:maven-model:%s" % _MAVEN_VERSION, "org.apache.maven:maven-model-builder:%s" % _MAVEN_VERSION, + "org.apache.maven:maven-repository-metadata:%s" % _MAVEN_VERSION, "org.apache.maven:maven-settings:%s" % _MAVEN_VERSION, "org.apache.maven:maven-settings-builder:%s" % _MAVEN_VERSION, "org.apache.maven:maven-resolver-provider:%s" % _MAVEN_VERSION, @@ -99,6 +100,7 @@ def rules_jvm_external_deps( "software.amazon.awssdk:s3:2.26.12", "org.bouncycastle:bcprov-jdk15on:1.68", "org.bouncycastle:bcpg-jdk15on:1.68", + "com.google.http-client:google-http-client:1.45.0", ], maven_install_json = deps_lock_file, fail_if_repin_required = True, diff --git a/rules_jvm_external_deps_install.json b/rules_jvm_external_deps_install.json index c5aaaf9e5..b348d6b68 100644 --- a/rules_jvm_external_deps_install.json +++ b/rules_jvm_external_deps_install.json @@ -1,7 +1,7 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": -951616327, - "__RESOLVED_ARTIFACTS_HASH": -84218881, + "__INPUT_ARTIFACTS_HASH": -1749186864, + "__RESOLVED_ARTIFACTS_HASH": -1920773832, "artifacts": { "aopalliance:aopalliance": { "shasums": { @@ -166,10 +166,10 @@ }, "com.google.errorprone:error_prone_annotations": { "shasums": { - "jar": "f3fc8a3a0a4020706a373b00e7f57c2512dd26d1f83d28c7d38768f8682b231e", - "sources": "2936e9b315d790d8a6364f0574bcec9c8b2d78688b317e1765c4a16f9ef80632" + "jar": "144f3aefbd6e27daec55d3753b2c6b13c1afdaf0cf04816cdb564588ed92f1bd", + "sources": "8a05d3e543d7cbefb8e035edc9a92d1b5562d36761754f4d881ae66186c49de1" }, - "version": "2.28.0" + "version": "2.30.0" }, "com.google.googlejavaformat:google-java-format": { "shasums": { @@ -201,10 +201,10 @@ }, "com.google.http-client:google-http-client": { "shasums": { - "jar": "390618d7b51704240b8fd28e1230fa35d220f93f4b4ba80f63e38db00dacb09e", - "sources": "9419537a2973195619b43f76be92388b1e37a785503717d76afff5764884ebc2" + "jar": "8340bbaf4410bd25921842e424cac3db9c916fdf47adfe73b8cba8dba7a06103", + "sources": "638c13eeffbef4bdb551d705f32885cb87247fa41d2e59754d05b131550a7681" }, - "version": "1.44.2" + "version": "1.45.0" }, "com.google.http-client:google-http-client-apache-v2": { "shasums": { @@ -299,10 +299,10 @@ }, "io.grpc:grpc-api": { "shasums": { - "jar": "2e896944cf513e0e5cfd32bcd72c89601a27c6ca56916f84b20f3a13bacf1b1f", - "sources": "aa2974982805cc998f79e7c4d5d536744fd5520b56eb15b0179f9331c1edb3b7" + "jar": "8fadb1f4f0a18971c082497f34cbb78a51897ca8af4b212aa2a99c7de9ad995c", + "sources": "6d0df2072702a1badfaaab3cce14f2629d4eaec85cf696347561bec19736ce8c" }, - "version": "1.62.2" + "version": "1.66.0" }, "io.grpc:grpc-auth": { "shasums": { @@ -313,10 +313,10 @@ }, "io.grpc:grpc-context": { "shasums": { - "jar": "9959747df6a753119e1c1a3dff01aa766d2455f5e4860acaa305359e1d533a05", - "sources": "c656b874e58c84ca975c3708f2e001dba76233385b6a5b7cb098868bd6ce38b1" + "jar": "7b7521aa2116014d08dc08825e13d70eac8eb646d09dd44980b6f4d1883e6713", + "sources": "c4638347d6d0964eb7a3987cd9d943554cbdf5e334b26f6d119dfc0e85fb1899" }, - "version": "1.62.2" + "version": "1.66.0" }, "io.grpc:grpc-core": { "shasums": {