feat: add confluence source

langstream-agents/langstream-agents-atlassian/pom.xml

@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright DataStax, Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>langstream-agents</artifactId>
+    <groupId>ai.langstream</groupId>
+    <version>0.23.1-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>langstream-agents-atlassian</artifactId>
+  <dependencyManagement>
+
+  </dependencyManagement>
+
+
+  <dependencies>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>langstream-api</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>langstream-agents-commons</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>ai.langstream</groupId>
+      <artifactId>langstream-agents-commons-storage-provider</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.dropbox.core</groupId>
+      <artifactId>dropbox-core-sdk</artifactId>
+      <version>7.0.0</version>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.azure</groupId>
+      <artifactId>azure-core</artifactId>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.nifi</groupId>
+        <artifactId>nifi-nar-maven-plugin</artifactId>
+        <extensions>true</extensions>
+        <configuration>
+          <classifier>nar</classifier>
+        </configuration>
+        <executions>
+          <execution>
+            <id>default-nar</id>
+            <phase>package</phase>
+            <goals>
+              <goal>nar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

langstream-agents/langstream-agents-atlassian/src/main/java/ai/langstream/agents/atlassian/AtlassianAgentsCodeProvider.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.agents.atlassian;
+
+import ai.langstream.agents.atlassian.confluence.ConfluenceSource;
+import ai.langstream.api.runner.code.AgentCode;
+import ai.langstream.api.runner.code.AgentCodeProvider;
+import java.util.List;
+
+public class AtlassianAgentsCodeProvider implements AgentCodeProvider {
+
+    public static final String CONFLUENCE_SOURCE = "confluence-source";
+    private static final List<String> AGENTS = List.of(CONFLUENCE_SOURCE);
+
+    @Override
+    public boolean supports(String agentType) {
+        return AGENTS.contains(agentType);
+    }
+
+    @Override
+    public AgentCode createInstance(String agentType) {
+        switch (agentType) {
+            case CONFLUENCE_SOURCE:
+                return new ConfluenceSource();
+            default:
+                throw new IllegalArgumentException("Unsupported agent type: " + agentType);
+        }
+    }
+}

langstream-agents/langstream-agents-atlassian/src/main/java/ai/langstream/agents/atlassian/confluence/ConfluenceSource.java

@@ -0,0 +1,145 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.agents.atlassian.confluence;
+
+import ai.langstream.agents.atlassian.confluence.client.ConfluencePage;
+import ai.langstream.agents.atlassian.confluence.client.ConfluenceRestAPIClient;
+import ai.langstream.agents.atlassian.confluence.client.ConfluenceSpace;
+import ai.langstream.ai.agents.commons.storage.provider.StorageProviderObjectReference;
+import ai.langstream.ai.agents.commons.storage.provider.StorageProviderSource;
+import ai.langstream.ai.agents.commons.storage.provider.StorageProviderSourceState;
+import ai.langstream.api.runner.code.Header;
+import ai.langstream.api.runner.code.SimpleRecord;
+import ai.langstream.api.util.ConfigurationUtils;
+import com.dropbox.core.v2.files.*;
+
+import java.util.*;
+import java.util.function.Consumer;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class ConfluenceSource extends StorageProviderSource<ConfluenceSource.ConfluenceSourceState> {
+
+    public static class ConfluenceSourceState extends StorageProviderSourceState {}
+
+    private ConfluenceRestAPIClient client;
+
+    private List<String> spaces;
+    private Set<String> rootParents;
+
+    @Override
+    public Class<ConfluenceSourceState> getStateClass() {
+        return ConfluenceSourceState.class;
+    }
+
+    @Override
+    public void initializeClientAndConfig(Map<String, Object> configuration) {
+        String username =
+                ConfigurationUtils.requiredField(
+                        configuration, "username", () -> "confluence source");
+        String apiToken =
+                ConfigurationUtils.requiredField(
+                        configuration, "api-token", () -> "confluence source");
+        String domain =
+                ConfigurationUtils.requiredField(
+                        configuration, "domain", () -> "confluence source");
+        client = new ConfluenceRestAPIClient(username, apiToken, domain);
+        initializeConfig(configuration);
+    }
+
+    void initializeConfig(Map<String, Object> configuration) {
+        spaces = ConfigurationUtils.getList("spaces", configuration);
+        if (spaces.isEmpty()) {
+            throw new IllegalArgumentException("At least one space (name or key) must be specified");
+        }
+        rootParents = ConfigurationUtils.getSet("root-parents", configuration);
+    }
+
+    @Override
+    public String getBucketName() {
+        return "";
+    }
+
+    @Override
+    public boolean isDeleteObjects() {
+        return false;
+    }
+
+    @Override
+    public Collection<StorageProviderObjectReference> listObjects() throws Exception {
+        List<StorageProviderObjectReference> collect = new ArrayList<>();
+        for (String space : spaces) {
+            List<ConfluenceSpace> spacesForSpace = client.findSpaceByNameOrKeyOrId(space);
+            if (spacesForSpace.isEmpty()) {
+                log.error("Space {} not found, make sure you inserted the name or the key or the id of the space", space);
+                continue;
+            }
+            for (ConfluenceSpace confluenceSpace : spacesForSpace) {
+                log.info("Found space {}", confluenceSpace);
+                int before = collect.size();
+                client.visitSpacePages(confluenceSpace.id(), rootParents, confluencePage -> collect.add(new StorageProviderObjectReference() {
+                    @Override
+                    public String id() {
+                        return confluencePage.id();
+                    }
+
+                    @Override
+                    public long size() {
+                        return -1;
+                    }
+
+                    @Override
+                    public String contentDigest() {
+                        return confluencePage.pageVersion();
+                    }
+
+                    @Override
+                    public Collection<Header> additionalRecordHeaders() {
+                        return List.of(
+                                SimpleRecord.SimpleHeader.of("confluence-space-name", confluenceSpace.name()),
+                                SimpleRecord.SimpleHeader.of("confluence-space-key", confluenceSpace.key()),
+                                SimpleRecord.SimpleHeader.of("confluence-space-id", confluenceSpace.key()),
+                                SimpleRecord.SimpleHeader.of("confluence-page-title", confluencePage.title())
+                                );
+                    }
+                }));
+                log.info("Found {} pages in space {}", collect.size() - before, confluenceSpace);
+            }
+        }
+        return collect;
+    }
+
+    @Override
+    public byte[] downloadObject(StorageProviderObjectReference object) throws Exception {
+        try {
+            return client.exportPage(object.id());
+        } catch (Exception e) {
+            log.error("Error downloading page {} ({})", object.id(), object.additionalRecordHeaders(), e);
+            throw e;
+        }
+    }
+
+    @Override
+    public void deleteObject(String id) throws Exception {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean isStateStorageRequired() {
+        return true;
+    }
+}

langstream-agents/langstream-agents-atlassian/src/main/java/ai/langstream/agents/atlassian/confluence/client/ConfluencePage.java

@@ -0,0 +1,5 @@
+package ai.langstream.agents.atlassian.confluence.client;
+
+public record ConfluencePage(long spaceId, String id, String title, String pageVersion) {
+
+}