Merge branch 'main' into thiagohora/OPIK-218_remove_limitations_in_da…

…taset_items

.github/dependabot.yml

@@ -0,0 +1,15 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+ - package-ecosystem: "maven"
+   directory: "apps/opik-backend"
+   schedule:
+     interval: "weekly"      # How often to check for updates
+   ignore:
+     - dependency-name: "*"
+       update-types: ["version-update:semver-patch"] 
+   open-pull-requests-limit: 5

apps/opik-backend/pom.xml

@@ -25,7 +25,7 @@
         <jakarta.annotation.version>2.1.1</jakarta.annotation.version>
         <liquibase-clickhouse.version>0.7.2</liquibase-clickhouse.version>
         <clickhouse-java.version>0.6.5</clickhouse-java.version>
-        <org.mapstruct.version>1.5.5.Final</org.mapstruct.version>
+        <org.mapstruct.version>1.6.2</org.mapstruct.version>
         <testcontainers.version>1.20.0</testcontainers.version>
         <uuid.java.generator.version>5.1.0</uuid.java.generator.version>
         <wiremock.version>3.9.1</wiremock.version>
@@ -79,7 +79,7 @@
         <dependency>
             <groupId>software.amazon.jdbc</groupId>
             <artifactId>aws-advanced-jdbc-wrapper</artifactId>
-            <version>2.3.8</version>
+            <version>2.4.0</version>
         </dependency>
         <dependency>
             <groupId>io.dropwizard</groupId>
@@ -289,12 +289,12 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-jar-plugin</artifactId>
-                    <version>3.3.0</version>
+                    <version>3.4.2</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-compiler-plugin</artifactId>
-                    <version>3.12.1</version>
+                    <version>3.13.0</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
@@ -309,7 +309,7 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-javadoc-plugin</artifactId>
-                    <version>3.6.3</version>
+                    <version>3.10.1</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>

apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/DatasetsResourceTest.java

@@ -36,10 +36,8 @@
 import com.comet.opik.utils.JsonUtils;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.TextNode;
 import com.fasterxml.uuid.Generators;
 import com.fasterxml.uuid.impl.TimeBasedEpochGenerator;
-import com.google.common.collect.Maps;
 import com.redis.testcontainers.RedisContainer;
 import jakarta.ws.rs.client.Entity;
 import jakarta.ws.rs.core.GenericType;
@@ -78,7 +76,6 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -3117,18 +3114,19 @@ void getDatasetItemsByDatasetId__whenItemsWereUpdated__thenReturnCorrectItemsCou
 
     }
 
-    private static void assertPage(List<DatasetItem> items, List<DatasetItem> actualItems) {
+    private static void assertPage(List<DatasetItem> expectedItems, List<DatasetItem> actualItems) {
 
         List<String> ignoredFields = new ArrayList<>(Arrays.asList(IGNORED_FIELDS_DATA_ITEM));
         ignoredFields.add("data");
 
         assertThat(actualItems)
                 .usingRecursiveFieldByFieldElementComparatorIgnoringFields(ignoredFields.toArray(String[]::new))
-                .isEqualTo(items);
+                .isEqualTo(expectedItems);
 
+        assertThat(actualItems).hasSize(expectedItems.size());
         for (int i = 0; i < actualItems.size(); i++) {
             var actualDatasetItem = actualItems.get(i);
-            var expectedDatasetItem = items.get(i);
+            var expectedDatasetItem = expectedItems.get(i);
 
             Map<String, JsonNode> data = Optional.ofNullable(expectedDatasetItem.data())
                     .orElse(Map.of());

apps/opik-backend/src/test/java/com/comet/opik/podam/manufacturer/DatasetItemTypeManufacturer.java

@@ -39,16 +39,16 @@ public DatasetItem getType(DataProviderStrategy strategy, AttributeMetadata meta
                 : null;
 
         Map<String, JsonNode> data = IntStream.range(0, 5)
-                    .mapToObj(i -> {
-                        if (i % 2 == 0) {
-                            return Map.entry(RandomStringUtils.randomAlphanumeric(10),
-                                    TextNode.valueOf(RandomStringUtils.randomAlphanumeric(10)));
-                        }
-
+                .mapToObj(i -> {
+                    if (i % 2 == 0) {
                         return Map.entry(RandomStringUtils.randomAlphanumeric(10),
-                                strategy.getTypeValue(metadata, context, JsonNode.class));
-                    })
-                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                                TextNode.valueOf(RandomStringUtils.randomAlphanumeric(10)));
+                    }
+
+                    return Map.entry(RandomStringUtils.randomAlphanumeric(10),
+                            strategy.getTypeValue(metadata, context, JsonNode.class));
+                })
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 
         return DatasetItem.builder()
                 .source(source)

apps/opik-documentation/python-sdk-docs/requirements.txt

@@ -2,3 +2,6 @@ sphinx-autobuild
 sphinx
 furo
 sphinx-click
+
+# Dependency to document LlamaIndex integration
+llama_index

apps/opik-frontend/src/components/shared/TraceDetailsPanel/TraceTreeViewer/TraceTreeViewer.tsx

@@ -173,7 +173,7 @@ const TraceTreeViewer: React.FunctionComponent<TraceTreeViewerProps> = ({
   );
 
   const maxWidth = useMemo(() => {
-    const map: Record<string, number> = {};
+    const map: Record<string, number | undefined> = {};
     const list: ItemWidthObject[] = traceSpans.map((s) => ({
       id: s.id,
       name: s.name || "",
@@ -193,7 +193,13 @@ const TraceTreeViewer: React.FunctionComponent<TraceTreeViewerProps> = ({
 
     list.forEach((item) => {
       if (item.parentId) {
-        list[map[item.parentId]].children.push(item);
+        const listIndex = map[item.parentId];
+
+        if (listIndex !== undefined) {
+          list[listIndex].children.push(item);
+        } else {
+          console.warn(`Parent ${item.parentId} not found for ${item.id}`);
+        }
       } else {
         rootElement.children.push(item);
       }

build_and_run.sh

@@ -114,6 +114,7 @@ fi
 
 if $HELM_UPDATE; then
   helm repo add bitnami https://charts.bitnami.com/bitnami
+  helm repo add clickhouse-operator https://docs.altinity.com/clickhouse-operator
   helm dependency build
 fi
 helm upgrade --install opik -n opik --create-namespace -f values.yaml \

deployment/helm_chart/opik/templates/clickhouseinstallation.yaml

@@ -29,8 +29,11 @@ spec:
       {{- else }}
       {{ .Values.clickhouse.monitoring.username }}/password: {{ .Values.clickhouse.monitoring.password }}
       {{- end }}
+      {{ .Values.clickhouse.monitoring.username }}/networks/ip:
+        - '127.0.0.1/32'
+        - '0.0.0.0/0'
       {{- end }}
-    {{- if .Values.clickhouse.monitoring.enables }}
+    {{- if .Values.clickhouse.monitoring.enabled }}
     profiles:
       {{ .Values.clickhouse.monitoring.username }}/readonly: 1
     {{- end }}

sdks/python/src/opik/integrations/langchain/opik_tracer.py

@@ -1,4 +1,5 @@
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Literal, Set
+import logging
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Literal, Set, Union
 
 from langchain_core.tracers import BaseTracer
 
@@ -14,6 +15,8 @@
 
     from langchain_core.tracers.schemas import Run
 
+LOGGER = logging.getLogger(__name__)
+
 
 def _get_span_type(run: "Run") -> Literal["llm", "tool", "general"]:
     if run.run_type in ["llm", "tool"]:
@@ -23,21 +26,21 @@ def _get_span_type(run: "Run") -> Literal["llm", "tool", "general"]:
 
 
 class OpikTracer(BaseTracer):
-    """Opik Tracer."""
+    """Langchain Opik Tracer.
+
+    Args:
+        tags: List of tags to be applied to each trace logged by the tracer.
+        metadata: Additional metadata for each trace logged by the tracer.
+        project_name: The name of the project to log data.
+    """
 
     def __init__(
         self,
         tags: Optional[List[str]] = None,
         metadata: Optional[Dict[str, Any]] = None,
+        project_name: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
-        """
-        Initialize the Opik Tracer.
-
-        Args:
-            tags: List of tags to be applied to each trace logged by the tracer.
-            metadata: Additional metadata for each trace logged by the tracer.
-        """
         super().__init__(**kwargs)
         self._trace_default_metadata = metadata if metadata is not None else {}
         self._trace_default_tags = tags
@@ -52,7 +55,11 @@ def __init__(
 
         self._externally_created_traces_ids: Set[str] = set()
 
-        self._opik_client = opik_client.get_client_cached()
+        self._project_name = project_name
+
+        self._opik_client = opik_client.Opik(
+            _use_batching=True, project_name=project_name
+        )
 
     def _persist_run(self, run: "Run") -> None:
         run_dict: Dict[str, Any] = run.dict()
@@ -74,13 +81,17 @@ def _process_start_trace(self, run: "Run") -> None:
             self._track_root_run(run_dict)
         else:
             parent_span_data = self._span_data_map[run.parent_run_id]
+
+            project_name = self._get_project_name(parent_span_data)
+
             span_data = span.SpanData(
                 trace_id=parent_span_data.trace_id,
                 parent_span_id=parent_span_data.id,
                 input=run_dict["inputs"],
                 metadata=run_dict["extra"],
                 name=run.name,
                 type=_get_span_type(run),
+                project_name=project_name,
             )
 
             self._span_data_map[run.id] = span_data
@@ -90,6 +101,24 @@ def _process_start_trace(self, run: "Run") -> None:
                     run.parent_run_id
                 ]
 
+    def _get_project_name(
+        self, parent_data: Union[trace.TraceData, span.SpanData]
+    ) -> Optional[str]:
+        if parent_data.project_name != self._project_name:
+            # if the user has specified a project name -> print warning
+            if self._project_name is not None:
+                LOGGER.warning(
+                    "You are attempting to log data into a nested span under "
+                    f'the project name "{self._project_name}". '
+                    f'However, the project name "{parent_data.project_name}" '
+                    "from parent span will be used instead."
+                )
+            project_name = parent_data.project_name
+        else:
+            project_name = self._project_name
+
+        return project_name
+
     def _track_root_run(self, run_dict: Dict[str, Any]) -> None:
         run_metadata = run_dict["extra"].get("metadata", {})
         root_metadata = dict_utils.deepmerge(self._trace_default_metadata, run_metadata)
@@ -123,6 +152,7 @@ def _initialize_span_and_trace_from_scratch(
             input=run_dict["inputs"],
             metadata=root_metadata,
             tags=self._trace_default_tags,
+            project_name=self._project_name,
         )
 
         self._created_traces_data_map[run_dict["id"]] = trace_data
@@ -134,6 +164,7 @@ def _initialize_span_and_trace_from_scratch(
             input=run_dict["inputs"],
             metadata=root_metadata,
             tags=self._trace_default_tags,
+            project_name=self._project_name,
         )
 
         self._span_data_map[run_dict["id"]] = span_
@@ -144,13 +175,16 @@ def _attach_span_to_existing_span(
         current_span_data: span.SpanData,
         root_metadata: Dict[str, Any],
     ) -> None:
+        project_name = self._get_project_name(current_span_data)
+
         span_data = span.SpanData(
             trace_id=current_span_data.trace_id,
             parent_span_id=current_span_data.id,
             name=run_dict["name"],
             input=run_dict["inputs"],
             metadata=root_metadata,
             tags=self._trace_default_tags,
+            project_name=project_name,
         )
         self._span_data_map[run_dict["id"]] = span_data
         self._externally_created_traces_ids.add(span_data.trace_id)
@@ -161,13 +195,16 @@ def _attach_span_to_existing_trace(
         current_trace_data: trace.TraceData,
         root_metadata: Dict[str, Any],
     ) -> None:
+        project_name = self._get_project_name(current_trace_data)
+
         span_data = span.SpanData(
             trace_id=current_trace_data.id,
             parent_span_id=None,
             name=run_dict["name"],
             input=run_dict["inputs"],
             metadata=root_metadata,
             tags=self._trace_default_tags,
+            project_name=project_name,
         )
         self._span_data_map[run_dict["id"]] = span_data
         self._externally_created_traces_ids.add(current_trace_data.id)