Merge pull request #3443 from ingef/release

Merge Release

backend/pom.xml

@@ -303,7 +303,7 @@
         <dependency>
             <groupId>com.nimbusds</groupId>
             <artifactId>oauth2-oidc-sdk</artifactId>
-            <version>9.1</version>
+            <version>11.12</version>
         </dependency>
         <dependency>
             <groupId>org.keycloak</groupId>
@@ -389,5 +389,25 @@
             <artifactId>ngdbc</artifactId>
             <version>2.17.10</version>
         </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_dropwizard</artifactId>
+            <version>0.16.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>prometheus-metrics-simpleclient-bridge</artifactId>
+            <version>1.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>prometheus-metrics-exporter-servlet-jakarta</artifactId>
+            <version>1.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>prometheus-metrics-core</artifactId>
+            <version>1.2.1</version>
+        </dependency>
     </dependencies>
 </project>

backend/src/main/java/com/bakdata/conquery/Conquery.java

@@ -9,6 +9,7 @@
 import com.bakdata.conquery.commands.ShardNode;
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.MutableInjectableValues;
+import com.bakdata.conquery.metrics.prometheus.PrometheusBundle;
 import com.bakdata.conquery.mode.Manager;
 import com.bakdata.conquery.mode.ManagerProvider;
 import com.bakdata.conquery.mode.cluster.ClusterManagerProvider;
@@ -82,6 +83,8 @@ public void initialize(Bootstrap<?> bootstrap) {
 						bootstrap.getConfigurationSourceProvider(), StringSubstitutor.createInterpolator()));
 			}
 		});
+
+		bootstrap.addBundle(new PrometheusBundle());
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -56,6 +56,7 @@
 import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.error.ConqueryError;
+import com.bakdata.conquery.models.exceptions.ValidatorHelper;
 import com.bakdata.conquery.models.execution.ExecutionState;
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.i18n.I18n;
@@ -84,6 +85,7 @@
 import com.google.common.collect.MutableClassToInstanceMap;
 import jakarta.inject.Inject;
 import jakarta.servlet.http.HttpServletRequest;
+import jakarta.validation.Validator;
 import jakarta.ws.rs.BadRequestException;
 import jakarta.ws.rs.core.Response;
 import jakarta.ws.rs.core.UriBuilder;
@@ -102,6 +104,8 @@ public class QueryProcessor {
 	private MetaStorage storage;
 	@Inject
 	private ConqueryConfig config;
+	@Inject
+	private Validator validator;
 
 
 
@@ -339,6 +343,9 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri
 		final EntityPreviewForm form =
 				EntityPreviewForm.create(entity, idKind, dateRange, sources, previewConfig.getSelects(), previewConfig.getTimeStratifiedSelects(), datasetRegistry);
 
+		// Validate our own form because we provide it directly to the processor, which does not validate.
+		ValidatorHelper.failOnError(log, validator.validate(form));
+
 		// TODO make sure that subqueries are also system
 		// TODO do not persist system queries
 		final EntityPreviewExecution execution = (EntityPreviewExecution) postQuery(dataset, form, subject, true);

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java

@@ -99,6 +99,7 @@ public static CQConcept forSelect(Select select) {
 			table.setConnector(((Connector) select.getHolder()));
 
 			table.setSelects(List.of(select));
+			table.setConcept(cqConcept);
 		}
 		else {
 			cqConcept.setTables(((Concept<?>) select.getHolder())

backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java

@@ -4,8 +4,6 @@
 import java.util.Objects;
 import java.util.OptionalInt;
 
-import jakarta.validation.Validator;
-
 import com.bakdata.conquery.io.storage.xodus.stores.CachedStore;
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.config.StoreFactory;
@@ -19,6 +17,7 @@
 import com.bakdata.conquery.models.worker.WorkerToBucketsMap;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
+import jakarta.validation.Validator;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -34,7 +33,7 @@ public class NamespaceStorage extends NamespacedStorage {
 	protected CachedStore<String, Integer> entity2Bucket;
 
 	public NamespaceStorage(StoreFactory storageFactory, String pathName, Validator validator) {
-		super(storageFactory, pathName, validator);
+		super(storageFactory, pathName);
 	}
 
 

backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java

@@ -3,7 +3,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Set;
 
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.config.StoreFactory;
@@ -15,16 +14,13 @@
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
-import com.bakdata.conquery.models.exceptions.ValidatorHelper;
 import com.bakdata.conquery.models.identifiable.CentralRegistry;
 import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId;
 import com.bakdata.conquery.models.identifiable.ids.specific.ImportId;
 import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId;
 import com.bakdata.conquery.models.identifiable.ids.specific.TableId;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
-import jakarta.validation.ConstraintViolation;
-import jakarta.validation.Validator;
 import lombok.Getter;
 import lombok.SneakyThrows;
 import lombok.ToString;
@@ -48,19 +44,15 @@ public abstract class NamespacedStorage extends ConqueryStorage {
 	@Getter
 	private final StoreFactory storageFactory;
 
-	@Getter
-	private final Validator validator;
-
 	protected SingletonStore<Dataset> dataset;
 	protected IdentifiableStore<SecondaryIdDescription> secondaryIds;
 	protected IdentifiableStore<Table> tables;
 	protected IdentifiableStore<Import> imports;
 	protected IdentifiableStore<Concept<?>> concepts;
 
-	public NamespacedStorage(StoreFactory storageFactory, String pathName, Validator validator) {
+	public NamespacedStorage(StoreFactory storageFactory, String pathName) {
 		this.pathName = pathName;
 		this.storageFactory = storageFactory;
-		this.validator = validator;
 	}
 
 	public void openStores(ObjectMapper objectMapper) {
@@ -123,13 +115,6 @@ private void decorateConceptStore(IdentifiableStore<Concept<?>> store) {
 
 			concept.initElements();
 
-			if (log.isTraceEnabled()) {
-				// Validating concepts is quite slow, so we only validate when requested.
-				final Set<ConstraintViolation<Concept<?>>> violations = validator.validate(concept);
-
-				ValidatorHelper.failOnError(log, violations);
-			}
-
 			concept.getSelects().forEach(centralRegistry::register);
 			for (Connector connector : concept.getConnectors()) {
 				centralRegistry.register(connector);

backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java

@@ -26,7 +26,7 @@ public class WorkerStorage extends NamespacedStorage {
 	private IdentifiableStore<CBlock> cBlocks;
 
 	public WorkerStorage(StoreFactory storageFactory, Validator validator, String pathName) {
-		super(storageFactory, pathName, validator);
+		super(storageFactory, pathName);
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/metrics/prometheus/PrometheusBundle.java

@@ -0,0 +1,28 @@
+package com.bakdata.conquery.metrics.prometheus;
+
+import com.bakdata.conquery.models.config.ConqueryConfig;
+import io.dropwizard.core.ConfiguredBundle;
+import io.dropwizard.core.setup.Bootstrap;
+import io.dropwizard.core.setup.Environment;
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.dropwizard.DropwizardExports;
+import io.prometheus.metrics.exporter.servlet.jakarta.PrometheusMetricsServlet;
+import io.prometheus.metrics.simpleclient.bridge.SimpleclientCollector;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+public class PrometheusBundle implements ConfiguredBundle<ConqueryConfig> {
+
+	@Override
+	public void initialize(Bootstrap<?> bootstrap) {
+		// Init simple_client https://www.robustperception.io/exposing-dropwizard-metrics-to-prometheus/
+		CollectorRegistry.defaultRegistry.register(new DropwizardExports(bootstrap.getMetricRegistry()));
+		// Init client_java bridge: https://prometheus.github.io/client_java/migration/simpleclient/
+		SimpleclientCollector.builder().register();
+	}
+
+	@Override
+	public void run(ConqueryConfig configuration, Environment environment) throws Exception {
+
+		environment.getAdminContext().addServlet(new ServletHolder(new PrometheusMetricsServlet()), "/metrics-prometheus");
+	}
+}

backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java

@@ -10,39 +10,30 @@
 import com.bakdata.conquery.mode.ManagerProvider;
 import com.bakdata.conquery.mode.NamespaceHandler;
 import com.bakdata.conquery.models.config.ConqueryConfig;
-import com.bakdata.conquery.models.config.SqlConnectorConfig;
 import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.LocalNamespace;
 import com.bakdata.conquery.models.worker.ShardNodeInformation;
-import com.bakdata.conquery.sql.DslContextFactory;
-import com.bakdata.conquery.sql.SqlContext;
-import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect;
-import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect;
-import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
-import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory;
-import com.bakdata.conquery.sql.execution.SqlExecutionService;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory;
 import io.dropwizard.core.setup.Environment;
-import org.jooq.DSLContext;
 
 public class LocalManagerProvider implements ManagerProvider {
 
 	private static final Supplier<Collection<ShardNodeInformation>> EMPTY_NODE_PROVIDER = Collections::emptyList;
 
-	public DelegateManager<LocalNamespace> provideManager(ConqueryConfig config, Environment environment) {
+	private final SqlDialectFactory dialectFactory;
 
-		InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator());
+	public LocalManagerProvider() {
+		this.dialectFactory = new SqlDialectFactory();
+	}
 
-		SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig();
-		DSLContext dslContext = DslContextFactory.create(sqlConnectorConfig);
-		SqlDialect sqlDialect = createSqlDialect(sqlConnectorConfig, dslContext);
-		SqlContext sqlContext = new SqlContext(sqlConnectorConfig, sqlDialect);
+	public LocalManagerProvider(SqlDialectFactory dialectFactory) {
+		this.dialectFactory = dialectFactory;
+	}
 
-		SqlExecutionService sqlExecutionService = new SqlExecutionService(
-				sqlDialect.getDSLContext(),
-				ResultSetProcessorFactory.create(sqlDialect)
-		);
+	public DelegateManager<LocalNamespace> provideManager(ConqueryConfig config, Environment environment) {
 
-		NamespaceHandler<LocalNamespace> namespaceHandler = new LocalNamespaceHandler(config, creator, sqlContext, sqlExecutionService);
+		InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator());
+		NamespaceHandler<LocalNamespace> namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory);
 		DatasetRegistry<LocalNamespace> datasetRegistry = ManagerProvider.createLocalDatasetRegistry(namespaceHandler, config, creator);
 		creator.init(datasetRegistry);
 
@@ -59,10 +50,4 @@ public DelegateManager<LocalNamespace> provideManager(ConqueryConfig config, Env
 		);
 	}
 
-	protected SqlDialect createSqlDialect(SqlConnectorConfig sqlConnectorConfig, DSLContext dslContext) {
-		return switch (sqlConnectorConfig.getDialect()) {
-			case POSTGRESQL -> new PostgreSqlDialect(dslContext);
-			case HANA -> new HanaSqlDialect(dslContext);
-		};
-	}
 }

backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java

@@ -6,33 +6,57 @@
 import com.bakdata.conquery.mode.NamespaceHandler;
 import com.bakdata.conquery.mode.NamespaceSetupData;
 import com.bakdata.conquery.models.config.ConqueryConfig;
+import com.bakdata.conquery.models.config.DatabaseConfig;
+import com.bakdata.conquery.models.config.SqlConnectorConfig;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
 import com.bakdata.conquery.models.index.IndexService;
 import com.bakdata.conquery.models.query.ExecutionManager;
 import com.bakdata.conquery.models.worker.LocalNamespace;
-import com.bakdata.conquery.sql.SqlContext;
+import com.bakdata.conquery.sql.DSLContextWrapper;
+import com.bakdata.conquery.sql.DslContextFactory;
 import com.bakdata.conquery.sql.conquery.SqlExecutionManager;
+import com.bakdata.conquery.sql.conversion.SqlConverter;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory;
+import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory;
+import com.bakdata.conquery.sql.execution.SqlExecutionResult;
 import com.bakdata.conquery.sql.execution.SqlExecutionService;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.jooq.DSLContext;
 
 @RequiredArgsConstructor
+@Slf4j
 public class LocalNamespaceHandler implements NamespaceHandler<LocalNamespace> {
 
 	private final ConqueryConfig config;
 	private final InternalObjectMapperCreator mapperCreator;
-	private final SqlContext sqlContext;
-	private final SqlExecutionService sqlExecutionService;
+	private final SqlDialectFactory dialectFactory;
 
 	@Override
 	public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, IndexService indexService) {
+
 		NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator, indexService);
-		ExecutionManager executionManager = new SqlExecutionManager(sqlContext, sqlExecutionService, metaStorage);
+
+		SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig();
+		DatabaseConfig databaseConfig = sqlConnectorConfig.getDatabaseConfig(namespaceStorage.getDataset());
+
+		DSLContextWrapper dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig);
+		DSLContext dslContext = dslContextWrapper.getDslContext();
+		SqlDialect sqlDialect = dialectFactory.createSqlDialect(databaseConfig.getDialect());
+
+		SqlConverter sqlConverter = new SqlConverter(sqlDialect, dslContext, databaseConfig);
+		SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(sqlDialect));
+		ExecutionManager<SqlExecutionResult> executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage);
+		SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService);
+
 		return new LocalNamespace(
 				namespaceData.getPreprocessMapper(),
 				namespaceData.getCommunicationMapper(),
 				namespaceStorage,
 				executionManager,
-				sqlExecutionService,
+				dslContextWrapper,
+				sqlStorageHandler,
 				namespaceData.getJobManager(),
 				namespaceData.getFilterSearch(),
 				namespaceData.getIndexService(),

backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java

@@ -0,0 +1,23 @@
+package com.bakdata.conquery.models.config;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class DatabaseConfig {
+
+	private static final String DEFAULT_PRIMARY_COLUMN = "pid";
+
+	private Dialect dialect;
+
+	private String databaseUsername;
+
+	private String databasePassword;
+
+	private String jdbcConnectionUrl;
+
+	@Builder.Default
+	private String primaryColumn = DEFAULT_PRIMARY_COLUMN;
+
+}