From 0fd004c719770af5d83b996ac335172729ade2ee Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Thu, 12 Sep 2024 09:52:36 +0200 Subject: [PATCH 01/20] [NU-1806] Add activity parameters --- .../api/component/NodesDeploymentData.scala | 27 +++++----- .../engine/api/process/Source.scala | 4 ++ .../ui/api/ActivityInfoResources.scala | 39 ++++++++++++++ .../ui/api/ManagementResources.scala | 4 +- .../newactivity/ScenarioActivityService.scala | 54 +++++++++++++++++++ .../server/AkkaHttpBasedRouteProvider.scala | 10 +++- .../test/base/it/NuResourcesTest.scala | 9 +++- .../ui/api/TestInfoResourcesSpec.scala | 37 +++++++++++++ .../flink/table/source/TableSource.scala | 2 +- .../kafka/source/flink/FlinkKafkaSource.scala | 21 +++++++- .../activity/ActivityInfoProvider.scala | 10 ++++ .../ModelDataActivityInfoProvider.scala | 51 ++++++++++++++++++ 12 files changed, 249 insertions(+), 19 deletions(-) create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ScenarioActivityService.scala create mode 100644 scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala create mode 100644 scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala index f032a97c297..8893f78eaa6 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala @@ -1,5 +1,6 @@ package pl.touk.nussknacker.engine.api.component +import io.circe.generic.JsonCodec import io.circe.generic.extras.semiauto.{deriveUnwrappedDecoder, deriveUnwrappedEncoder} import io.circe.{Decoder, Encoder} import pl.touk.nussknacker.engine.api.NodeId @@ -19,18 +20,20 @@ object NodesDeploymentData { } -sealed trait NodeDeploymentData +@JsonCodec sealed trait NodeDeploymentData final case class SqlFilteringExpression(sqlExpression: String) extends NodeDeploymentData -object NodeDeploymentData { - - implicit val nodeDeploymentDataEncoder: Encoder[NodeDeploymentData] = - deriveUnwrappedEncoder[SqlFilteringExpression].contramap { case sqlExpression: SqlFilteringExpression => - sqlExpression - } - - implicit val nodeDeploymentDataDecoder: Decoder[NodeDeploymentData] = - deriveUnwrappedDecoder[SqlFilteringExpression].map(identity) - -} +final case class KafkaSourceDeploymentData(offset: String) extends NodeDeploymentData + +//object NodeDeploymentData { +// +// implicit val nodeDeploymentDataEncoder: Encoder[NodeDeploymentData] = +// deriveUnwrappedEncoder[SqlFilteringExpression].contramap { case sqlExpression: SqlFilteringExpression => +// sqlExpression +// } +// +// implicit val nodeDeploymentDataDecoder: Decoder[NodeDeploymentData] = +// deriveUnwrappedDecoder[SqlFilteringExpression].map(identity) +// +//} diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala index dbfb9904573..ed783d853c0 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala @@ -49,6 +49,10 @@ trait TestWithParametersSupport[+T] { self: Source => def parametersToTestData(params: Map[ParameterName, AnyRef]): T } +trait WithActivityParameters { self: Source => + def activityParametersDefinition: Map[String, List[Parameter]] +} + /** * [[pl.touk.nussknacker.engine.api.process.SourceFactory]] has to have method annotated with [[pl.touk.nussknacker.engine.api.MethodToInvoke]] * that returns [[pl.touk.nussknacker.engine.api.process.Source]] diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala new file mode 100644 index 00000000000..7aaaa04b4bb --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala @@ -0,0 +1,39 @@ +package pl.touk.nussknacker.ui.api + +import akka.http.scaladsl.server.{Directives, Route} +import com.typesafe.scalalogging.LazyLogging +import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport +import pl.touk.nussknacker.engine.api.graph.ScenarioGraph +import pl.touk.nussknacker.ui.process.ProcessService +import pl.touk.nussknacker.ui.process.newactivity.ScenarioActivityService +import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataProvider +import pl.touk.nussknacker.ui.security.api.LoggedUser + +import scala.concurrent.ExecutionContext + +class ActivityInfoResources( + protected val processService: ProcessService, + scenarioActivityServices: ProcessingTypeDataProvider[ScenarioActivityService, _] +)(implicit val ec: ExecutionContext) + extends Directives + with FailFastCirceSupport + with RouteWithUser + with ProcessDirectives + with LazyLogging { + + def securedRoute(implicit user: LoggedUser): Route = { + pathPrefix("activityInfo" / ProcessNameSegment) { processName => + (post & processDetailsForName(processName)) { processDetails => + entity(as[ScenarioGraph]) { scenarioGraph => + val scenarioTestService = scenarioActivityServices.forProcessingTypeUnsafe(processDetails.processingType) + path("activityParameters") { + complete { + scenarioTestService.getActivityParameters(scenarioGraph, processName, processDetails.isFragment) + } + } + } + } + } + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala index c13b011c05c..1f9c3f5065a 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala @@ -134,7 +134,7 @@ class ManagementResources( RunDeploymentCommand( // adminProcessManagement endpoint is not used by the designer client. It is a part of API for tooling purpose commonData = CommonCommandData(processIdWithName, comment.map(ApiCallComment(_)), user), - nodesDeploymentData = NodesDeploymentData.empty, + nodesDeploymentData = NodesDeploymentData.empty, // TODO: here goes map of parameters defined by stateRestoringStrategy = StateRestoringStrategy.RestoreStateFromCustomSavepoint(savepointPath) ) ) @@ -156,7 +156,7 @@ class ManagementResources( .processCommand( RunDeploymentCommand( commonData = CommonCommandData(processIdWithName, comment.map(UserComment), user), - nodesDeploymentData = NodesDeploymentData.empty, + nodesDeploymentData = NodesDeploymentData.empty, // TODO: tu dostarczam offset stateRestoringStrategy = StateRestoringStrategy.RestoreStateFromReplacedJobSavepoint ) ) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ScenarioActivityService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ScenarioActivityService.scala new file mode 100644 index 00000000000..c0ec6d6aeef --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ScenarioActivityService.scala @@ -0,0 +1,54 @@ +package pl.touk.nussknacker.ui.process.newactivity + +import pl.touk.nussknacker.engine.api.definition.StringParameterEditor +import pl.touk.nussknacker.engine.api.graph.ScenarioGraph +import pl.touk.nussknacker.engine.api.process.ProcessName +import pl.touk.nussknacker.engine.api.typed.CanBeSubclassDeterminer +import pl.touk.nussknacker.engine.api.typed.typing.Typed +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.definition.activity.ActivityInfoProvider +import pl.touk.nussknacker.restmodel.definition.UISourceParameters +import pl.touk.nussknacker.ui.definition.DefinitionsService +import pl.touk.nussknacker.ui.security.api.LoggedUser +import pl.touk.nussknacker.ui.uiresolving.UIProcessResolver + +class ScenarioActivityService(activityInfoProvider: ActivityInfoProvider, processResolver: UIProcessResolver) { + + def getActivityParameters(scenarioGraph: ScenarioGraph, processName: ProcessName, isFragment: Boolean)( + implicit user: LoggedUser + ): Map[String, List[UISourceParameters]] = { + val canonical = toCanonicalProcess(scenarioGraph, processName, isFragment) + activityInfoProvider + .getActivityParameters(canonical) + .map { case (activityName, nodeParamsMap) => + activityName -> nodeParamsMap + .map { case (nodeId, params) => + UISourceParameters(nodeId, params.map(DefinitionsService.createUIParameter)) + } + .map(assignUserFriendlyEditor) + .toList + } + } + + // copied from ScenarioTestService + private def toCanonicalProcess( + scenarioGraph: ScenarioGraph, + processName: ProcessName, + isFragment: Boolean + )(implicit user: LoggedUser): CanonicalProcess = { + processResolver.validateAndResolve(scenarioGraph, processName, isFragment) + } + + // copied from ScenarioTestService + private def assignUserFriendlyEditor(uiSourceParameter: UISourceParameters): UISourceParameters = { + val adaptedParameters = uiSourceParameter.parameters.map { uiParameter => + if (CanBeSubclassDeterminer.canBeSubclassOf(uiParameter.typ, Typed.apply(classOf[String])).isValid) { + uiParameter.copy(editor = StringParameterEditor) + } else { + uiParameter + } + } + uiSourceParameter.copy(parameters = adaptedParameters) + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index 94087cbd276..73e05620239 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -12,6 +12,7 @@ import net.ceedubs.ficus.readers.ArbitraryTypeReader.arbitraryTypeValueReader import pl.touk.nussknacker.engine.api.component._ import pl.touk.nussknacker.engine.api.process.ProcessingType import pl.touk.nussknacker.engine.compile.ProcessValidator +import pl.touk.nussknacker.engine.definition.activity.ModelDataActivityInfoProvider import pl.touk.nussknacker.engine.definition.test.ModelDataTestInfoProvider import pl.touk.nussknacker.engine.dict.ProcessDictSubstitutor import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader @@ -55,7 +56,7 @@ import pl.touk.nussknacker.ui.process.deployment.{ import pl.touk.nussknacker.ui.process.fragment.{DefaultFragmentRepository, FragmentResolver} import pl.touk.nussknacker.ui.process.label.ScenarioLabelsService import pl.touk.nussknacker.ui.process.migrate.{HttpRemoteEnvironment, ProcessModelMigrator, TestModelMigrations} -import pl.touk.nussknacker.ui.process.newactivity.ActivityService +import pl.touk.nussknacker.ui.process.newactivity.{ActivityService, ScenarioActivityService} import pl.touk.nussknacker.ui.process.newdeployment.synchronize.{ DeploymentsStatusesSynchronizationConfig, DeploymentsStatusesSynchronizationScheduler, @@ -209,6 +210,12 @@ class AkkaHttpBasedRouteProvider( new ScenarioTestExecutorServiceImpl(scenarioResolver, deploymentManager) ) } + val scenarioActivityService = scenarioTestServiceDeps.mapValues { case (_, processResolver, _, modelData, _) => + new ScenarioActivityService( + new ModelDataActivityInfoProvider(modelData), + processResolver + ) + } val processValidator = scenarioTestServiceDeps.mapValues(_._1) val processResolver = scenarioTestServiceDeps.mapValues(_._2) @@ -479,6 +486,7 @@ class AkkaHttpBasedRouteProvider( } ), new TestInfoResources(processAuthorizer, processService, scenarioTestService), + new ActivityInfoResources(processService, scenarioActivityService), new StatusResources(stateDefinitionService), ) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala index 7b67706cb47..a8908d842d0 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala @@ -57,7 +57,8 @@ import pl.touk.nussknacker.ui.processreport.ProcessCounter import pl.touk.nussknacker.ui.security.api.{LoggedUser, RealLoggedUser} import pl.touk.nussknacker.ui.util.{MultipartUtils, NuPathMatchers} import slick.dbio.DBIOAction - +import pl.touk.nussknacker.engine.definition.activity.ModelDataActivityInfoProvider +import pl.touk.nussknacker.ui.process.newactivity.ScenarioActivityService import java.net.URI import scala.concurrent.{ExecutionContext, Future} @@ -215,6 +216,12 @@ trait NuResourcesTest ) ) + protected def createScenarioActivityService: ScenarioActivityService = + new ScenarioActivityService( + new ModelDataActivityInfoProvider(modelData), + processResolver + ) + protected def deployRoute() = new ManagementResources( processAuthorizer = processAuthorizer, diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestInfoResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestInfoResourcesSpec.scala index de8b45b9368..41437a0c7a7 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestInfoResourcesSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestInfoResourcesSpec.scala @@ -165,3 +165,40 @@ class TestInfoResourcesSpec } } + +class ActivityInfoResourcesSpec + extends AnyFunSuite + with ScalatestRouteTest + with Matchers + with FailFastCirceSupport + with NuResourcesTest + with PatientScalaFutures + with EitherValuesDetailedMessage { + + private val scenarioGraph: ScenarioGraph = ProcessTestData.sampleScenarioGraph + private val testPermissionAll = List(Permission.Deploy, Permission.Read, Permission.Write) + + private def route() = new ActivityInfoResources( + processService, + mapProcessingTypeDataProvider( + Streaming.stringify -> createScenarioActivityService + ) + ) + + test("get activity parameters") { + saveProcess(scenarioGraph) { + Post( + s"/activityInfo/${ProcessTestData.sampleProcessName}/activityParameters", + scenarioGraph.toJsonRequestEntity() + ) ~> withPermissions( + route(), + testPermissionAll: _* + ) ~> check { + status shouldEqual StatusCodes.OK + val content = entityAs[Json].noSpaces + content shouldBe """{}""" + } + } + } + +} diff --git a/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala b/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala index 5b826f57979..23c7dc55588 100644 --- a/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala +++ b/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala @@ -52,7 +52,7 @@ class TableSource( val selectQuery = tableEnv.from(tableDefinition.tableId.toString) val finalQuery = flinkNodeContext.nodeDeploymentData - .map { case SqlFilteringExpression(sqlExpression) => + .collect { case SqlFilteringExpression(sqlExpression) => tableEnv.executeSql( s"CREATE TEMPORARY VIEW $filteringInternalViewName AS SELECT * FROM ${tableDefinition.tableId} WHERE $sqlExpression" ) diff --git a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala index 574f371dee3..c95f4188017 100644 --- a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala +++ b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala @@ -11,12 +11,20 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaConsumerBase} import org.apache.kafka.clients.consumer.ConsumerRecord import pl.touk.nussknacker.engine.api.NodeId +import pl.touk.nussknacker.engine.api.component.KafkaSourceDeploymentData import pl.touk.nussknacker.engine.api.definition.Parameter +import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.namespaces.NamingStrategy import pl.touk.nussknacker.engine.api.parameter.ParameterName -import pl.touk.nussknacker.engine.api.process.{ContextInitializer, TestWithParametersSupport, TopicName} +import pl.touk.nussknacker.engine.api.process.{ + ContextInitializer, + TestWithParametersSupport, + TopicName, + WithActivityParameters +} import pl.touk.nussknacker.engine.api.runtimecontext.{ContextIdGenerator, EngineRuntimeContext} import pl.touk.nussknacker.engine.api.test.{TestRecord, TestRecordParser} +import pl.touk.nussknacker.engine.api.typed.typing.Typed import pl.touk.nussknacker.engine.flink.api.exception.ExceptionHandler import pl.touk.nussknacker.engine.flink.api.process.{ FlinkCustomNodeContext, @@ -55,7 +63,8 @@ class FlinkKafkaSource[T]( with Serializable with FlinkSourceTestSupport[T] with RecordFormatterBaseTestDataGenerator - with TestWithParametersSupport[T] { + with TestWithParametersSupport[T] + with WithActivityParameters { @silent("deprecated") override def sourceStream( @@ -73,11 +82,19 @@ class FlinkKafkaSource[T]( protected lazy val topics: NonEmptyList[TopicName.ForSource] = preparedTopics.map(_.prepared) + override def activityParametersDefinition: Map[String, List[Parameter]] = Map( + ScenarioActionName.Deploy.value -> List( + Parameter(ParameterName("offset"), Typed.apply[String]) + ) + ) + @silent("deprecated") protected def flinkSourceFunction( consumerGroupId: String, flinkNodeContext: FlinkCustomNodeContext ): SourceFunction[T] = { + // TODO: handle deployment parameters -> offset + val deploymentDataOpt = flinkNodeContext.nodeDeploymentData.collect { case d: KafkaSourceDeploymentData => d } topics.toList.foreach(KafkaUtils.setToLatestOffsetIfNeeded(kafkaConfig, _, consumerGroupId)) createFlinkSource(consumerGroupId, flinkNodeContext) } diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala new file mode 100644 index 00000000000..0c679dad44a --- /dev/null +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala @@ -0,0 +1,10 @@ +package pl.touk.nussknacker.engine.definition.activity + +import pl.touk.nussknacker.engine.api.definition.Parameter +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess + +trait ActivityInfoProvider { + + def getActivityParameters(scenario: CanonicalProcess): Map[String, Map[String, List[Parameter]]] + +} diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala new file mode 100644 index 00000000000..f74d02717d3 --- /dev/null +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala @@ -0,0 +1,51 @@ +package pl.touk.nussknacker.engine.definition.activity + +import pl.touk.nussknacker.engine.ModelData +import pl.touk.nussknacker.engine.api.definition.Parameter +import pl.touk.nussknacker.engine.api.process.{ComponentUseCase, WithActivityParameters} +import pl.touk.nussknacker.engine.api.{MetaData, NodeId, process} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.compile.ExpressionCompiler +import pl.touk.nussknacker.engine.compile.nodecompilation.{LazyParameterCreationStrategy, NodeCompiler} +import pl.touk.nussknacker.engine.definition.fragment.FragmentParametersDefinitionExtractor +import pl.touk.nussknacker.engine.graph.node.{SourceNodeData, asSource} +import pl.touk.nussknacker.engine.resultcollector.ProductionServiceInvocationCollector +import shapeless.syntax.typeable._ + +class ModelDataActivityInfoProvider(modelData: ModelData) extends ActivityInfoProvider { + + private lazy val expressionCompiler = ExpressionCompiler.withoutOptimization(modelData).withLabelsDictTyper + + private lazy val nodeCompiler = new NodeCompiler( + modelData.modelDefinition, + new FragmentParametersDefinitionExtractor(modelData.modelClassLoader.classLoader), + expressionCompiler, + modelData.modelClassLoader.classLoader, + Seq.empty, + ProductionServiceInvocationCollector, + ComponentUseCase.TestDataGeneration, + nonServicesLazyParamStrategy = LazyParameterCreationStrategy.default + ) + + private def prepareSourceObj( + source: SourceNodeData + )(implicit metaData: MetaData, nodeId: NodeId): Option[process.Source] = { + nodeCompiler.compileSource(source).compiledObject.toOption + } + + override def getActivityParameters(scenario: CanonicalProcess): Map[String, Map[String, List[Parameter]]] = { + modelData.withThisAsContextClassLoader { + val asdf = scenario.collectAllNodes.flatMap(asSource) + val compiledSources = for { + source <- scenario.collectAllNodes.flatMap(asSource) + sourceObj <- prepareSourceObj(source)(scenario.metaData, NodeId(source.id)) + } yield sourceObj + val stefan = compiledSources + .flatMap(_.cast[WithActivityParameters]) + .map(_.activityParametersDefinition) + + Map.empty + } + } + +} From 6719ebe90c5b4873b8ad7007bde888b50503b947 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Thu, 12 Sep 2024 10:14:49 +0200 Subject: [PATCH 02/20] [NU-1806] Better place for ManagementApiEndpoints --- .../ui/api/ManagementApiHttpService.scala | 12 ++++++++---- .../{ => description}/ManagementApiEndpoints.scala | 14 +++++++------- .../ui/server/AkkaHttpBasedRouteProvider.scala | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) rename designer/server/src/main/scala/pl/touk/nussknacker/ui/api/{ => description}/ManagementApiEndpoints.scala (87%) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiHttpService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiHttpService.scala index 37964bc3a96..d6ab6658147 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiHttpService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiHttpService.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.ui.services +package pl.touk.nussknacker.ui.api import cats.data.{EitherT, Validated} import cats.syntax.all._ @@ -9,9 +9,13 @@ import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, Pro import pl.touk.nussknacker.engine.deployment.CustomActionDefinition import pl.touk.nussknacker.restmodel.CustomActionRequest import pl.touk.nussknacker.restmodel.validation.PrettyValidationErrors -import pl.touk.nussknacker.ui.api.ManagementApiEndpoints.ManagementApiError -import pl.touk.nussknacker.ui.api.ManagementApiEndpoints.ManagementApiError.{NoActionDefinition, NoScenario} -import pl.touk.nussknacker.ui.api.{BaseHttpService, CustomActionValidationDto, ManagementApiEndpoints} +import pl.touk.nussknacker.ui.api.description.ManagementApiEndpoints +import pl.touk.nussknacker.ui.api.description.ManagementApiEndpoints.Dtos.{ + CustomActionValidationDto, + ManagementApiError, + NoActionDefinition, + NoScenario +} import pl.touk.nussknacker.ui.process.ProcessService import pl.touk.nussknacker.ui.process.deployment.DeploymentManagerDispatcher import pl.touk.nussknacker.ui.security.api.{AuthManager, LoggedUser} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/ManagementApiEndpoints.scala similarity index 87% rename from designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiEndpoints.scala rename to designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/ManagementApiEndpoints.scala index a64a65b4812..c168561e1de 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/ManagementApiEndpoints.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.ui.api +package pl.touk.nussknacker.ui.api.description import derevo.circe.encoder import derevo.derive @@ -9,8 +9,7 @@ import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions.SecuredEndpoint import pl.touk.nussknacker.restmodel.validation.ValidationResults.NodeValidationError import pl.touk.nussknacker.restmodel.{BaseEndpointDefinitions, CustomActionRequest} import pl.touk.nussknacker.security.AuthCredentials -import pl.touk.nussknacker.ui.api.ManagementApiEndpoints.ManagementApiError -import pl.touk.nussknacker.ui.api.ManagementApiEndpoints.ManagementApiError.{NoActionDefinition, NoScenario} +import pl.touk.nussknacker.ui.api.description.ManagementApiEndpoints.Dtos.{CustomActionValidationDto, ManagementApiError, NoActionDefinition, NoScenario} import pl.touk.nussknacker.ui.api.TapirCodecs.ScenarioNameCodec._ import pl.touk.nussknacker.ui.api.TapirCodecs.ClassCodec._ import pl.touk.nussknacker.ui.api.BaseHttpService.CustomAuthorizationError @@ -59,14 +58,15 @@ class ManagementApiEndpoints(auth: EndpointInput[AuthCredentials]) extends BaseE } -@derive(schema, encoder) -final case class CustomActionValidationDto(validationErrors: List[NodeValidationError], validationPerformed: Boolean) object ManagementApiEndpoints { - sealed trait ManagementApiError + object Dtos { - object ManagementApiError { + @derive(schema, encoder) + final case class CustomActionValidationDto(validationErrors: List[NodeValidationError], validationPerformed: Boolean) + + sealed trait ManagementApiError final case object NoPermission extends ManagementApiError with CustomAuthorizationError final case class NoScenario(scenarioName: ProcessName) extends ManagementApiError final case class NoActionDefinition(scenarioName: ProcessName, actionName: ScenarioActionName) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index 73e05620239..37e125a59d2 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -72,7 +72,7 @@ import pl.touk.nussknacker.ui.process.test.{PreliminaryScenarioTestDataSerDe, Sc import pl.touk.nussknacker.ui.process.version.{ScenarioGraphVersionRepository, ScenarioGraphVersionService} import pl.touk.nussknacker.ui.processreport.ProcessCounter import pl.touk.nussknacker.ui.security.api.{AuthManager, AuthenticationResources} -import pl.touk.nussknacker.ui.services.{ManagementApiHttpService, NuDesignerExposedApiHttpService} +import pl.touk.nussknacker.ui.services.NuDesignerExposedApiHttpService import pl.touk.nussknacker.ui.statistics.repository.FingerprintRepositoryImpl import pl.touk.nussknacker.ui.statistics.{ FingerprintService, From 24172650950ad896bc18aa6c457573090c59d638 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Thu, 12 Sep 2024 11:57:22 +0200 Subject: [PATCH 03/20] cleanup --- .../ui/api/ManagementResources.scala | 6 ++- .../ui/api/ActivityInfoResourcesSpec.scala | 53 +++++++++++++++++++ .../ui/api/TestInfoResourcesSpec.scala | 37 ------------- 3 files changed, 57 insertions(+), 39 deletions(-) create mode 100644 designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala index 1f9c3f5065a..879b1d865d8 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala @@ -134,7 +134,8 @@ class ManagementResources( RunDeploymentCommand( // adminProcessManagement endpoint is not used by the designer client. It is a part of API for tooling purpose commonData = CommonCommandData(processIdWithName, comment.map(ApiCallComment(_)), user), - nodesDeploymentData = NodesDeploymentData.empty, // TODO: here goes map of parameters defined by + nodesDeploymentData = + NodesDeploymentData.empty, // TODO: here goes map of parameters defined by activityParameters stateRestoringStrategy = StateRestoringStrategy.RestoreStateFromCustomSavepoint(savepointPath) ) ) @@ -156,7 +157,8 @@ class ManagementResources( .processCommand( RunDeploymentCommand( commonData = CommonCommandData(processIdWithName, comment.map(UserComment), user), - nodesDeploymentData = NodesDeploymentData.empty, // TODO: tu dostarczam offset + nodesDeploymentData = + NodesDeploymentData.empty, // TODO: here goes map of parameters defined by activityParameters stateRestoringStrategy = StateRestoringStrategy.RestoreStateFromReplacedJobSavepoint ) ) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala new file mode 100644 index 00000000000..76694f952be --- /dev/null +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala @@ -0,0 +1,53 @@ +package pl.touk.nussknacker.ui.api + +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.testkit.ScalatestRouteTest +import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport +import io.circe.Json +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import pl.touk.nussknacker.engine.api.graph.ScenarioGraph +import pl.touk.nussknacker.security.Permission +import pl.touk.nussknacker.test.base.it.NuResourcesTest +import pl.touk.nussknacker.test.config.WithSimplifiedDesignerConfig.TestProcessingType.Streaming +import pl.touk.nussknacker.test.utils.domain.ProcessTestData +import pl.touk.nussknacker.test.utils.domain.TestFactory.{mapProcessingTypeDataProvider, withPermissions} +import pl.touk.nussknacker.test.{EitherValuesDetailedMessage, PatientScalaFutures} +import pl.touk.nussknacker.test.utils.scalas.AkkaHttpExtensions.toRequestEntity + +class ActivityInfoResourcesSpec + extends AnyFunSuite + with ScalatestRouteTest + with Matchers + with FailFastCirceSupport + with NuResourcesTest + with PatientScalaFutures + with EitherValuesDetailedMessage { + + private val scenarioGraph: ScenarioGraph = ProcessTestData.sampleScenarioGraph + private val testPermissionAll = List(Permission.Deploy, Permission.Read, Permission.Write) + + private def route() = new ActivityInfoResources( + processService, + mapProcessingTypeDataProvider( + Streaming.stringify -> createScenarioActivityService + ) + ) + + test("get activity parameters") { + saveProcess(scenarioGraph) { + Post( + s"/activityInfo/${ProcessTestData.sampleProcessName}/activityParameters", + scenarioGraph.toJsonRequestEntity() + ) ~> withPermissions( + route(), + testPermissionAll: _* + ) ~> check { + status shouldEqual StatusCodes.OK + val content = entityAs[Json].noSpaces + content shouldBe """{}""" + } + } + } + +} diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestInfoResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestInfoResourcesSpec.scala index 41437a0c7a7..de8b45b9368 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestInfoResourcesSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestInfoResourcesSpec.scala @@ -165,40 +165,3 @@ class TestInfoResourcesSpec } } - -class ActivityInfoResourcesSpec - extends AnyFunSuite - with ScalatestRouteTest - with Matchers - with FailFastCirceSupport - with NuResourcesTest - with PatientScalaFutures - with EitherValuesDetailedMessage { - - private val scenarioGraph: ScenarioGraph = ProcessTestData.sampleScenarioGraph - private val testPermissionAll = List(Permission.Deploy, Permission.Read, Permission.Write) - - private def route() = new ActivityInfoResources( - processService, - mapProcessingTypeDataProvider( - Streaming.stringify -> createScenarioActivityService - ) - ) - - test("get activity parameters") { - saveProcess(scenarioGraph) { - Post( - s"/activityInfo/${ProcessTestData.sampleProcessName}/activityParameters", - scenarioGraph.toJsonRequestEntity() - ) ~> withPermissions( - route(), - testPermissionAll: _* - ) ~> check { - status shouldEqual StatusCodes.OK - val content = entityAs[Json].noSpaces - content shouldBe """{}""" - } - } - } - -} From d0341cd3442c9969949c6aaa819d97f07acb9440 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Fri, 13 Sep 2024 11:47:13 +0200 Subject: [PATCH 04/20] [NU-1806] Test with BoundedSourceWithOffset --- .../api/component/NodesDeploymentData.scala | 37 ++-- .../description/DeploymentApiEndpoints.scala | 2 +- .../ui/api/ActivityInfoResourcesSpec.scala | 127 +++++++----- .../ComponentApiHttpServiceBusinessSpec.scala | 1 + .../ComponentApiHttpServiceSecuritySpec.scala | 4 + ...DeploymentApiHttpServiceBusinessSpec.scala | 2 +- ...tApiHttpServiceDeploymentCommentSpec.scala | 6 +- docs-internal/api/nu-designer-openapi.yaml | 189 ++++++++++-------- .../flink/util/source/CollectionSource.scala | 10 +- .../kafka/source/flink/FlinkKafkaSource.scala | 6 +- .../sample/DevProcessConfigCreator.scala | 13 +- .../sample/source/BoundedSource.scala | 41 +++- .../ModelDataActivityInfoProvider.scala | 44 ++-- 13 files changed, 308 insertions(+), 174 deletions(-) diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala index 8893f78eaa6..4f1e0196ec8 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala @@ -1,8 +1,9 @@ package pl.touk.nussknacker.engine.api.component -import io.circe.generic.JsonCodec -import io.circe.generic.extras.semiauto.{deriveUnwrappedDecoder, deriveUnwrappedEncoder} +import cats.syntax.functor._ import io.circe.{Decoder, Encoder} +import io.circe.generic.auto._ +import io.circe.syntax._ import pl.touk.nussknacker.engine.api.NodeId final case class NodesDeploymentData(dataByNodeId: Map[NodeId, NodeDeploymentData]) @@ -20,20 +21,24 @@ object NodesDeploymentData { } -@JsonCodec sealed trait NodeDeploymentData +sealed trait NodeDeploymentData final case class SqlFilteringExpression(sqlExpression: String) extends NodeDeploymentData -final case class KafkaSourceDeploymentData(offset: String) extends NodeDeploymentData - -//object NodeDeploymentData { -// -// implicit val nodeDeploymentDataEncoder: Encoder[NodeDeploymentData] = -// deriveUnwrappedEncoder[SqlFilteringExpression].contramap { case sqlExpression: SqlFilteringExpression => -// sqlExpression -// } -// -// implicit val nodeDeploymentDataDecoder: Decoder[NodeDeploymentData] = -// deriveUnwrappedDecoder[SqlFilteringExpression].map(identity) -// -//} +final case class KafkaSourceOffset(offset: Long) extends NodeDeploymentData + +object NodeDeploymentData { + + implicit val nodeDeploymentDataEncoder: Encoder[NodeDeploymentData] = + Encoder.instance { + case s: SqlFilteringExpression => s.asJson + case o: KafkaSourceOffset => o.asJson + } + + implicit val nodeDeploymentDataDecoder: Decoder[NodeDeploymentData] = + List[Decoder[NodeDeploymentData]]( + Decoder[SqlFilteringExpression].widen, + Decoder[KafkaSourceOffset].widen + ).reduceLeft(_ or _) + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala index 971887ae8d2..8e14c688134 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala @@ -210,7 +210,7 @@ object DeploymentApiEndpoints { modifiedAt: Instant ) - implicit val nodeDeploymentDataCodec: Schema[NodeDeploymentData] = Schema.string[SqlFilteringExpression].as + implicit val nodeDeploymentDataCodec: Schema[NodeDeploymentData] = Schema.derived implicit val nodesDeploymentDataCodec: Schema[NodesDeploymentData] = Schema .schemaForMap[NodeId, NodeDeploymentData](_.id) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala index 76694f952be..6fa9d2e521f 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala @@ -1,52 +1,89 @@ package pl.touk.nussknacker.ui.api -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.testkit.ScalatestRouteTest -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport -import io.circe.Json -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.api.graph.ScenarioGraph -import pl.touk.nussknacker.security.Permission -import pl.touk.nussknacker.test.base.it.NuResourcesTest -import pl.touk.nussknacker.test.config.WithSimplifiedDesignerConfig.TestProcessingType.Streaming -import pl.touk.nussknacker.test.utils.domain.ProcessTestData -import pl.touk.nussknacker.test.utils.domain.TestFactory.{mapProcessingTypeDataProvider, withPermissions} -import pl.touk.nussknacker.test.{EitherValuesDetailedMessage, PatientScalaFutures} -import pl.touk.nussknacker.test.utils.scalas.AkkaHttpExtensions.toRequestEntity +import io.restassured.RestAssured.`given` +import io.restassured.module.scala.RestAssuredSupport.AddThenToResponse +import org.hamcrest.Matchers.equalTo +import org.scalatest.freespec.AnyFreeSpecLike +import pl.touk.nussknacker.engine.build.ScenarioBuilder +import pl.touk.nussknacker.test.base.it.{NuItTest, WithSimplifiedConfigScenarioHelper} +import pl.touk.nussknacker.test.config.{WithBusinessCaseRestAssuredUsersExtensions, WithSimplifiedDesignerConfig} +import pl.touk.nussknacker.test.{NuRestAssureMatchers, RestAssuredVerboseLoggingIfValidationFails} +import pl.touk.nussknacker.engine.spel.SpelExtension._ +import pl.touk.nussknacker.test.utils.domain.TestProcessUtil class ActivityInfoResourcesSpec - extends AnyFunSuite - with ScalatestRouteTest - with Matchers - with FailFastCirceSupport - with NuResourcesTest - with PatientScalaFutures - with EitherValuesDetailedMessage { - - private val scenarioGraph: ScenarioGraph = ProcessTestData.sampleScenarioGraph - private val testPermissionAll = List(Permission.Deploy, Permission.Read, Permission.Write) - - private def route() = new ActivityInfoResources( - processService, - mapProcessingTypeDataProvider( - Streaming.stringify -> createScenarioActivityService - ) - ) - - test("get activity parameters") { - saveProcess(scenarioGraph) { - Post( - s"/activityInfo/${ProcessTestData.sampleProcessName}/activityParameters", - scenarioGraph.toJsonRequestEntity() - ) ~> withPermissions( - route(), - testPermissionAll: _* - ) ~> check { - status shouldEqual StatusCodes.OK - val content = entityAs[Json].noSpaces - content shouldBe """{}""" - } + extends AnyFreeSpecLike + with NuItTest + with WithSimplifiedDesignerConfig + with WithSimplifiedConfigScenarioHelper + with WithBusinessCaseRestAssuredUsersExtensions + with NuRestAssureMatchers + with RestAssuredVerboseLoggingIfValidationFails { + + "The scenario activity info endpoint when" - { + "return activity parameters when defined" in { + val scenario = ScenarioBuilder + .streaming("scenarioWithSourceWithDeployParameters") + .source("sourceWithParametersId", "boundedSourceWithOffset", "elements" -> "{'one', 'two', 'three'}".spel) + .emptySink("exampleSinkId", "emptySink") + + given() + .applicationState { + createSavedScenario(scenario) + } + .when() + .basicAuthAllPermUser() + .jsonBody(TestProcessUtil.toJson(scenario).noSpaces) + .post(s"$nuDesignerHttpAddress/api/activityInfo/${scenario.name.value}/activityParameters") + .Then() + .statusCode(200) + .body( + "DEPLOY[0].sourceId", + equalTo("sourceWithParametersId"), + "DEPLOY[0].parameters[0].name", + equalTo("offset"), + "DEPLOY[0].parameters[0].typ.display", + equalTo("Long") + ) + } + + "return empty map when no activity parameters" in { + val scenario = ScenarioBuilder + .streaming("scenarioWithoutParameters") + .source("sourceNoParamsId", "boundedSource", "elements" -> "{'one', 'two', 'three'}".spel) + .emptySink("exampleSinkId", "emptySink") + + given() + .applicationState { + createSavedScenario(scenario) + } + .when() + .basicAuthAllPermUser() + .jsonBody(TestProcessUtil.toJson(scenario).noSpaces) + .post(s"$nuDesignerHttpAddress/api/activityInfo/${scenario.name.value}/activityParameters") + .Then() + .statusCode(200) + .equalsJsonBody( + "{}" + ) + } + + "return no data found when there is no scenario" in { + val scenario = ScenarioBuilder + .streaming("invalidScenario") + .source("exampleSource", "boundedSource", "elements" -> "{'one', 'two', 'three'}".spel) + .emptySink("exampleSinkId", "emptySink") + + given() + .when() + .basicAuthAllPermUser() + .jsonBody(TestProcessUtil.toJson(scenario).noSpaces) + .post(s"$nuDesignerHttpAddress/api/activityInfo/${scenario.name.value}/activityParameters") + .Then() + .statusCode(404) + .equalsPlainBody( + s"No scenario ${scenario.name.value} found" + ) } } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceBusinessSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceBusinessSpec.scala index 449d2f851a2..549c7ca717f 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceBusinessSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceBusinessSpec.scala @@ -178,6 +178,7 @@ class ComponentApiHttpServiceBusinessSpec "streaming-sink-monitor", "streaming-sink-sendsms", "streaming-source-boundedsource", + "streaming-source-boundedsourcewithoffset", "streaming-source-classinstancesource", "streaming-source-communicationsource", "streaming-source-csv-source", diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceSecuritySpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceSecuritySpec.scala index 218897b3200..efbe512da75 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceSecuritySpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceSecuritySpec.scala @@ -262,6 +262,7 @@ class ComponentApiHttpServiceSecuritySpec "streaming1-sink-monitor", "streaming1-sink-sendsms", "streaming1-source-boundedsource", + "streaming1-source-boundedsourcewithoffset", "streaming1-source-classinstancesource", "streaming1-source-communicationsource", "streaming1-source-csv-source", @@ -327,6 +328,7 @@ class ComponentApiHttpServiceSecuritySpec "streaming2-sink-monitor", "streaming2-sink-sendsms", "streaming2-source-boundedsource", + "streaming2-source-boundedsourcewithoffset", "streaming2-source-classinstancesource", "streaming2-source-communicationsource", "streaming2-source-csv-source", @@ -391,6 +393,7 @@ class ComponentApiHttpServiceSecuritySpec "streaming1-sink-monitor", "streaming1-sink-sendsms", "streaming1-source-boundedsource", + "streaming1-source-boundedsourcewithoffset", "streaming1-source-classinstancesource", "streaming1-source-communicationsource", "streaming1-source-csv-source", @@ -448,6 +451,7 @@ class ComponentApiHttpServiceSecuritySpec "streaming2-sink-monitor", "streaming2-sink-sendsms", "streaming2-source-boundedsource", + "streaming2-source-boundedsourcewithoffset", "streaming2-source-classinstancesource", "streaming2-source-communicationsource", "streaming2-source-csv-source", diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceBusinessSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceBusinessSpec.scala index 5f12530554a..c4620212221 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceBusinessSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceBusinessSpec.scala @@ -65,7 +65,7 @@ class DeploymentApiHttpServiceBusinessSpec private val correctDeploymentRequest = s"""{ | "scenarioName": "$scenarioName", | "nodesDeploymentData": { - | "$sourceNodeId": "`date` = '2024-01-01'" + | "$sourceNodeId": {"sqlExpression":"`date` = '2024-01-01'"} | } |}""".stripMargin diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceDeploymentCommentSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceDeploymentCommentSpec.scala index aedab7354bb..e6042ead2a1 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceDeploymentCommentSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceDeploymentCommentSpec.scala @@ -79,7 +79,7 @@ class DeploymentApiHttpServiceDeploymentCommentSpec .jsonBody(s"""{ | "scenarioName": "$scenarioName", | "nodesDeploymentData": { - | "$sourceNodeId": "`date` = '2024-01-01'" + | "$sourceNodeId": {"sqlExpression":"`date` = '2024-01-01'"} | } |}""".stripMargin) .put(s"$nuDesignerHttpAddress/api/deployments/${DeploymentId.generate}") @@ -99,7 +99,7 @@ class DeploymentApiHttpServiceDeploymentCommentSpec .jsonBody(s"""{ | "scenarioName": "$scenarioName", | "nodesDeploymentData": { - | "$sourceNodeId": "`date` = '2024-01-01'" + | "$sourceNodeId": {"sqlExpression":"`date` = '2024-01-01'"} | }, | "comment": "deployment comment not matching configured pattern" |}""".stripMargin) @@ -121,7 +121,7 @@ class DeploymentApiHttpServiceDeploymentCommentSpec .jsonBody(s"""{ | "scenarioName": "$scenarioName", | "nodesDeploymentData": { - | "$sourceNodeId": "`date` = '2024-01-01'" + | "$sourceNodeId": {"sqlExpression":"`date` = '2024-01-01'"} | }, | "comment": "comment with $configuredPhrase" |}""".stripMargin) diff --git a/docs-internal/api/nu-designer-openapi.yaml b/docs-internal/api/nu-designer-openapi.yaml index 92c2e42c763..18bd2a47cfc 100644 --- a/docs-internal/api/nu-designer-openapi.yaml +++ b/docs-internal/api/nu-designer-openapi.yaml @@ -229,87 +229,6 @@ paths: security: - {} - httpAuth: [] - /api/processManagement/customAction/{scenarioName}/validation: - post: - tags: - - CustomAction - summary: Endpoint to validate input in custom action fields - operationId: postApiProcessmanagementCustomactionScenarionameValidation - parameters: - - name: Nu-Impersonate-User-Identity - in: header - required: false - schema: - type: - - string - - 'null' - - name: scenarioName - in: path - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/CustomActionRequest' - required: true - responses: - '200': - description: '' - content: - application/json: - schema: - $ref: '#/components/schemas/CustomActionValidationDto' - '400': - description: 'Invalid value for: header Nu-Impersonate-User-Identity, Invalid - value for: body' - content: - text/plain: - schema: - type: string - '401': - description: '' - content: - text/plain: - schema: - type: string - examples: - Example: - summary: Authentication failed - value: The supplied authentication is invalid - '403': - description: '' - content: - text/plain: - schema: - type: string - examples: - Example: - summary: Authorization failed - value: The supplied authentication is not authorized to access this - resource - '404': - description: Identity provided in the Nu-Impersonate-User-Identity header - did not match any user - content: - text/plain: - schema: - type: string - '501': - description: Impersonation is not supported for defined authentication mechanism - content: - text/plain: - schema: - type: string - examples: - Example: - summary: Cannot authenticate impersonated user as impersonation - is not supported by the authentication mechanism - value: Provided authentication method does not support impersonation - security: - - {} - - httpAuth: [] /api/app/healthCheck: get: tags: @@ -898,7 +817,8 @@ paths: example: scenarioName: scenario1 nodesDeploymentData: - sourceNodeId1: field1 = 'value' + sourceNodeId1: + sqlExpression: field1 = 'value' required: true responses: '202': @@ -1154,6 +1074,87 @@ paths: security: - {} - httpAuth: [] + /api/processManagement/customAction/{scenarioName}/validation: + post: + tags: + - CustomAction + summary: Endpoint to validate input in custom action fields + operationId: postApiProcessmanagementCustomactionScenarionameValidation + parameters: + - name: Nu-Impersonate-User-Identity + in: header + required: false + schema: + type: + - string + - 'null' + - name: scenarioName + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CustomActionRequest' + required: true + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/CustomActionValidationDto' + '400': + description: 'Invalid value for: header Nu-Impersonate-User-Identity, Invalid + value for: body' + content: + text/plain: + schema: + type: string + '401': + description: '' + content: + text/plain: + schema: + type: string + examples: + Example: + summary: Authentication failed + value: The supplied authentication is invalid + '403': + description: '' + content: + text/plain: + schema: + type: string + examples: + Example: + summary: Authorization failed + value: The supplied authentication is not authorized to access this + resource + '404': + description: Identity provided in the Nu-Impersonate-User-Identity header + did not match any user + content: + text/plain: + schema: + type: string + '501': + description: Impersonation is not supported for defined authentication mechanism + content: + text/plain: + schema: + type: string + examples: + Example: + summary: Cannot authenticate impersonated user as impersonation + is not supported by the authentication mechanism + value: Provided authentication method does not support impersonation + security: + - {} + - httpAuth: [] /api/migrate: post: tags: @@ -4806,6 +4807,15 @@ components: JsonParameterEditor: title: JsonParameterEditor type: object + KafkaSourceOffset: + title: KafkaSourceOffset + type: object + required: + - offset + properties: + offset: + type: integer + format: int64 LayoutData: title: LayoutData type: object @@ -4861,7 +4871,7 @@ components: title: Map_NodeId_NodeDeploymentData type: object additionalProperties: - type: string + $ref: '#/components/schemas/NodeDeploymentData' Map_String: title: Map_String type: object @@ -5289,6 +5299,11 @@ components: type: string type: $ref: '#/components/schemas/NodeTypes12' + NodeDeploymentData: + title: NodeDeploymentData + oneOf: + - $ref: '#/components/schemas/KafkaSourceOffset' + - $ref: '#/components/schemas/SqlFilteringExpression' NodeTypes: title: NodeTypes type: string @@ -6029,6 +6044,14 @@ components: SpelTemplateParameterEditor: title: SpelTemplateParameterEditor type: object + SqlFilteringExpression: + title: SqlFilteringExpression + type: object + required: + - sqlExpression + properties: + sqlExpression: + type: string SqlParameterEditor: title: SqlParameterEditor type: object diff --git a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/source/CollectionSource.scala b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/source/CollectionSource.scala index 1482658f7bb..5e996846743 100644 --- a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/source/CollectionSource.scala +++ b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/source/CollectionSource.scala @@ -26,10 +26,18 @@ case class CollectionSource[T]( ) extends StandardFlinkSource[T] with ReturningType { - @silent("deprecated") override def sourceStream( env: StreamExecutionEnvironment, flinkNodeContext: FlinkCustomNodeContext + ): DataStreamSource[T] = { + createSourceStream(list, env, flinkNodeContext) + } + + @silent("deprecated") + protected def createSourceStream[T]( + list: List[T], + env: StreamExecutionEnvironment, + flinkNodeContext: FlinkCustomNodeContext ): DataStreamSource[T] = { val typeInformation = TypeInformationDetection.instance.forType[T](returnType) boundedness match { diff --git a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala index c95f4188017..8d2f6265f5f 100644 --- a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala +++ b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala @@ -11,7 +11,7 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaConsumerBase} import org.apache.kafka.clients.consumer.ConsumerRecord import pl.touk.nussknacker.engine.api.NodeId -import pl.touk.nussknacker.engine.api.component.KafkaSourceDeploymentData +import pl.touk.nussknacker.engine.api.component.KafkaSourceOffset import pl.touk.nussknacker.engine.api.definition.Parameter import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.namespaces.NamingStrategy @@ -84,7 +84,7 @@ class FlinkKafkaSource[T]( override def activityParametersDefinition: Map[String, List[Parameter]] = Map( ScenarioActionName.Deploy.value -> List( - Parameter(ParameterName("offset"), Typed.apply[String]) + Parameter(ParameterName("offset"), Typed.apply[Long]) ) ) @@ -94,7 +94,7 @@ class FlinkKafkaSource[T]( flinkNodeContext: FlinkCustomNodeContext ): SourceFunction[T] = { // TODO: handle deployment parameters -> offset - val deploymentDataOpt = flinkNodeContext.nodeDeploymentData.collect { case d: KafkaSourceDeploymentData => d } + val deploymentDataOpt = flinkNodeContext.nodeDeploymentData.collect { case d: KafkaSourceOffset => d } topics.toList.foreach(KafkaUtils.setToLatestOffsetIfNeeded(kafkaConfig, _, consumerGroupId)) createFlinkSource(consumerGroupId, flinkNodeContext) } diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala index 0ce3696e419..21a1772d617 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala @@ -91,12 +91,13 @@ class DevProcessConfigCreator extends ProcessConfigCreator { )(TypeInformation.of(classOf[SampleProduct])) ) ), - "kafka-transaction" -> all(SourceFactory.noParamUnboundedStreamFactory[String](new NoEndingSource)), - "boundedSource" -> all(BoundedSource), - "oneSource" -> categories(SourceFactory.noParamUnboundedStreamFactory[String](new OneSource)), - "communicationSource" -> categories(DynamicParametersSource), - "csv-source" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new CsvSource)), - "csv-source-lite" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new LiteCsvSource(_))), + "kafka-transaction" -> all(SourceFactory.noParamUnboundedStreamFactory[String](new NoEndingSource)), + "boundedSource" -> all(BoundedSource), + "boundedSourceWithOffset" -> all(BoundedSourceWithOffset), + "oneSource" -> categories(SourceFactory.noParamUnboundedStreamFactory[String](new OneSource)), + "communicationSource" -> categories(DynamicParametersSource), + "csv-source" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new CsvSource)), + "csv-source-lite" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new LiteCsvSource(_))), "genericSourceWithCustomVariables" -> categories(GenericSourceWithCustomVariablesSample), "sql-source" -> categories(SqlSource), "classInstanceSource" -> all(new ReturningClassInstanceSource) diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala index 05a090f6347..62d680bdbe4 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala @@ -1,9 +1,15 @@ package pl.touk.nussknacker.engine.management.sample.source -import pl.touk.nussknacker.engine.api.component.UnboundedStreamComponent -import pl.touk.nussknacker.engine.api.process.SourceFactory -import pl.touk.nussknacker.engine.api.typed.typing.Unknown +import org.apache.flink.streaming.api.datastream.DataStreamSource +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment +import pl.touk.nussknacker.engine.api.component.{KafkaSourceOffset, UnboundedStreamComponent} +import pl.touk.nussknacker.engine.api.definition.Parameter +import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName +import pl.touk.nussknacker.engine.api.parameter.ParameterName +import pl.touk.nussknacker.engine.api.process.{SourceFactory, WithActivityParameters} +import pl.touk.nussknacker.engine.api.typed.typing.{Typed, Unknown} import pl.touk.nussknacker.engine.api.{MethodToInvoke, ParamName} +import pl.touk.nussknacker.engine.flink.api.process.FlinkCustomNodeContext import pl.touk.nussknacker.engine.flink.util.source.CollectionSource import scala.jdk.CollectionConverters._ @@ -15,3 +21,32 @@ object BoundedSource extends SourceFactory with UnboundedStreamComponent { new CollectionSource[Any](elements.asScala.toList, None, Unknown) } + +object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamComponent { + + @MethodToInvoke + def source(@ParamName("elements") elements: java.util.List[Any]) = + new CollectionSource[Any](elements.asScala.toList, None, Unknown) with WithActivityParameters { + + override def activityParametersDefinition: Map[String, List[Parameter]] = Map( + ScenarioActionName.Deploy.value -> List( + Parameter(ParameterName("offset"), Typed.apply[Long]) + ) + ) + + override protected def createSourceStream[T]( + list: List[T], + env: StreamExecutionEnvironment, + flinkNodeContext: FlinkCustomNodeContext + ): DataStreamSource[T] = { + val deploymentDataOpt = flinkNodeContext.nodeDeploymentData.collect { case d: KafkaSourceOffset => d } + val elementsWithOffset = deploymentDataOpt match { + case Some(data) => list.drop(data.offset.toInt) + case _ => list + } + super.createSourceStream(elementsWithOffset, env, flinkNodeContext) + } + + } + +} diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala index f74d02717d3..bfe87bb6773 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala @@ -8,9 +8,9 @@ import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.compile.ExpressionCompiler import pl.touk.nussknacker.engine.compile.nodecompilation.{LazyParameterCreationStrategy, NodeCompiler} import pl.touk.nussknacker.engine.definition.fragment.FragmentParametersDefinitionExtractor -import pl.touk.nussknacker.engine.graph.node.{SourceNodeData, asSource} +import pl.touk.nussknacker.engine.graph.node.{SourceNodeData, asFragmentInputDefinition, asSource} import pl.touk.nussknacker.engine.resultcollector.ProductionServiceInvocationCollector -import shapeless.syntax.typeable._ +import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap class ModelDataActivityInfoProvider(modelData: ModelData) extends ActivityInfoProvider { @@ -35,17 +35,37 @@ class ModelDataActivityInfoProvider(modelData: ModelData) extends ActivityInfoPr override def getActivityParameters(scenario: CanonicalProcess): Map[String, Map[String, List[Parameter]]] = { modelData.withThisAsContextClassLoader { - val asdf = scenario.collectAllNodes.flatMap(asSource) - val compiledSources = for { - source <- scenario.collectAllNodes.flatMap(asSource) - sourceObj <- prepareSourceObj(source)(scenario.metaData, NodeId(source.id)) - } yield sourceObj - val stefan = compiledSources - .flatMap(_.cast[WithActivityParameters]) - .map(_.activityParametersDefinition) - - Map.empty + val nodeToActivityToParameters = collectAllSources(scenario) + .map(source => source.id -> getActivityParameters(source, scenario.metaData)) + .toMap + groupByActivity(nodeToActivityToParameters) } } + private def groupByActivity( + nodeToActivityToParameters: Map[String, Map[String, List[Parameter]]] + ): Map[String, Map[String, List[Parameter]]] = { + val activityToNodeToParameters = for { + (node, activityToParams) <- nodeToActivityToParameters.toList + (activity, params) <- activityToParams.toList + } yield (activity, node -> params) + activityToNodeToParameters + .groupBy(_._1) + .mapValuesNow(_.map(_._2).toMap) + } + + private def getActivityParameters(source: SourceNodeData, metaData: MetaData): Map[String, List[Parameter]] = { + modelData.withThisAsContextClassLoader { + val compiledSource = prepareSourceObj(source)(metaData, NodeId(source.id)) + compiledSource match { + case Some(s: WithActivityParameters) => s.activityParametersDefinition + case _ => Map.empty + } + } + } + + private def collectAllSources(scenario: CanonicalProcess): List[SourceNodeData] = { + scenario.collectAllNodes.flatMap(asSource) ++ scenario.collectAllNodes.flatMap(asFragmentInputDefinition) + } + } From 3907192688b842629857db553e28b9b608160dd0 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Sat, 14 Sep 2024 12:12:51 +0200 Subject: [PATCH 05/20] [NU-1806] extract common parts, minor cleanups --- .../ui/api/ActivityInfoResources.scala | 9 ++-- ...ervice.scala => ActivityInfoService.scala} | 14 +++--- .../server/AkkaHttpBasedRouteProvider.scala | 4 +- .../test/base/it/NuResourcesTest.scala | 8 ---- .../activity/ActivityInfoProvider.scala | 3 +- .../CommonModelDataInfoProvider.scala | 38 +++++++++++++++ .../ModelDataActivityInfoProvider.scala | 46 +++++-------------- .../test/ModelDataTestInfoProvider.scala | 44 ++++-------------- 8 files changed, 74 insertions(+), 92 deletions(-) rename designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/{ScenarioActivityService.scala => ActivityInfoService.scala} (77%) create mode 100644 scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/CommonModelDataInfoProvider.scala diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala index 7aaaa04b4bb..c769f1bf014 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala @@ -5,7 +5,7 @@ import com.typesafe.scalalogging.LazyLogging import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport import pl.touk.nussknacker.engine.api.graph.ScenarioGraph import pl.touk.nussknacker.ui.process.ProcessService -import pl.touk.nussknacker.ui.process.newactivity.ScenarioActivityService +import pl.touk.nussknacker.ui.process.newactivity.ActivityInfoService import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataProvider import pl.touk.nussknacker.ui.security.api.LoggedUser @@ -13,7 +13,7 @@ import scala.concurrent.ExecutionContext class ActivityInfoResources( protected val processService: ProcessService, - scenarioActivityServices: ProcessingTypeDataProvider[ScenarioActivityService, _] + activityInfoService: ProcessingTypeDataProvider[ActivityInfoService, _] )(implicit val ec: ExecutionContext) extends Directives with FailFastCirceSupport @@ -25,10 +25,11 @@ class ActivityInfoResources( pathPrefix("activityInfo" / ProcessNameSegment) { processName => (post & processDetailsForName(processName)) { processDetails => entity(as[ScenarioGraph]) { scenarioGraph => - val scenarioTestService = scenarioActivityServices.forProcessingTypeUnsafe(processDetails.processingType) path("activityParameters") { complete { - scenarioTestService.getActivityParameters(scenarioGraph, processName, processDetails.isFragment) + activityInfoService + .forProcessingTypeUnsafe(processDetails.processingType) + .getActivityParameters(scenarioGraph, processDetails.processVersionUnsafe, processDetails.isFragment) } } } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ScenarioActivityService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala similarity index 77% rename from designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ScenarioActivityService.scala rename to designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala index c0ec6d6aeef..0b0b4ea370c 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ScenarioActivityService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala @@ -1,5 +1,6 @@ package pl.touk.nussknacker.ui.process.newactivity +import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.definition.StringParameterEditor import pl.touk.nussknacker.engine.api.graph.ScenarioGraph import pl.touk.nussknacker.engine.api.process.ProcessName @@ -12,14 +13,15 @@ import pl.touk.nussknacker.ui.definition.DefinitionsService import pl.touk.nussknacker.ui.security.api.LoggedUser import pl.touk.nussknacker.ui.uiresolving.UIProcessResolver -class ScenarioActivityService(activityInfoProvider: ActivityInfoProvider, processResolver: UIProcessResolver) { +// TODO: move to ActivityService? execute node compilation only once with ScenarioTestService? +class ActivityInfoService(activityInfoProvider: ActivityInfoProvider, processResolver: UIProcessResolver) { - def getActivityParameters(scenarioGraph: ScenarioGraph, processName: ProcessName, isFragment: Boolean)( + def getActivityParameters(scenarioGraph: ScenarioGraph, processVersion: ProcessVersion, isFragment: Boolean)( implicit user: LoggedUser ): Map[String, List[UISourceParameters]] = { - val canonical = toCanonicalProcess(scenarioGraph, processName, isFragment) + val canonical = toCanonicalProcess(scenarioGraph, processVersion, isFragment) activityInfoProvider - .getActivityParameters(canonical) + .getActivityParameters(processVersion, canonical) .map { case (activityName, nodeParamsMap) => activityName -> nodeParamsMap .map { case (nodeId, params) => @@ -33,10 +35,10 @@ class ScenarioActivityService(activityInfoProvider: ActivityInfoProvider, proces // copied from ScenarioTestService private def toCanonicalProcess( scenarioGraph: ScenarioGraph, - processName: ProcessName, + processVersion: ProcessVersion, isFragment: Boolean )(implicit user: LoggedUser): CanonicalProcess = { - processResolver.validateAndResolve(scenarioGraph, processName, isFragment) + processResolver.validateAndResolve(scenarioGraph, processVersion, isFragment) } // copied from ScenarioTestService diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index 37e125a59d2..9069142730e 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -56,7 +56,7 @@ import pl.touk.nussknacker.ui.process.deployment.{ import pl.touk.nussknacker.ui.process.fragment.{DefaultFragmentRepository, FragmentResolver} import pl.touk.nussknacker.ui.process.label.ScenarioLabelsService import pl.touk.nussknacker.ui.process.migrate.{HttpRemoteEnvironment, ProcessModelMigrator, TestModelMigrations} -import pl.touk.nussknacker.ui.process.newactivity.{ActivityService, ScenarioActivityService} +import pl.touk.nussknacker.ui.process.newactivity.{ActivityInfoService, ActivityService} import pl.touk.nussknacker.ui.process.newdeployment.synchronize.{ DeploymentsStatusesSynchronizationConfig, DeploymentsStatusesSynchronizationScheduler, @@ -211,7 +211,7 @@ class AkkaHttpBasedRouteProvider( ) } val scenarioActivityService = scenarioTestServiceDeps.mapValues { case (_, processResolver, _, modelData, _) => - new ScenarioActivityService( + new ActivityInfoService( new ModelDataActivityInfoProvider(modelData), processResolver ) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala index a8908d842d0..ae6f2822264 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala @@ -57,8 +57,6 @@ import pl.touk.nussknacker.ui.processreport.ProcessCounter import pl.touk.nussknacker.ui.security.api.{LoggedUser, RealLoggedUser} import pl.touk.nussknacker.ui.util.{MultipartUtils, NuPathMatchers} import slick.dbio.DBIOAction -import pl.touk.nussknacker.engine.definition.activity.ModelDataActivityInfoProvider -import pl.touk.nussknacker.ui.process.newactivity.ScenarioActivityService import java.net.URI import scala.concurrent.{ExecutionContext, Future} @@ -216,12 +214,6 @@ trait NuResourcesTest ) ) - protected def createScenarioActivityService: ScenarioActivityService = - new ScenarioActivityService( - new ModelDataActivityInfoProvider(modelData), - processResolver - ) - protected def deployRoute() = new ManagementResources( processAuthorizer = processAuthorizer, diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala index 0c679dad44a..d69048e7284 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala @@ -1,10 +1,11 @@ package pl.touk.nussknacker.engine.definition.activity +import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.definition.Parameter import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess trait ActivityInfoProvider { - def getActivityParameters(scenario: CanonicalProcess): Map[String, Map[String, List[Parameter]]] + def getActivityParameters(processVersion: ProcessVersion,scenario: CanonicalProcess): Map[String, Map[String, List[Parameter]]] } diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/CommonModelDataInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/CommonModelDataInfoProvider.scala new file mode 100644 index 00000000000..71b301a4505 --- /dev/null +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/CommonModelDataInfoProvider.scala @@ -0,0 +1,38 @@ +package pl.touk.nussknacker.engine.definition.activity + +import pl.touk.nussknacker.engine.ModelData +import pl.touk.nussknacker.engine.api.process.ComponentUseCase +import pl.touk.nussknacker.engine.api.{JobData, NodeId, process} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.compile.ExpressionCompiler +import pl.touk.nussknacker.engine.compile.nodecompilation.{LazyParameterCreationStrategy, NodeCompiler} +import pl.touk.nussknacker.engine.definition.fragment.FragmentParametersDefinitionExtractor +import pl.touk.nussknacker.engine.graph.node.{SourceNodeData, asFragmentInputDefinition, asSource} +import pl.touk.nussknacker.engine.resultcollector.ProductionServiceInvocationCollector + +class CommonModelDataInfoProvider(modelData: ModelData) { + + private lazy val expressionCompiler = ExpressionCompiler.withoutOptimization(modelData).withLabelsDictTyper + + private lazy val nodeCompiler = new NodeCompiler( + modelData.modelDefinition, + new FragmentParametersDefinitionExtractor(modelData.modelClassLoader.classLoader), + expressionCompiler, + modelData.modelClassLoader.classLoader, + Seq.empty, + ProductionServiceInvocationCollector, + ComponentUseCase.TestDataGeneration, + nonServicesLazyParamStrategy = LazyParameterCreationStrategy.default + ) + + protected def prepareSourceObj( + source: SourceNodeData + )(implicit jobData: JobData, nodeId: NodeId): Option[process.Source] = { + nodeCompiler.compileSource(source).compiledObject.toOption + } + + protected def collectAllSources(scenario: CanonicalProcess): List[SourceNodeData] = { + scenario.collectAllNodes.flatMap(asSource) ++ scenario.collectAllNodes.flatMap(asFragmentInputDefinition) + } + +} diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala index bfe87bb6773..eede23e7dda 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala @@ -2,41 +2,21 @@ package pl.touk.nussknacker.engine.definition.activity import pl.touk.nussknacker.engine.ModelData import pl.touk.nussknacker.engine.api.definition.Parameter -import pl.touk.nussknacker.engine.api.process.{ComponentUseCase, WithActivityParameters} -import pl.touk.nussknacker.engine.api.{MetaData, NodeId, process} +import pl.touk.nussknacker.engine.api.process.WithActivityParameters +import pl.touk.nussknacker.engine.api.{JobData, MetaData, NodeId, ProcessVersion} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.compile.ExpressionCompiler -import pl.touk.nussknacker.engine.compile.nodecompilation.{LazyParameterCreationStrategy, NodeCompiler} -import pl.touk.nussknacker.engine.definition.fragment.FragmentParametersDefinitionExtractor -import pl.touk.nussknacker.engine.graph.node.{SourceNodeData, asFragmentInputDefinition, asSource} -import pl.touk.nussknacker.engine.resultcollector.ProductionServiceInvocationCollector +import pl.touk.nussknacker.engine.graph.node.SourceNodeData import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap -class ModelDataActivityInfoProvider(modelData: ModelData) extends ActivityInfoProvider { +class ModelDataActivityInfoProvider(modelData: ModelData) + extends CommonModelDataInfoProvider(modelData) + with ActivityInfoProvider { - private lazy val expressionCompiler = ExpressionCompiler.withoutOptimization(modelData).withLabelsDictTyper - - private lazy val nodeCompiler = new NodeCompiler( - modelData.modelDefinition, - new FragmentParametersDefinitionExtractor(modelData.modelClassLoader.classLoader), - expressionCompiler, - modelData.modelClassLoader.classLoader, - Seq.empty, - ProductionServiceInvocationCollector, - ComponentUseCase.TestDataGeneration, - nonServicesLazyParamStrategy = LazyParameterCreationStrategy.default - ) - - private def prepareSourceObj( - source: SourceNodeData - )(implicit metaData: MetaData, nodeId: NodeId): Option[process.Source] = { - nodeCompiler.compileSource(source).compiledObject.toOption - } - - override def getActivityParameters(scenario: CanonicalProcess): Map[String, Map[String, List[Parameter]]] = { + override def getActivityParameters(processVersion: ProcessVersion,scenario: CanonicalProcess): Map[String, Map[String, List[Parameter]]] = { + val jobData = JobData(scenario.metaData, processVersion) modelData.withThisAsContextClassLoader { val nodeToActivityToParameters = collectAllSources(scenario) - .map(source => source.id -> getActivityParameters(source, scenario.metaData)) + .map(source => source.id -> getActivityParameters(source, jobData)) .toMap groupByActivity(nodeToActivityToParameters) } @@ -54,9 +34,9 @@ class ModelDataActivityInfoProvider(modelData: ModelData) extends ActivityInfoPr .mapValuesNow(_.map(_._2).toMap) } - private def getActivityParameters(source: SourceNodeData, metaData: MetaData): Map[String, List[Parameter]] = { + private def getActivityParameters(source: SourceNodeData, jobData: JobData): Map[String, List[Parameter]] = { modelData.withThisAsContextClassLoader { - val compiledSource = prepareSourceObj(source)(metaData, NodeId(source.id)) + val compiledSource = prepareSourceObj(source)(jobData, NodeId(source.id)) compiledSource match { case Some(s: WithActivityParameters) => s.activityParametersDefinition case _ => Map.empty @@ -64,8 +44,4 @@ class ModelDataActivityInfoProvider(modelData: ModelData) extends ActivityInfoPr } } - private def collectAllSources(scenario: CanonicalProcess): List[SourceNodeData] = { - scenario.collectAllNodes.flatMap(asSource) ++ scenario.collectAllNodes.flatMap(asFragmentInputDefinition) - } - } diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/test/ModelDataTestInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/test/ModelDataTestInfoProvider.scala index 9a1ef401097..bc336eac449 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/test/ModelDataTestInfoProvider.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/test/ModelDataTestInfoProvider.scala @@ -4,37 +4,19 @@ import cats.data.NonEmptyList import com.typesafe.scalalogging.LazyLogging import pl.touk.nussknacker.engine.ModelData import pl.touk.nussknacker.engine.api.definition.Parameter -import pl.touk.nussknacker.engine.api.process.{ - ComponentUseCase, - SourceTestSupport, - TestDataGenerator, - TestWithParametersSupport -} +import pl.touk.nussknacker.engine.api.process.{SourceTestSupport, TestDataGenerator, TestWithParametersSupport} import pl.touk.nussknacker.engine.api.test.{ScenarioTestData, ScenarioTestJsonRecord} -import pl.touk.nussknacker.engine.api.{JobData, MetaData, NodeId, ProcessVersion, process} +import pl.touk.nussknacker.engine.api.{JobData, NodeId, ProcessVersion} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.compile.ExpressionCompiler -import pl.touk.nussknacker.engine.compile.nodecompilation.{LazyParameterCreationStrategy, NodeCompiler} -import pl.touk.nussknacker.engine.definition.fragment.FragmentParametersDefinitionExtractor -import pl.touk.nussknacker.engine.graph.node.{SourceNodeData, asFragmentInputDefinition, asSource} -import pl.touk.nussknacker.engine.resultcollector.ProductionServiceInvocationCollector +import pl.touk.nussknacker.engine.definition.activity.CommonModelDataInfoProvider +import pl.touk.nussknacker.engine.graph.node.SourceNodeData import pl.touk.nussknacker.engine.util.ListUtil import shapeless.syntax.typeable._ -class ModelDataTestInfoProvider(modelData: ModelData) extends TestInfoProvider with LazyLogging { - - private lazy val expressionCompiler = ExpressionCompiler.withoutOptimization(modelData).withLabelsDictTyper - - private lazy val nodeCompiler = new NodeCompiler( - modelData.modelDefinition, - new FragmentParametersDefinitionExtractor(modelData.modelClassLoader.classLoader), - expressionCompiler, - modelData.modelClassLoader.classLoader, - Seq.empty, - ProductionServiceInvocationCollector, - ComponentUseCase.TestDataGeneration, - nonServicesLazyParamStrategy = LazyParameterCreationStrategy.default - ) +class ModelDataTestInfoProvider(modelData: ModelData) + extends CommonModelDataInfoProvider(modelData) + with TestInfoProvider + with LazyLogging { override def getTestingCapabilities( processVersion: ProcessVersion, @@ -127,12 +109,6 @@ class ModelDataTestInfoProvider(modelData: ModelData) extends TestInfoProvider w .getOrElse(Left("Scenario doesn't have any source supporting test data generation")) } - private def prepareSourceObj( - source: SourceNodeData - )(implicit jobData: JobData, nodeId: NodeId): Option[process.Source] = { - nodeCompiler.compileSource(source).compiledObject.toOption - } - private def generateTestData(generators: NonEmptyList[(NodeId, TestDataGenerator)], size: Int) = { modelData.withThisAsContextClassLoader { val sourceTestDataList = generators.map { case (sourceId, testDataGenerator) => @@ -167,10 +143,6 @@ class ModelDataTestInfoProvider(modelData: ModelData) extends TestInfoProvider w .map(scenarioTestRecords => ScenarioTestData(scenarioTestRecords.toList)) } - private def collectAllSources(scenario: CanonicalProcess): List[SourceNodeData] = { - scenario.collectAllNodes.flatMap(asSource) ++ scenario.collectAllNodes.flatMap(asFragmentInputDefinition) - } - private def formatError(error: String, recordIdx: Int): String = { s"Record ${recordIdx + 1} - $error" } From 1355b98d696f4fcf5845d5baa524f4f968317682 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Wed, 18 Sep 2024 17:03:57 +0200 Subject: [PATCH 06/20] [NU-1806] rebase --- .../pl/touk/nussknacker/ui/api/ActivityInfoResources.scala | 7 ++++++- .../ui/process/newactivity/ActivityInfoService.scala | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala index c769f1bf014..aee3cec6b4d 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala @@ -4,6 +4,7 @@ import akka.http.scaladsl.server.{Directives, Route} import com.typesafe.scalalogging.LazyLogging import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport import pl.touk.nussknacker.engine.api.graph.ScenarioGraph +import pl.touk.nussknacker.ui.api.utils.ScenarioDetailsOps.ScenarioWithDetailsOps import pl.touk.nussknacker.ui.process.ProcessService import pl.touk.nussknacker.ui.process.newactivity.ActivityInfoService import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataProvider @@ -29,7 +30,11 @@ class ActivityInfoResources( complete { activityInfoService .forProcessingTypeUnsafe(processDetails.processingType) - .getActivityParameters(scenarioGraph, processDetails.processVersionUnsafe, processDetails.isFragment) + .getActivityParameters( + scenarioGraph, + processDetails.processVersionUnsafe, + processDetails.isFragment + ) } } } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala index 0b0b4ea370c..78747abc840 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala @@ -3,7 +3,6 @@ package pl.touk.nussknacker.ui.process.newactivity import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.definition.StringParameterEditor import pl.touk.nussknacker.engine.api.graph.ScenarioGraph -import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.api.typed.CanBeSubclassDeterminer import pl.touk.nussknacker.engine.api.typed.typing.Typed import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess @@ -16,7 +15,11 @@ import pl.touk.nussknacker.ui.uiresolving.UIProcessResolver // TODO: move to ActivityService? execute node compilation only once with ScenarioTestService? class ActivityInfoService(activityInfoProvider: ActivityInfoProvider, processResolver: UIProcessResolver) { - def getActivityParameters(scenarioGraph: ScenarioGraph, processVersion: ProcessVersion, isFragment: Boolean)( + def getActivityParameters( + scenarioGraph: ScenarioGraph, + processVersion: ProcessVersion, + isFragment: Boolean + )( implicit user: LoggedUser ): Map[String, List[UISourceParameters]] = { val canonical = toCanonicalProcess(scenarioGraph, processVersion, isFragment) From ddcdc320bc556264b02fdbf36e184622d1b2e30a Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Mon, 23 Sep 2024 21:13:42 +0200 Subject: [PATCH 07/20] offsetResetStrategy parameter --- .../api/component/NodesDeploymentData.scala | 2 +- .../kafka/source/flink/FlinkKafkaSource.scala | 20 +++++++++--- .../sample/source/BoundedSource.scala | 32 +++++++++++++++---- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala index 4f1e0196ec8..7a9151fbf91 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala @@ -25,7 +25,7 @@ sealed trait NodeDeploymentData final case class SqlFilteringExpression(sqlExpression: String) extends NodeDeploymentData -final case class KafkaSourceOffset(offset: Long) extends NodeDeploymentData +final case class KafkaSourceOffset(offsetResetStrategy: Long) extends NodeDeploymentData object NodeDeploymentData { diff --git a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala index 8d2f6265f5f..3f01e894d96 100644 --- a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala +++ b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala @@ -12,7 +12,7 @@ import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKaf import org.apache.kafka.clients.consumer.ConsumerRecord import pl.touk.nussknacker.engine.api.NodeId import pl.touk.nussknacker.engine.api.component.KafkaSourceOffset -import pl.touk.nussknacker.engine.api.definition.Parameter +import pl.touk.nussknacker.engine.api.definition.{FixedExpressionValue, FixedValuesParameterEditor, Parameter} import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.namespaces.NamingStrategy import pl.touk.nussknacker.engine.api.parameter.ParameterName @@ -82,11 +82,21 @@ class FlinkKafkaSource[T]( protected lazy val topics: NonEmptyList[TopicName.ForSource] = preparedTopics.map(_.prepared) - override def activityParametersDefinition: Map[String, List[Parameter]] = Map( - ScenarioActionName.Deploy.value -> List( - Parameter(ParameterName("offset"), Typed.apply[Long]) + override def activityParametersDefinition: Map[String, List[Parameter]] = { + import pl.touk.nussknacker.engine.spel.SpelExtension._ + val defaultValue = if (kafkaConfig.forceLatestRead.contains(true)) Some("'LATEST'".spel) else Some("'NONE'".spel) + val offsetResetStrategyValues = List( + FixedExpressionValue("'LATEST'", "LATEST"), + FixedExpressionValue("'EARLIEST'", "EARLIEST"), + FixedExpressionValue("'NONE'", "NONE"), ) - ) + Map( + ScenarioActionName.Deploy.value -> List( + Parameter(ParameterName("offsetResetStrategy"), Typed.apply[String]) + .copy(editor = Some(FixedValuesParameterEditor(offsetResetStrategyValues)), defaultValue = defaultValue), + ) + ) + } @silent("deprecated") protected def flinkSourceFunction( diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala index 62d680bdbe4..5b40c849edc 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala @@ -3,7 +3,12 @@ package pl.touk.nussknacker.engine.management.sample.source import org.apache.flink.streaming.api.datastream.DataStreamSource import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import pl.touk.nussknacker.engine.api.component.{KafkaSourceOffset, UnboundedStreamComponent} -import pl.touk.nussknacker.engine.api.definition.Parameter +import pl.touk.nussknacker.engine.api.definition.{ + FixedExpressionValue, + FixedValuesParameterEditor, + Parameter, + StringParameterEditor +} import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.parameter.ParameterName import pl.touk.nussknacker.engine.api.process.{SourceFactory, WithActivityParameters} @@ -28,11 +33,26 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone def source(@ParamName("elements") elements: java.util.List[Any]) = new CollectionSource[Any](elements.asScala.toList, None, Unknown) with WithActivityParameters { - override def activityParametersDefinition: Map[String, List[Parameter]] = Map( - ScenarioActionName.Deploy.value -> List( - Parameter(ParameterName("offset"), Typed.apply[Long]) + override def activityParametersDefinition: Map[String, List[Parameter]] = { + + import pl.touk.nussknacker.engine.spel.SpelExtension._ + + val offsetResetStrategyValues = List( + FixedExpressionValue("'LATEST'", "LATEST"), + FixedExpressionValue("'EARLIEST'", "EARLIEST"), + FixedExpressionValue("'NONE'", "NONE"), ) - ) + + Map( + ScenarioActionName.Deploy.value -> List( + Parameter(ParameterName("offset"), Typed.apply[Long]), + Parameter(ParameterName("sometext"), Typed.apply[String]) + .copy(editor = Some(StringParameterEditor), defaultValue = Some("'example'".spel)), + Parameter(ParameterName("offsetResetStrategy"), Typed.apply[String]) + .copy(editor = Some(FixedValuesParameterEditor(offsetResetStrategyValues))), + ) + ) + } override protected def createSourceStream[T]( list: List[T], @@ -41,7 +61,7 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone ): DataStreamSource[T] = { val deploymentDataOpt = flinkNodeContext.nodeDeploymentData.collect { case d: KafkaSourceOffset => d } val elementsWithOffset = deploymentDataOpt match { - case Some(data) => list.drop(data.offset.toInt) + case Some(data) => list.drop(data.offsetResetStrategy.toInt) case _ => list } super.createSourceStream(elementsWithOffset, env, flinkNodeContext) From 809dc91b0fa5048c4196c81deca5753f39481809 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Wed, 25 Sep 2024 18:29:28 +0200 Subject: [PATCH 08/20] upd --- .../api/component/NodesDeploymentData.scala | 31 ++-------- .../engine/api/process/Source.scala | 9 ++- .../restmodel/definition/package.scala | 7 +++ .../ui/api/DeploymentApiHttpService.scala | 5 +- .../description/DeploymentApiEndpoints.scala | 15 +---- .../newactivity/ActivityInfoService.scala | 46 +++++++-------- .../ui/api/ActivityInfoResourcesSpec.scala | 10 ++-- ...DeploymentApiHttpServiceBusinessSpec.scala | 2 +- ...tApiHttpServiceDeploymentCommentSpec.scala | 6 +- docs-internal/api/nu-designer-openapi.yaml | 33 ++--------- .../api/process/FlinkCustomNodeContext.scala | 2 +- .../flink/table/source/TableSourceTest.scala | 9 +-- .../flink/table/source/TableSource.scala | 5 +- .../kafka/source/flink/FlinkKafkaSource.scala | 35 +++++++---- .../sample/source/BoundedSource.scala | 58 ++++++++++--------- .../activity/ActivityInfoProvider.scala | 4 +- .../ModelDataActivityInfoProvider.scala | 16 +++-- 17 files changed, 140 insertions(+), 153 deletions(-) diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala index 7a9151fbf91..f43126590f5 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala @@ -1,15 +1,18 @@ package pl.touk.nussknacker.engine.api.component -import cats.syntax.functor._ import io.circe.{Decoder, Encoder} -import io.circe.generic.auto._ -import io.circe.syntax._ import pl.touk.nussknacker.engine.api.NodeId +import pl.touk.nussknacker.engine.api.component.NodesDeploymentData.NodeDeploymentData final case class NodesDeploymentData(dataByNodeId: Map[NodeId, NodeDeploymentData]) object NodesDeploymentData { + // Raw deployment parameters (name -> value) that are used as additional node configuration during deployment. + // Each node can be provided with dedicated set of parameters. + // TODO: consider replacing NodeDeploymentData with Json + type NodeDeploymentData = Map[String, String] + val empty: NodesDeploymentData = NodesDeploymentData(Map.empty) implicit val nodesDeploymentDataEncoder: Encoder[NodesDeploymentData] = Encoder @@ -20,25 +23,3 @@ object NodesDeploymentData { Decoder.decodeMap[NodeId, NodeDeploymentData].map(NodesDeploymentData(_)) } - -sealed trait NodeDeploymentData - -final case class SqlFilteringExpression(sqlExpression: String) extends NodeDeploymentData - -final case class KafkaSourceOffset(offsetResetStrategy: Long) extends NodeDeploymentData - -object NodeDeploymentData { - - implicit val nodeDeploymentDataEncoder: Encoder[NodeDeploymentData] = - Encoder.instance { - case s: SqlFilteringExpression => s.asJson - case o: KafkaSourceOffset => o.asJson - } - - implicit val nodeDeploymentDataDecoder: Decoder[NodeDeploymentData] = - List[Decoder[NodeDeploymentData]]( - Decoder[SqlFilteringExpression].widen, - Decoder[KafkaSourceOffset].widen - ).reduceLeft(_ or _) - -} diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala index ed783d853c0..0a213e5bc6b 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala @@ -1,7 +1,7 @@ package pl.touk.nussknacker.engine.api.process import pl.touk.nussknacker.engine.api.component.Component._ -import pl.touk.nussknacker.engine.api.component.{Component, ProcessingMode} +import pl.touk.nussknacker.engine.api.component.{Component, ParameterConfig, ProcessingMode} import pl.touk.nussknacker.engine.api.context.ContextTransformation import pl.touk.nussknacker.engine.api.definition.{Parameter, WithExplicitTypesToExtract} import pl.touk.nussknacker.engine.api.parameter.ParameterName @@ -49,8 +49,13 @@ trait TestWithParametersSupport[+T] { self: Source => def parametersToTestData(params: Map[ParameterName, AnyRef]): T } +/** + * Used to define Source parameters for each activity + * e.g. + * {"DEPLOY": { "parametername": ...parameter configuration... } + */ trait WithActivityParameters { self: Source => - def activityParametersDefinition: Map[String, List[Parameter]] + def activityParametersDefinition: Map[String, Map[String, ParameterConfig]] } /** diff --git a/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/definition/package.scala b/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/definition/package.scala index fbc46c634d5..f660eb49f2b 100644 --- a/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/definition/package.scala +++ b/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/definition/package.scala @@ -132,6 +132,13 @@ package object definition { hintText: Option[String] ) + @JsonCodec final case class UiActivityParameterConfig( + defaultValue: Option[String], + editor: ParameterEditor, + label: Option[String], + hintText: Option[String] + ) + object UIParameter { implicit def decoder(implicit typing: Decoder[TypingResult]): Decoder[UIParameter] = deriveConfiguredDecoder[UIParameter] diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala index 5284d34e724..24461e8def3 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala @@ -1,5 +1,6 @@ package pl.touk.nussknacker.ui.api +import pl.touk.nussknacker.engine.api.component.NodesDeploymentData import pl.touk.nussknacker.engine.api.deployment.ProblemDeploymentStatus import pl.touk.nussknacker.ui.api.description.DeploymentApiEndpoints import pl.touk.nussknacker.ui.api.description.DeploymentApiEndpoints.Dtos._ @@ -29,7 +30,9 @@ class DeploymentApiHttpService( RunDeploymentCommand( id = deploymentId, scenarioName = request.scenarioName, - nodesDeploymentData = request.nodesDeploymentData, + nodesDeploymentData = NodesDeploymentData(request.nodesDeploymentData.map { case (n, p) => + (n, Map("sqlExpression" -> p)) + }), user = loggedUser ), request.comment diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala index 8e14c688134..db1fb23c53f 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala @@ -4,7 +4,6 @@ import cats.data.NonEmptyList import derevo.circe.{decoder, encoder} import derevo.derive import pl.touk.nussknacker.engine.api.NodeId -import pl.touk.nussknacker.engine.api.component.{NodeDeploymentData, NodesDeploymentData, SqlFilteringExpression} import pl.touk.nussknacker.engine.api.context.ProcessCompilationError.{ EmptyProcess, ExpressionParserCompilationError, @@ -48,9 +47,7 @@ class DeploymentApiEndpoints(auth: EndpointInput[AuthCredentials]) extends BaseE .example( RunDeploymentRequest( scenarioName = ProcessName("scenario1"), - NodesDeploymentData( - Map(NodeId("sourceNodeId1") -> SqlFilteringExpression("field1 = 'value'")) - ), + nodesDeploymentData = Map(NodeId("sourceNodeId1") -> "field1 = 'value'"), comment = None ) ) @@ -197,7 +194,7 @@ object DeploymentApiEndpoints { @derive(encoder, decoder, schema) final case class RunDeploymentRequest( scenarioName: ProcessName, - nodesDeploymentData: NodesDeploymentData, + nodesDeploymentData: Map[NodeId, String], // nodeId -> single parameter value comment: Option[ApiCallComment] ) @@ -210,13 +207,7 @@ object DeploymentApiEndpoints { modifiedAt: Instant ) - implicit val nodeDeploymentDataCodec: Schema[NodeDeploymentData] = Schema.derived - - implicit val nodesDeploymentDataCodec: Schema[NodesDeploymentData] = Schema - .schemaForMap[NodeId, NodeDeploymentData](_.id) - .map[NodesDeploymentData]((map: Map[NodeId, NodeDeploymentData]) => Some(NodesDeploymentData(map)))( - _.dataByNodeId - ) + implicit val nodesDeploymentDataCodec: Schema[Map[NodeId, String]] = Schema.schemaForMap[NodeId, String](_.id) sealed trait RunDeploymentError diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala index 78747abc840..74d59891cd9 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala @@ -1,14 +1,14 @@ package pl.touk.nussknacker.ui.process.newactivity +import io.circe.generic.JsonCodec +import pl.touk.nussknacker.engine.api.NodeId import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.api.definition.StringParameterEditor +import pl.touk.nussknacker.engine.api.definition.RawParameterEditor import pl.touk.nussknacker.engine.api.graph.ScenarioGraph -import pl.touk.nussknacker.engine.api.typed.CanBeSubclassDeterminer -import pl.touk.nussknacker.engine.api.typed.typing.Typed import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.definition.activity.ActivityInfoProvider -import pl.touk.nussknacker.restmodel.definition.UISourceParameters -import pl.touk.nussknacker.ui.definition.DefinitionsService +import pl.touk.nussknacker.restmodel.definition.UiActivityParameterConfig +import pl.touk.nussknacker.ui.process.newactivity.ActivityInfoService.{ActivityName, UiActivityNodeParameters} import pl.touk.nussknacker.ui.security.api.LoggedUser import pl.touk.nussknacker.ui.uiresolving.UIProcessResolver @@ -21,17 +21,24 @@ class ActivityInfoService(activityInfoProvider: ActivityInfoProvider, processRes isFragment: Boolean )( implicit user: LoggedUser - ): Map[String, List[UISourceParameters]] = { + ): Map[ActivityName, List[UiActivityNodeParameters]] = { val canonical = toCanonicalProcess(scenarioGraph, processVersion, isFragment) activityInfoProvider .getActivityParameters(processVersion, canonical) .map { case (activityName, nodeParamsMap) => - activityName -> nodeParamsMap - .map { case (nodeId, params) => - UISourceParameters(nodeId, params.map(DefinitionsService.createUIParameter)) - } - .map(assignUserFriendlyEditor) - .toList + activityName -> nodeParamsMap.map { case (nodeId, params) => + UiActivityNodeParameters( + NodeId(nodeId), + params.map { case (name, value) => + name -> UiActivityParameterConfig( + value.defaultValue, + value.editor.getOrElse(RawParameterEditor), + value.label, + value.hintText + ) + } + ) + }.toList } } @@ -44,16 +51,9 @@ class ActivityInfoService(activityInfoProvider: ActivityInfoProvider, processRes processResolver.validateAndResolve(scenarioGraph, processVersion, isFragment) } - // copied from ScenarioTestService - private def assignUserFriendlyEditor(uiSourceParameter: UISourceParameters): UISourceParameters = { - val adaptedParameters = uiSourceParameter.parameters.map { uiParameter => - if (CanBeSubclassDeterminer.canBeSubclassOf(uiParameter.typ, Typed.apply(classOf[String])).isValid) { - uiParameter.copy(editor = StringParameterEditor) - } else { - uiParameter - } - } - uiSourceParameter.copy(parameters = adaptedParameters) - } +} +object ActivityInfoService { + type ActivityName = String + @JsonCodec case class UiActivityNodeParameters(nodeId: NodeId, parameters: Map[String, UiActivityParameterConfig]) } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala index 6fa9d2e521f..d5c92402589 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala @@ -2,7 +2,7 @@ package pl.touk.nussknacker.ui.api import io.restassured.RestAssured.`given` import io.restassured.module.scala.RestAssuredSupport.AddThenToResponse -import org.hamcrest.Matchers.equalTo +import org.hamcrest.Matchers.{emptyOrNullString, equalTo, is, notNullValue} import org.scalatest.freespec.AnyFreeSpecLike import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.test.base.it.{NuItTest, WithSimplifiedConfigScenarioHelper} @@ -38,12 +38,10 @@ class ActivityInfoResourcesSpec .Then() .statusCode(200) .body( - "DEPLOY[0].sourceId", + "DEPLOY[0].nodeId", equalTo("sourceWithParametersId"), - "DEPLOY[0].parameters[0].name", - equalTo("offset"), - "DEPLOY[0].parameters[0].typ.display", - equalTo("Long") + "DEPLOY[0].parameters.offset", + notNullValue(), ) } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceBusinessSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceBusinessSpec.scala index c4620212221..5f12530554a 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceBusinessSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceBusinessSpec.scala @@ -65,7 +65,7 @@ class DeploymentApiHttpServiceBusinessSpec private val correctDeploymentRequest = s"""{ | "scenarioName": "$scenarioName", | "nodesDeploymentData": { - | "$sourceNodeId": {"sqlExpression":"`date` = '2024-01-01'"} + | "$sourceNodeId": "`date` = '2024-01-01'" | } |}""".stripMargin diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceDeploymentCommentSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceDeploymentCommentSpec.scala index e6042ead2a1..aedab7354bb 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceDeploymentCommentSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpServiceDeploymentCommentSpec.scala @@ -79,7 +79,7 @@ class DeploymentApiHttpServiceDeploymentCommentSpec .jsonBody(s"""{ | "scenarioName": "$scenarioName", | "nodesDeploymentData": { - | "$sourceNodeId": {"sqlExpression":"`date` = '2024-01-01'"} + | "$sourceNodeId": "`date` = '2024-01-01'" | } |}""".stripMargin) .put(s"$nuDesignerHttpAddress/api/deployments/${DeploymentId.generate}") @@ -99,7 +99,7 @@ class DeploymentApiHttpServiceDeploymentCommentSpec .jsonBody(s"""{ | "scenarioName": "$scenarioName", | "nodesDeploymentData": { - | "$sourceNodeId": {"sqlExpression":"`date` = '2024-01-01'"} + | "$sourceNodeId": "`date` = '2024-01-01'" | }, | "comment": "deployment comment not matching configured pattern" |}""".stripMargin) @@ -121,7 +121,7 @@ class DeploymentApiHttpServiceDeploymentCommentSpec .jsonBody(s"""{ | "scenarioName": "$scenarioName", | "nodesDeploymentData": { - | "$sourceNodeId": {"sqlExpression":"`date` = '2024-01-01'"} + | "$sourceNodeId": "`date` = '2024-01-01'" | }, | "comment": "comment with $configuredPhrase" |}""".stripMargin) diff --git a/docs-internal/api/nu-designer-openapi.yaml b/docs-internal/api/nu-designer-openapi.yaml index 18bd2a47cfc..8b9cc74632e 100644 --- a/docs-internal/api/nu-designer-openapi.yaml +++ b/docs-internal/api/nu-designer-openapi.yaml @@ -817,8 +817,7 @@ paths: example: scenarioName: scenario1 nodesDeploymentData: - sourceNodeId1: - sqlExpression: field1 = 'value' + sourceNodeId1: field1 = 'value' required: true responses: '202': @@ -4807,15 +4806,6 @@ components: JsonParameterEditor: title: JsonParameterEditor type: object - KafkaSourceOffset: - title: KafkaSourceOffset - type: object - required: - - offset - properties: - offset: - type: integer - format: int64 LayoutData: title: LayoutData type: object @@ -4867,11 +4857,11 @@ components: type: object additionalProperties: $ref: '#/components/schemas/Map_TypingResultInJson' - Map_NodeId_NodeDeploymentData: - title: Map_NodeId_NodeDeploymentData + Map_NodeId_String: + title: Map_NodeId_String type: object additionalProperties: - $ref: '#/components/schemas/NodeDeploymentData' + type: string Map_String: title: Map_String type: object @@ -5299,11 +5289,6 @@ components: type: string type: $ref: '#/components/schemas/NodeTypes12' - NodeDeploymentData: - title: NodeDeploymentData - oneOf: - - $ref: '#/components/schemas/KafkaSourceOffset' - - $ref: '#/components/schemas/SqlFilteringExpression' NodeTypes: title: NodeTypes type: string @@ -5734,7 +5719,7 @@ components: scenarioName: type: string nodesDeploymentData: - $ref: '#/components/schemas/Map_NodeId_NodeDeploymentData' + $ref: '#/components/schemas/Map_NodeId_String' comment: type: - string @@ -6044,14 +6029,6 @@ components: SpelTemplateParameterEditor: title: SpelTemplateParameterEditor type: object - SqlFilteringExpression: - title: SqlFilteringExpression - type: object - required: - - sqlExpression - properties: - sqlExpression: - type: string SqlParameterEditor: title: SqlParameterEditor type: object diff --git a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkCustomNodeContext.scala b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkCustomNodeContext.scala index 25e7e8b2502..b58aa347af2 100644 --- a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkCustomNodeContext.scala +++ b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkCustomNodeContext.scala @@ -3,7 +3,7 @@ package pl.touk.nussknacker.engine.flink.api.process import com.github.ghik.silencer.silent import org.apache.flink.api.common.functions.RuntimeContext import org.apache.flink.api.common.typeinfo.TypeInformation -import pl.touk.nussknacker.engine.api.component.NodeDeploymentData +import pl.touk.nussknacker.engine.api.component.NodesDeploymentData.NodeDeploymentData import pl.touk.nussknacker.engine.api.context.ValidationContext import pl.touk.nussknacker.engine.api.process.ComponentUseCase import pl.touk.nussknacker.engine.api.runtimecontext.EngineRuntimeContext diff --git a/engine/flink/components/base-tests/src/test/scala/pl/touk/nussknacker/engine/flink/table/source/TableSourceTest.scala b/engine/flink/components/base-tests/src/test/scala/pl/touk/nussknacker/engine/flink/table/source/TableSourceTest.scala index 36e2b17f08f..8857b738315 100644 --- a/engine/flink/components/base-tests/src/test/scala/pl/touk/nussknacker/engine/flink/table/source/TableSourceTest.scala +++ b/engine/flink/components/base-tests/src/test/scala/pl/touk/nussknacker/engine/flink/table/source/TableSourceTest.scala @@ -7,11 +7,12 @@ import org.scalatest.LoneElement import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.api.NodeId -import pl.touk.nussknacker.engine.api.component.{ComponentDefinition, NodesDeploymentData, SqlFilteringExpression} +import pl.touk.nussknacker.engine.api.component.{ComponentDefinition, NodesDeploymentData} import pl.touk.nussknacker.engine.api.process.ProcessObjectDependencies import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.flink.table.FlinkTableComponentProvider import pl.touk.nussknacker.engine.flink.table.definition.{FlinkDataDefinition, StubbedCatalogFactory} +import pl.touk.nussknacker.engine.flink.table.source.TableSource.SQL_EXPRESSION_PARAMETER_NAME import pl.touk.nussknacker.engine.flink.test.FlinkSpec import pl.touk.nussknacker.engine.flink.util.test.FlinkTestScenarioRunner import pl.touk.nussknacker.engine.process.FlinkJobConfig.ExecutionMode @@ -87,7 +88,7 @@ class TableSourceTest val result = runner .runWithoutData[Row]( scenario, - nodesData = NodesDeploymentData(Map(NodeId("start") -> SqlFilteringExpression("true = true"))) + nodesData = NodesDeploymentData(Map(NodeId("start") -> Map(SQL_EXPRESSION_PARAMETER_NAME -> "true = true"))) ) .validValue result.errors shouldBe empty @@ -127,7 +128,7 @@ class TableSourceTest val resultWithoutFiltering = runnerWithCatalogConfiguration .runWithoutData[Row]( scenario, - nodesData = NodesDeploymentData(Map(NodeId("start") -> SqlFilteringExpression("true = true"))) + nodesData = NodesDeploymentData(Map(NodeId("start") -> Map(SQL_EXPRESSION_PARAMETER_NAME -> "true = true"))) ) .validValue resultWithoutFiltering.errors shouldBe empty @@ -136,7 +137,7 @@ class TableSourceTest val resultWithFiltering = runnerWithCatalogConfiguration .runWithoutData[Row]( scenario, - nodesData = NodesDeploymentData(Map(NodeId("start") -> SqlFilteringExpression("true = false"))) + nodesData = NodesDeploymentData(Map(NodeId("start") -> Map(SQL_EXPRESSION_PARAMETER_NAME -> "true = false"))) ) .validValue resultWithFiltering.errors shouldBe empty diff --git a/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala b/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala index 23c7dc55588..5f475f58314 100644 --- a/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala +++ b/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala @@ -6,7 +6,6 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment import org.apache.flink.table.api.{DataTypes, Schema} import org.apache.flink.table.catalog.Column.{ComputedColumn, MetadataColumn, PhysicalColumn} import org.apache.flink.types.Row -import pl.touk.nussknacker.engine.api.component.SqlFilteringExpression import pl.touk.nussknacker.engine.api.definition.Parameter import pl.touk.nussknacker.engine.api.parameter.ParameterName import pl.touk.nussknacker.engine.api.process.{ @@ -52,7 +51,8 @@ class TableSource( val selectQuery = tableEnv.from(tableDefinition.tableId.toString) val finalQuery = flinkNodeContext.nodeDeploymentData - .collect { case SqlFilteringExpression(sqlExpression) => + .flatMap(_.get(SQL_EXPRESSION_PARAMETER_NAME)) + .collect { case sqlExpression => tableEnv.executeSql( s"CREATE TEMPORARY VIEW $filteringInternalViewName AS SELECT * FROM ${tableDefinition.tableId} WHERE $sqlExpression" ) @@ -126,5 +126,6 @@ class TableSource( } object TableSource { + val SQL_EXPRESSION_PARAMETER_NAME = "sqlExpression" private val filteringInternalViewName = "filteringView" } diff --git a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala index 3f01e894d96..f05c08aa1ac 100644 --- a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala +++ b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala @@ -11,7 +11,7 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaConsumerBase} import org.apache.kafka.clients.consumer.ConsumerRecord import pl.touk.nussknacker.engine.api.NodeId -import pl.touk.nussknacker.engine.api.component.KafkaSourceOffset +import pl.touk.nussknacker.engine.api.component.ParameterConfig import pl.touk.nussknacker.engine.api.definition.{FixedExpressionValue, FixedValuesParameterEditor, Parameter} import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.namespaces.NamingStrategy @@ -82,18 +82,28 @@ class FlinkKafkaSource[T]( protected lazy val topics: NonEmptyList[TopicName.ForSource] = preparedTopics.map(_.prepared) - override def activityParametersDefinition: Map[String, List[Parameter]] = { - import pl.touk.nussknacker.engine.spel.SpelExtension._ - val defaultValue = if (kafkaConfig.forceLatestRead.contains(true)) Some("'LATEST'".spel) else Some("'NONE'".spel) - val offsetResetStrategyValues = List( - FixedExpressionValue("'LATEST'", "LATEST"), - FixedExpressionValue("'EARLIEST'", "EARLIEST"), - FixedExpressionValue("'NONE'", "NONE"), + private val OFFSET_RESET_STRATEGY_PARAM_NAME = "offsetResetStrategy" + + override def activityParametersDefinition: Map[String, Map[String, ParameterConfig]] = { + val defaultValue = if (kafkaConfig.forceLatestRead.contains(true)) Some("LATEST") else Some("NONE") + val editor = Some( + FixedValuesParameterEditor( + List( + FixedExpressionValue("LATEST", "LATEST"), + FixedExpressionValue("EARLIEST", "EARLIEST"), + FixedExpressionValue("NONE", "NONE"), + ) + ) ) Map( - ScenarioActionName.Deploy.value -> List( - Parameter(ParameterName("offsetResetStrategy"), Typed.apply[String]) - .copy(editor = Some(FixedValuesParameterEditor(offsetResetStrategyValues)), defaultValue = defaultValue), + ScenarioActionName.Deploy.value -> Map( + OFFSET_RESET_STRATEGY_PARAM_NAME -> ParameterConfig( + defaultValue = defaultValue, + editor = editor, + validators = None, + label = None, + hintText = None + ), ) ) } @@ -104,7 +114,8 @@ class FlinkKafkaSource[T]( flinkNodeContext: FlinkCustomNodeContext ): SourceFunction[T] = { // TODO: handle deployment parameters -> offset - val deploymentDataOpt = flinkNodeContext.nodeDeploymentData.collect { case d: KafkaSourceOffset => d } + val offsetResetStrategy = + flinkNodeContext.nodeDeploymentData.flatMap(_.get(OFFSET_RESET_STRATEGY_PARAM_NAME)).getOrElse() topics.toList.foreach(KafkaUtils.setToLatestOffsetIfNeeded(kafkaConfig, _, consumerGroupId)) createFlinkSource(consumerGroupId, flinkNodeContext) } diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala index 5b40c849edc..4cce041c505 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala @@ -2,13 +2,8 @@ package pl.touk.nussknacker.engine.management.sample.source import org.apache.flink.streaming.api.datastream.DataStreamSource import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment -import pl.touk.nussknacker.engine.api.component.{KafkaSourceOffset, UnboundedStreamComponent} -import pl.touk.nussknacker.engine.api.definition.{ - FixedExpressionValue, - FixedValuesParameterEditor, - Parameter, - StringParameterEditor -} +import pl.touk.nussknacker.engine.api.component.{ParameterConfig, UnboundedStreamComponent} +import pl.touk.nussknacker.engine.api.definition.{FixedExpressionValue, FixedValuesParameterEditor, RawParameterEditor} import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.parameter.ParameterName import pl.touk.nussknacker.engine.api.process.{SourceFactory, WithActivityParameters} @@ -33,23 +28,34 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone def source(@ParamName("elements") elements: java.util.List[Any]) = new CollectionSource[Any](elements.asScala.toList, None, Unknown) with WithActivityParameters { - override def activityParametersDefinition: Map[String, List[Parameter]] = { - - import pl.touk.nussknacker.engine.spel.SpelExtension._ - - val offsetResetStrategyValues = List( - FixedExpressionValue("'LATEST'", "LATEST"), - FixedExpressionValue("'EARLIEST'", "EARLIEST"), - FixedExpressionValue("'NONE'", "NONE"), + override def activityParametersDefinition: Map[String, Map[String, ParameterConfig]] = { + val fixedValuesEditor = Some( + FixedValuesParameterEditor( + List( + FixedExpressionValue("LATEST", "LATEST"), + FixedExpressionValue("EARLIEST", "EARLIEST"), + FixedExpressionValue("NONE", "NONE"), + ) + ) ) - Map( - ScenarioActionName.Deploy.value -> List( - Parameter(ParameterName("offset"), Typed.apply[Long]), - Parameter(ParameterName("sometext"), Typed.apply[String]) - .copy(editor = Some(StringParameterEditor), defaultValue = Some("'example'".spel)), - Parameter(ParameterName("offsetResetStrategy"), Typed.apply[String]) - .copy(editor = Some(FixedValuesParameterEditor(offsetResetStrategyValues))), + ScenarioActionName.Deploy.value -> Map( + "offset" -> ParameterConfig( + defaultValue = None, + editor = Some(RawParameterEditor), + validators = None, + label = None, + hintText = Some( + "Set offset to setup source to emit elements from specified start point in input collection. Empty field resets collection to the beginning." + ) + ), + "offsetResetStrategy" -> ParameterConfig( + defaultValue = Some("EARLIEST"), + editor = fixedValuesEditor, + validators = None, + label = None, + hintText = Some("Example of parameter with fixed values") + ), ) ) } @@ -59,10 +65,10 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone env: StreamExecutionEnvironment, flinkNodeContext: FlinkCustomNodeContext ): DataStreamSource[T] = { - val deploymentDataOpt = flinkNodeContext.nodeDeploymentData.collect { case d: KafkaSourceOffset => d } - val elementsWithOffset = deploymentDataOpt match { - case Some(data) => list.drop(data.offsetResetStrategy.toInt) - case _ => list + val offsetOpt = flinkNodeContext.nodeDeploymentData.flatMap(_.get("offset")) + val elementsWithOffset = offsetOpt match { + case Some(offset) => list.drop(offset.toInt) + case _ => list } super.createSourceStream(elementsWithOffset, env, flinkNodeContext) } diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala index d69048e7284..090d9dbf425 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala @@ -1,11 +1,11 @@ package pl.touk.nussknacker.engine.definition.activity import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.api.definition.Parameter +import pl.touk.nussknacker.engine.api.component.ParameterConfig import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess trait ActivityInfoProvider { - def getActivityParameters(processVersion: ProcessVersion,scenario: CanonicalProcess): Map[String, Map[String, List[Parameter]]] + def getActivityParameters(processVersion: ProcessVersion, scenario: CanonicalProcess): Map[String, Map[String, Map[String, ParameterConfig]]] } diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala index eede23e7dda..e92f6e3e0d2 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala @@ -1,7 +1,7 @@ package pl.touk.nussknacker.engine.definition.activity import pl.touk.nussknacker.engine.ModelData -import pl.touk.nussknacker.engine.api.definition.Parameter +import pl.touk.nussknacker.engine.api.component.ParameterConfig import pl.touk.nussknacker.engine.api.process.WithActivityParameters import pl.touk.nussknacker.engine.api.{JobData, MetaData, NodeId, ProcessVersion} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess @@ -12,7 +12,10 @@ class ModelDataActivityInfoProvider(modelData: ModelData) extends CommonModelDataInfoProvider(modelData) with ActivityInfoProvider { - override def getActivityParameters(processVersion: ProcessVersion,scenario: CanonicalProcess): Map[String, Map[String, List[Parameter]]] = { + override def getActivityParameters( + processVersion: ProcessVersion, + scenario: CanonicalProcess + ): Map[String, Map[String, Map[String, ParameterConfig]]] = { val jobData = JobData(scenario.metaData, processVersion) modelData.withThisAsContextClassLoader { val nodeToActivityToParameters = collectAllSources(scenario) @@ -23,8 +26,8 @@ class ModelDataActivityInfoProvider(modelData: ModelData) } private def groupByActivity( - nodeToActivityToParameters: Map[String, Map[String, List[Parameter]]] - ): Map[String, Map[String, List[Parameter]]] = { + nodeToActivityToParameters: Map[String, Map[String, Map[String, ParameterConfig]]] + ): Map[String, Map[String, Map[String, ParameterConfig]]] = { val activityToNodeToParameters = for { (node, activityToParams) <- nodeToActivityToParameters.toList (activity, params) <- activityToParams.toList @@ -34,7 +37,10 @@ class ModelDataActivityInfoProvider(modelData: ModelData) .mapValuesNow(_.map(_._2).toMap) } - private def getActivityParameters(source: SourceNodeData, jobData: JobData): Map[String, List[Parameter]] = { + private def getActivityParameters( + source: SourceNodeData, + jobData: JobData + ): Map[String, Map[String, ParameterConfig]] = { modelData.withThisAsContextClassLoader { val compiledSource = prepareSourceObj(source)(jobData, NodeId(source.id)) compiledSource match { From a76217c71a786600b2503e0af28bd93a34d841d6 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Thu, 26 Sep 2024 00:11:38 +0200 Subject: [PATCH 09/20] change deploy api --- designer/client/src/http/HttpService.ts | 15 ++++++++---- .../ui/api/ManagementResources.scala | 24 ++++++++++++------- .../test/base/it/NuResourcesTest.scala | 6 ++++- .../ui/api/ManagementResourcesSpec.scala | 12 ++++++++-- 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/designer/client/src/http/HttpService.ts b/designer/client/src/http/HttpService.ts index 0776de1c467..2eccd59af46 100644 --- a/designer/client/src/http/HttpService.ts +++ b/designer/client/src/http/HttpService.ts @@ -18,7 +18,7 @@ import { } from "../components/Process/types"; import { ToolbarsConfig } from "../components/toolbarSettings/types"; import { AuthenticationSettings } from "../reducers/settings"; -import { Expression, NodeType, ProcessAdditionalFields, ProcessDefinitionData, ReturnedType, ScenarioGraph, VariableTypes } from "../types"; +import { Expression, NodeId, NodeType, ProcessAdditionalFields, ProcessDefinitionData, ScenarioGraph, VariableTypes } from "../types"; import { Instant, WithId } from "../types/common"; import { BackendNotification } from "../containers/Notifications"; import { ProcessCounts } from "../reducers/graph"; @@ -27,7 +27,6 @@ import { AdditionalInfo } from "../components/graph/node-modal/NodeAdditionalInf import { withoutHackOfEmptyEdges } from "../components/graph/GraphPartialsInTS/EdgeUtils"; import { CaretPosition2d, ExpressionSuggestion } from "../components/graph/node-modal/editors/expression/ExpressionSuggester"; import { GenericValidationRequest } from "../actions/nk/genericAction"; -import { EventTrackingSelector } from "../containers/event-tracking"; import { EventTrackingSelectorType, EventTrackingType } from "../containers/event-tracking/use-register-tracking-events"; import { AvailableScenarioLabels, ScenarioLabelsValidationResponse } from "../components/Labels/types"; @@ -315,9 +314,17 @@ class HttpService { ); } - deploy(processName: string, comment?: string): Promise<{ isSuccess: boolean }> { + deploy( + processName: string, + comment?: string, + nodesDeploymentData?: Record>, + ): Promise<{ isSuccess: boolean }> { + const runDeploymentRequest = { + ...(nodesDeploymentData && { nodesDeploymentData: nodesDeploymentData }), + ...(comment && { comment: comment }), + }; return api - .post(`/processManagement/deploy/${encodeURIComponent(processName)}`, comment) + .post(`/processManagement/deploy/${encodeURIComponent(processName)}`, runDeploymentRequest) .then(() => { return { isSuccess: true }; }) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala index 879b1d865d8..23b237f2113 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala @@ -6,14 +6,17 @@ import akka.http.scaladsl.server._ import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} import com.typesafe.scalalogging.LazyLogging import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport +import io.circe.generic.JsonCodec import io.circe.generic.extras.semiauto.deriveConfiguredEncoder import io.circe.{Decoder, Encoder, Json, parser} import io.dropwizard.metrics5.MetricRegistry import pl.touk.nussknacker.engine.ModelData +import pl.touk.nussknacker.engine.api.NodeId import pl.touk.nussknacker.engine.api.component.NodesDeploymentData import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.graph.ScenarioGraph +import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.testmode.TestProcess._ import pl.touk.nussknacker.restmodel.{CustomActionRequest, CustomActionResponse} import pl.touk.nussknacker.ui.api.description.NodesApiEndpoints.Dtos.{ @@ -72,6 +75,11 @@ object ManagementResources { } + @JsonCodec final case class RunDeploymentRequest( + nodesDeploymentData: Option[NodesDeploymentData], + comment: Option[ApiCallComment] + ) + } class ManagementResources( @@ -125,17 +133,16 @@ class ManagementResources( } } ~ path("deploy" / ProcessNameSegment) { processName => - (post & processId(processName) & entity(as[Option[String]]) & parameters(Symbol("savepointPath"))) { - (processIdWithName, comment, savepointPath) => + (post & processId(processName) & entity(as[RunDeploymentRequest]) & parameters(Symbol("savepointPath"))) { + (processIdWithName, request, savepointPath) => canDeploy(processIdWithName) { complete { deploymentService .processCommand( RunDeploymentCommand( // adminProcessManagement endpoint is not used by the designer client. It is a part of API for tooling purpose - commonData = CommonCommandData(processIdWithName, comment.map(ApiCallComment(_)), user), - nodesDeploymentData = - NodesDeploymentData.empty, // TODO: here goes map of parameters defined by activityParameters + commonData = CommonCommandData(processIdWithName, request.comment, user), + nodesDeploymentData = request.nodesDeploymentData.getOrElse(NodesDeploymentData.empty), stateRestoringStrategy = StateRestoringStrategy.RestoreStateFromCustomSavepoint(savepointPath) ) ) @@ -149,16 +156,15 @@ class ManagementResources( pathPrefix("processManagement") { path("deploy" / ProcessNameSegment) { processName => - (post & processId(processName) & entity(as[Option[String]])) { (processIdWithName, comment) => + (post & processId(processName) & entity(as[RunDeploymentRequest])) { (processIdWithName, request) => canDeploy(processIdWithName) { complete { measureTime("deployment", metricRegistry) { deploymentService .processCommand( RunDeploymentCommand( - commonData = CommonCommandData(processIdWithName, comment.map(UserComment), user), - nodesDeploymentData = - NodesDeploymentData.empty, // TODO: here goes map of parameters defined by activityParameters + commonData = CommonCommandData(processIdWithName, request.comment, user), + nodesDeploymentData = request.nodesDeploymentData.getOrElse(NodesDeploymentData.empty), stateRestoringStrategy = StateRestoringStrategy.RestoreStateFromReplacedJobSavepoint ) ) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala index ae6f2822264..228bcf3a39f 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala @@ -38,6 +38,7 @@ import pl.touk.nussknacker.test.utils.domain.TestFactory._ import pl.touk.nussknacker.test.utils.domain.{ProcessTestData, TestFactory} import pl.touk.nussknacker.test.utils.scalas.AkkaHttpExtensions.toRequestEntity import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig +import pl.touk.nussknacker.ui.api.ManagementResources.RunDeploymentRequest import pl.touk.nussknacker.ui.api._ import pl.touk.nussknacker.ui.config.FeatureTogglesConfig import pl.touk.nussknacker.ui.config.scenariotoolbar.CategoriesScenarioToolbarsConfigParser @@ -332,7 +333,10 @@ trait NuResourcesTest ): RouteTestResult = Post( s"/processManagement/deploy/$processName", - HttpEntity(ContentTypes.`application/json`, comment.getOrElse("")) + HttpEntity( + ContentTypes.`application/json`, + RunDeploymentRequest(None, comment.map(ApiCallComment(_))).asJson.noSpaces + ) ) ~> withPermissions(deployRoute(), Permission.Deploy, Permission.Read) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ManagementResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ManagementResourcesSpec.scala index 988b54345e8..ec88edbcc7e 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ManagementResourcesSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ManagementResourcesSpec.scala @@ -1,6 +1,6 @@ package pl.touk.nussknacker.ui.api -import akka.http.scaladsl.model.{ContentTypeRange, StatusCodes} +import akka.http.scaladsl.model.{ContentTypeRange, ContentTypes, HttpEntity, StatusCodes} import akka.http.scaladsl.server import akka.http.scaladsl.testkit.ScalatestRouteTest import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} @@ -26,6 +26,7 @@ import pl.touk.nussknacker.test.base.it.NuResourcesTest import pl.touk.nussknacker.test.mock.MockDeploymentManager import pl.touk.nussknacker.test.utils.domain.TestFactory.{withAllPermissions, withPermissions} import pl.touk.nussknacker.test.utils.domain.{ProcessTestData, TestFactory} +import pl.touk.nussknacker.ui.api.ManagementResources.RunDeploymentRequest import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos import pl.touk.nussknacker.ui.process.ScenarioQuery import pl.touk.nussknacker.ui.process.exception.ProcessIllegalAction @@ -234,8 +235,15 @@ class ManagementResourcesSpec } test("not authorize user with write permission to deploy") { + import io.circe.syntax._ saveCanonicalProcessAndAssertSuccess(ProcessTestData.sampleScenario) - Post(s"/processManagement/deploy/${ProcessTestData.sampleScenario.name}") ~> withPermissions( + Post( + s"/processManagement/deploy/${ProcessTestData.sampleScenario.name}", + HttpEntity( + ContentTypes.`application/json`, + RunDeploymentRequest(None, None).asJson.noSpaces + ) + ) ~> withPermissions( deployRoute(), Permission.Write ) ~> check { From 4de50b8b5734a1cf0951687094cb2004aef84049 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Thu, 26 Sep 2024 00:32:58 +0200 Subject: [PATCH 10/20] fe getActivityParameters --- designer/client/src/actions/actionTypes.ts | 1 + designer/client/src/actions/nk/process.ts | 10 ++++++++++ .../components/modals/DeployProcessDialog.tsx | 7 ++++++- .../GenericAction/useActivityCapabilities.tsx | 17 +++++++++++++++++ .../scenarioActions/buttons/DeployButton.tsx | 5 +++++ designer/client/src/http/HttpService.ts | 15 +++++++++++++++ designer/client/src/reducers/graph/reducer.ts | 6 ++++++ designer/client/src/reducers/graph/types.ts | 2 ++ designer/client/src/reducers/selectors/graph.ts | 2 ++ designer/client/src/types/activity.ts | 17 +++++++++++++++++ 10 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 designer/client/src/components/modals/GenericAction/useActivityCapabilities.tsx create mode 100644 designer/client/src/types/activity.ts diff --git a/designer/client/src/actions/actionTypes.ts b/designer/client/src/actions/actionTypes.ts index 577b0fd71db..133f5a22fcf 100644 --- a/designer/client/src/actions/actionTypes.ts +++ b/designer/client/src/actions/actionTypes.ts @@ -23,6 +23,7 @@ export type ActionTypes = | "PROCESS_RENAME" | "EDIT_LABELS" | "SHOW_METRICS" + | "UPDATE_ACTIVITY_PARAMETERS" | "UPDATE_TEST_CAPABILITIES" | "UPDATE_TEST_FORM_PARAMETERS" | "DISPLAY_PROCESS" diff --git a/designer/client/src/actions/nk/process.ts b/designer/client/src/actions/nk/process.ts index 6ac7d7e5b67..7b3aa127f61 100644 --- a/designer/client/src/actions/nk/process.ts +++ b/designer/client/src/actions/nk/process.ts @@ -37,6 +37,16 @@ export function loadProcessState(processName: ProcessName): ThunkAction { ); } +export function fetchActivityFormParameters(processName: ProcessName, scenarioGraph: ScenarioGraph) { + return (dispatch) => + HttpService.getActivityParameters(processName, scenarioGraph).then(({ data }) => { + dispatch({ + type: "UPDATE_ACTIVITY_PARAMETERS", + activityParameters: data, + }); + }); +} + export function fetchTestFormParameters(processName: ProcessName, scenarioGraph: ScenarioGraph) { return (dispatch) => HttpService.getTestFormParameters(processName, scenarioGraph).then(({ data }) => { diff --git a/designer/client/src/components/modals/DeployProcessDialog.tsx b/designer/client/src/components/modals/DeployProcessDialog.tsx index 369209ca610..a9c468f2b10 100644 --- a/designer/client/src/components/modals/DeployProcessDialog.tsx +++ b/designer/client/src/components/modals/DeployProcessDialog.tsx @@ -3,7 +3,7 @@ import { WindowButtonProps, WindowContentProps } from "@touk/window-manager"; import React, { useCallback, useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; import { useDispatch, useSelector } from "react-redux"; -import { getProcessName } from "../../reducers/selectors/graph"; +import { getActivityParameters, getProcessName } from "../../reducers/selectors/graph"; import { getFeatureSettings } from "../../reducers/selectors/settings"; import { ProcessName } from "../Process/types"; import { PromptContent, WindowKind } from "../../windowManager"; @@ -23,6 +23,11 @@ export function DeployProcessDialog(props: WindowContentProps { + if (isRenamed) return; + dispatch(fetchActivityFormParameters(scenarioName, scenarioGraph)); + }, [dispatch, isRenamed, scenarioName, scenarioGraph]); +} diff --git a/designer/client/src/components/toolbars/scenarioActions/buttons/DeployButton.tsx b/designer/client/src/components/toolbars/scenarioActions/buttons/DeployButton.tsx index 740601021fb..687c027c72c 100644 --- a/designer/client/src/components/toolbars/scenarioActions/buttons/DeployButton.tsx +++ b/designer/client/src/components/toolbars/scenarioActions/buttons/DeployButton.tsx @@ -13,6 +13,8 @@ import { ToolbarButton } from "../../../toolbarComponents/toolbarButtons"; import { ToolbarButtonProps } from "../../types"; import { ACTION_DIALOG_WIDTH } from "../../../../stylesheets/variables"; +import { useActivityCapabilities } from "../../../modals/GenericAction/useActivityCapabilities"; + export default function DeployButton(props: ToolbarButtonProps) { const dispatch = useDispatch(); const deployPossible = useSelector(isDeployPossible); @@ -22,6 +24,9 @@ export default function DeployButton(props: ToolbarButtonProps) { const capabilities = useSelector(getCapabilities); const { disabled, type } = props; + // TODO: find better place to reload activity capabilities and properties + useActivityCapabilities(); + const available = !disabled && deployPossible && capabilities.deploy; const { t } = useTranslation(); diff --git a/designer/client/src/http/HttpService.ts b/designer/client/src/http/HttpService.ts index 2eccd59af46..6a37525efec 100644 --- a/designer/client/src/http/HttpService.ts +++ b/designer/client/src/http/HttpService.ts @@ -592,6 +592,21 @@ class HttpService { return promise; } + getActivityParameters(processName: string, scenarioGraph: ScenarioGraph) { + const promise = api.post( + `/activityInfo/${encodeURIComponent(processName)}/activityParameters`, + this.#sanitizeScenarioGraph(scenarioGraph), + ); + promise.catch((error) => + this.#addError( + i18next.t("notification.error.failedToGetTestParameters", "Failed to get activity parameters definition"), + error, + true, + ), + ); + return promise; + } + generateTestData(processName: string, testSampleSize: string, scenarioGraph: ScenarioGraph): Promise { const promise = api.post( `/testInfo/${encodeURIComponent(processName)}/generate/${testSampleSize}`, diff --git a/designer/client/src/reducers/graph/reducer.ts b/designer/client/src/reducers/graph/reducer.ts index 3f3d28aad87..f89194d86b6 100644 --- a/designer/client/src/reducers/graph/reducer.ts +++ b/designer/client/src/reducers/graph/reducer.ts @@ -84,6 +84,12 @@ const graphReducer: Reducer = (state = emptyGraphState, action) => { testFormParameters: action.testFormParameters, }; } + case "UPDATE_ACTIVITY_PARAMETERS": { + return { + ...state, + activityParameters: action.activityParameters, + }; + } case "DISPLAY_PROCESS": { const { scenario } = action; return { diff --git a/designer/client/src/reducers/graph/types.ts b/designer/client/src/reducers/graph/types.ts index 4f4002057f5..5e16836f8c0 100644 --- a/designer/client/src/reducers/graph/types.ts +++ b/designer/client/src/reducers/graph/types.ts @@ -1,6 +1,7 @@ import { Layout, RefreshData } from "../../actions/nk"; import { Scenario } from "../../components/Process/types"; import { TestCapabilities, TestFormParameters, TestResults } from "../../common/TestResultUtils"; +import { ActivityParameters } from "../../types/activity"; export interface NodeCounts { errors?: number; @@ -15,6 +16,7 @@ export type GraphState = { scenario?: Scenario; selectionState?: string[]; layout: Layout; + activityParameters?: ActivityParameters; testCapabilities?: TestCapabilities; testFormParameters?: TestFormParameters[]; testResults: TestResults; diff --git a/designer/client/src/reducers/selectors/graph.ts b/designer/client/src/reducers/selectors/graph.ts index 41642d30f66..1d8f45e7b7f 100644 --- a/designer/client/src/reducers/selectors/graph.ts +++ b/designer/client/src/reducers/selectors/graph.ts @@ -8,6 +8,7 @@ import { ProcessCounts } from "../graph"; import { RootState } from "../index"; import { getProcessState } from "./scenarioState"; import { TestFormParameters } from "../../common/TestResultUtils"; +import { ActivityParameters } from "../../types/activity"; export const getGraph = (state: RootState) => state.graphReducer.history.present; @@ -65,6 +66,7 @@ export const isArchivePossible = createSelector( [getProcessState, isFragment], (state, isFragment) => isFragment || ProcessStateUtils.canArchive(state), ); +export const getActivityParameters = createSelector(getGraph, (g) => g.activityParameters || ({} as ActivityParameters)); export const getTestCapabilities = createSelector(getGraph, (g) => g.testCapabilities); export const getTestParameters = createSelector(getGraph, (g) => g.testFormParameters || ([] as TestFormParameters[])); export const getTestResults = createSelector(getGraph, (g) => g.testResults); diff --git a/designer/client/src/types/activity.ts b/designer/client/src/types/activity.ts new file mode 100644 index 00000000000..bd8e9892fcf --- /dev/null +++ b/designer/client/src/types/activity.ts @@ -0,0 +1,17 @@ +import { NodeId } from "./node"; + +export interface ActivityParameterConfig { + editor: any; + label: string; + defaultValue: string | null; + hintText: string | null; +} + +export interface ActivityNodeParameters { + nodeId: NodeId; + parameters: Record; +} + +export type ActivityName = string; + +export type ActivityParameters = Record; From f67e0bf167eb60ac8897167bf5fc200c3aebcaf6 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Thu, 26 Sep 2024 00:57:16 +0200 Subject: [PATCH 11/20] cleanup --- designer/client/src/http/HttpService.ts | 8 +++----- .../nussknacker/ui/api/DeploymentApiHttpService.scala | 4 ++-- .../pl/touk/nussknacker/ui/api/ManagementResources.scala | 2 -- .../ui/api/description/DeploymentApiEndpoints.scala | 2 +- .../touk/nussknacker/test/base/it/NuResourcesTest.scala | 1 + .../nussknacker/ui/api/ActivityInfoResourcesSpec.scala | 2 +- .../engine/kafka/source/flink/FlinkKafkaSource.scala | 3 +-- .../engine/management/sample/source/BoundedSource.scala | 4 ++-- 8 files changed, 11 insertions(+), 15 deletions(-) diff --git a/designer/client/src/http/HttpService.ts b/designer/client/src/http/HttpService.ts index 6a37525efec..0f1b29e68ed 100644 --- a/designer/client/src/http/HttpService.ts +++ b/designer/client/src/http/HttpService.ts @@ -95,6 +95,8 @@ export type SourceWithParametersTest = { parameterExpressions: { [paramName: string]: Expression }; }; +export type NodesDeploymentData = Record>; + export type NodeUsageData = { fragmentNodeId?: string; nodeId: string; @@ -314,11 +316,7 @@ class HttpService { ); } - deploy( - processName: string, - comment?: string, - nodesDeploymentData?: Record>, - ): Promise<{ isSuccess: boolean }> { + deploy(processName: string, comment?: string, nodesDeploymentData?: NodesDeploymentData): Promise<{ isSuccess: boolean }> { const runDeploymentRequest = { ...(nodesDeploymentData && { nodesDeploymentData: nodesDeploymentData }), ...(comment && { comment: comment }), diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala index 24461e8def3..02fe195048a 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala @@ -30,8 +30,8 @@ class DeploymentApiHttpService( RunDeploymentCommand( id = deploymentId, scenarioName = request.scenarioName, - nodesDeploymentData = NodesDeploymentData(request.nodesDeploymentData.map { case (n, p) => - (n, Map("sqlExpression" -> p)) + nodesDeploymentData = NodesDeploymentData(request.nodesDeploymentData.map { case (nodeId, paramValue) => + (nodeId, Map("sqlExpression" -> paramValue)) }), user = loggedUser ), diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala index 23b237f2113..668100cf97f 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala @@ -11,12 +11,10 @@ import io.circe.generic.extras.semiauto.deriveConfiguredEncoder import io.circe.{Decoder, Encoder, Json, parser} import io.dropwizard.metrics5.MetricRegistry import pl.touk.nussknacker.engine.ModelData -import pl.touk.nussknacker.engine.api.NodeId import pl.touk.nussknacker.engine.api.component.NodesDeploymentData import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.graph.ScenarioGraph -import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.testmode.TestProcess._ import pl.touk.nussknacker.restmodel.{CustomActionRequest, CustomActionResponse} import pl.touk.nussknacker.ui.api.description.NodesApiEndpoints.Dtos.{ diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala index db1fb23c53f..c80052f42e1 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala @@ -194,7 +194,7 @@ object DeploymentApiEndpoints { @derive(encoder, decoder, schema) final case class RunDeploymentRequest( scenarioName: ProcessName, - nodesDeploymentData: Map[NodeId, String], // nodeId -> single parameter value + nodesDeploymentData: Map[NodeId, String], // nodeId -> single parameter value (currently sqlExpression) comment: Option[ApiCallComment] ) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala index 228bcf3a39f..89b2de538b3 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala @@ -58,6 +58,7 @@ import pl.touk.nussknacker.ui.processreport.ProcessCounter import pl.touk.nussknacker.ui.security.api.{LoggedUser, RealLoggedUser} import pl.touk.nussknacker.ui.util.{MultipartUtils, NuPathMatchers} import slick.dbio.DBIOAction + import java.net.URI import scala.concurrent.{ExecutionContext, Future} diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala index d5c92402589..1bfc22968d2 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala @@ -2,7 +2,7 @@ package pl.touk.nussknacker.ui.api import io.restassured.RestAssured.`given` import io.restassured.module.scala.RestAssuredSupport.AddThenToResponse -import org.hamcrest.Matchers.{emptyOrNullString, equalTo, is, notNullValue} +import org.hamcrest.Matchers.{equalTo, notNullValue} import org.scalatest.freespec.AnyFreeSpecLike import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.test.base.it.{NuItTest, WithSimplifiedConfigScenarioHelper} diff --git a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala index f05c08aa1ac..458420c3eae 100644 --- a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala +++ b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala @@ -24,7 +24,6 @@ import pl.touk.nussknacker.engine.api.process.{ } import pl.touk.nussknacker.engine.api.runtimecontext.{ContextIdGenerator, EngineRuntimeContext} import pl.touk.nussknacker.engine.api.test.{TestRecord, TestRecordParser} -import pl.touk.nussknacker.engine.api.typed.typing.Typed import pl.touk.nussknacker.engine.flink.api.exception.ExceptionHandler import pl.touk.nussknacker.engine.flink.api.process.{ FlinkCustomNodeContext, @@ -113,7 +112,7 @@ class FlinkKafkaSource[T]( consumerGroupId: String, flinkNodeContext: FlinkCustomNodeContext ): SourceFunction[T] = { - // TODO: handle deployment parameters -> offset + // TODO: use deployment parameters -> offsetResetStrategy val offsetResetStrategy = flinkNodeContext.nodeDeploymentData.flatMap(_.get(OFFSET_RESET_STRATEGY_PARAM_NAME)).getOrElse() topics.toList.foreach(KafkaUtils.setToLatestOffsetIfNeeded(kafkaConfig, _, consumerGroupId)) diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala index 4cce041c505..b457a994ce1 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala @@ -5,9 +5,8 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import pl.touk.nussknacker.engine.api.component.{ParameterConfig, UnboundedStreamComponent} import pl.touk.nussknacker.engine.api.definition.{FixedExpressionValue, FixedValuesParameterEditor, RawParameterEditor} import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName -import pl.touk.nussknacker.engine.api.parameter.ParameterName import pl.touk.nussknacker.engine.api.process.{SourceFactory, WithActivityParameters} -import pl.touk.nussknacker.engine.api.typed.typing.{Typed, Unknown} +import pl.touk.nussknacker.engine.api.typed.typing.Unknown import pl.touk.nussknacker.engine.api.{MethodToInvoke, ParamName} import pl.touk.nussknacker.engine.flink.api.process.FlinkCustomNodeContext import pl.touk.nussknacker.engine.flink.util.source.CollectionSource @@ -49,6 +48,7 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone "Set offset to setup source to emit elements from specified start point in input collection. Empty field resets collection to the beginning." ) ), + // TODO: remove offsetResetStrategy "offsetResetStrategy" -> ParameterConfig( defaultValue = Some("EARLIEST"), editor = fixedValuesEditor, From 3abc32bc28da5f2b6e3830eaa0a4e36da34fd103 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Thu, 26 Sep 2024 01:04:30 +0200 Subject: [PATCH 12/20] cleanup --- designer/client/src/actions/nk/process.ts | 2 +- .../modals/GenericAction/useActivityCapabilities.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/designer/client/src/actions/nk/process.ts b/designer/client/src/actions/nk/process.ts index 7b3aa127f61..5f4beff86c1 100644 --- a/designer/client/src/actions/nk/process.ts +++ b/designer/client/src/actions/nk/process.ts @@ -37,7 +37,7 @@ export function loadProcessState(processName: ProcessName): ThunkAction { ); } -export function fetchActivityFormParameters(processName: ProcessName, scenarioGraph: ScenarioGraph) { +export function fetchActivityParameters(processName: ProcessName, scenarioGraph: ScenarioGraph) { return (dispatch) => HttpService.getActivityParameters(processName, scenarioGraph).then(({ data }) => { dispatch({ diff --git a/designer/client/src/components/modals/GenericAction/useActivityCapabilities.tsx b/designer/client/src/components/modals/GenericAction/useActivityCapabilities.tsx index 591974388af..632f900244c 100644 --- a/designer/client/src/components/modals/GenericAction/useActivityCapabilities.tsx +++ b/designer/client/src/components/modals/GenericAction/useActivityCapabilities.tsx @@ -1,7 +1,7 @@ import { useDispatch, useSelector } from "react-redux"; import { getProcessName, getScenarioGraph, isProcessRenamed } from "../../../reducers/selectors/graph"; import { useEffect } from "react"; -import { fetchActivityFormParameters } from "../../../actions/nk"; +import { fetchActivityParameters } from "../../../actions/nk"; export function useActivityCapabilities() { const dispatch = useDispatch(); @@ -12,6 +12,6 @@ export function useActivityCapabilities() { useEffect(() => { if (isRenamed) return; - dispatch(fetchActivityFormParameters(scenarioName, scenarioGraph)); + dispatch(fetchActivityParameters(scenarioName, scenarioGraph)); }, [dispatch, isRenamed, scenarioName, scenarioGraph]); } From d7fac0fb8663b537dbd7bc7f6839116e63694232 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Mon, 30 Sep 2024 12:17:10 +0200 Subject: [PATCH 13/20] demo node with different params (deleteme afterwards) --- .../sample/DevProcessConfigCreator.scala | 13 ++++---- .../sample/source/BoundedSource.scala | 30 ++++++++++++++++++- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala index 21a1772d617..9c7326a3164 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala @@ -91,12 +91,13 @@ class DevProcessConfigCreator extends ProcessConfigCreator { )(TypeInformation.of(classOf[SampleProduct])) ) ), - "kafka-transaction" -> all(SourceFactory.noParamUnboundedStreamFactory[String](new NoEndingSource)), - "boundedSource" -> all(BoundedSource), - "boundedSourceWithOffset" -> all(BoundedSourceWithOffset), - "oneSource" -> categories(SourceFactory.noParamUnboundedStreamFactory[String](new OneSource)), - "communicationSource" -> categories(DynamicParametersSource), - "csv-source" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new CsvSource)), + "kafka-transaction" -> all(SourceFactory.noParamUnboundedStreamFactory[String](new NoEndingSource)), + "boundedSource" -> all(BoundedSource), + "boundedSourceWithOffset" -> all(BoundedSourceWithOffset), + "boundedSourceWithOtherParam" -> all(DummyBoundedSourceToDelete), + "oneSource" -> categories(SourceFactory.noParamUnboundedStreamFactory[String](new OneSource)), + "communicationSource" -> categories(DynamicParametersSource), + "csv-source" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new CsvSource)), "csv-source-lite" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new LiteCsvSource(_))), "genericSourceWithCustomVariables" -> categories(GenericSourceWithCustomVariablesSample), "sql-source" -> categories(SqlSource), diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala index b457a994ce1..70158c0b88c 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala @@ -3,7 +3,12 @@ package pl.touk.nussknacker.engine.management.sample.source import org.apache.flink.streaming.api.datastream.DataStreamSource import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import pl.touk.nussknacker.engine.api.component.{ParameterConfig, UnboundedStreamComponent} -import pl.touk.nussknacker.engine.api.definition.{FixedExpressionValue, FixedValuesParameterEditor, RawParameterEditor} +import pl.touk.nussknacker.engine.api.definition.{ + BoolParameterEditor, + FixedExpressionValue, + FixedValuesParameterEditor, + RawParameterEditor +} import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.process.{SourceFactory, WithActivityParameters} import pl.touk.nussknacker.engine.api.typed.typing.Unknown @@ -76,3 +81,26 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone } } + +object DummyBoundedSourceToDelete extends SourceFactory with UnboundedStreamComponent { + + @MethodToInvoke + def source(@ParamName("elements") elements: java.util.List[Any]) = + new CollectionSource[Any](elements.asScala.toList, None, Unknown) with WithActivityParameters { + + override def activityParametersDefinition: Map[String, Map[String, ParameterConfig]] = + Map( + ScenarioActionName.Deploy.value -> Map( + "otherParameter" -> ParameterConfig( + defaultValue = None, + editor = Some(BoolParameterEditor), + validators = None, + label = Some("this is label"), + hintText = Some("this is hint") + ) + ) + ) + + } + +} From 8a0b15f0c1e2c0de8ebc252d364945dbdc7b3071 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Mon, 30 Sep 2024 21:01:35 +0200 Subject: [PATCH 14/20] upd labels --- .../engine/management/sample/source/BoundedSource.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala index 70158c0b88c..ecd083b9ab5 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala @@ -48,7 +48,7 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone defaultValue = None, editor = Some(RawParameterEditor), validators = None, - label = None, + label = Some("Offset"), hintText = Some( "Set offset to setup source to emit elements from specified start point in input collection. Empty field resets collection to the beginning." ) @@ -58,7 +58,7 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone defaultValue = Some("EARLIEST"), editor = fixedValuesEditor, validators = None, - label = None, + label = Some("Starting point strategy"), hintText = Some("Example of parameter with fixed values") ), ) @@ -95,7 +95,7 @@ object DummyBoundedSourceToDelete extends SourceFactory with UnboundedStreamComp defaultValue = None, editor = Some(BoolParameterEditor), validators = None, - label = Some("this is label"), + label = Some("Other parameter"), hintText = Some("this is hint") ) ) From 68932ac6298ad369b30d40b6ac0f17baf152b5fa Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Tue, 1 Oct 2024 14:02:46 +0200 Subject: [PATCH 15/20] new form with working deploy parameters --- .../modals/ActivityCommentTextField.tsx | 21 +++++ .../src/components/modals/ActivityHeader.tsx | 33 +++++++ .../components/modals/ActivityProperty.tsx | 49 ++++++++++ .../modals/AdvancedParametersSection.tsx | 26 ++++++ .../CalculateCounts/CalculateCountsDialog.tsx | 2 +- .../components/modals/CustomActionDialog.tsx | 28 +++--- .../components/modals/DeployProcessDialog.tsx | 89 +++++++++++++------ .../modals/GenerateTestDataDialog.tsx | 2 +- .../components/modals/SaveProcessDialog.tsx | 18 ++-- .../toolbars/process/buttons/SaveButton.tsx | 4 +- .../buttons/CancelDeployButton.tsx | 4 +- .../scenarioActions/buttons/DeployButton.tsx | 6 +- designer/client/src/types/activity.ts | 6 +- .../src/windowManager/PromptContent.tsx | 8 +- .../src/universal/conf/dev-application.conf | 4 +- 15 files changed, 231 insertions(+), 69 deletions(-) create mode 100644 designer/client/src/components/modals/ActivityCommentTextField.tsx create mode 100644 designer/client/src/components/modals/ActivityHeader.tsx create mode 100644 designer/client/src/components/modals/ActivityProperty.tsx create mode 100644 designer/client/src/components/modals/AdvancedParametersSection.tsx diff --git a/designer/client/src/components/modals/ActivityCommentTextField.tsx b/designer/client/src/components/modals/ActivityCommentTextField.tsx new file mode 100644 index 00000000000..256808dc198 --- /dev/null +++ b/designer/client/src/components/modals/ActivityCommentTextField.tsx @@ -0,0 +1,21 @@ +import { styled, TextField, TextFieldProps } from "@mui/material"; +import React from "react"; + +export const ActivityCommentTextField = styled((props: TextFieldProps) => ( + +))({ + flexDirection: "column", + ".MuiFormLabel-root": { + margin: 0, + flexDirection: "column", + }, +}); diff --git a/designer/client/src/components/modals/ActivityHeader.tsx b/designer/client/src/components/modals/ActivityHeader.tsx new file mode 100644 index 00000000000..4c02dcf8f43 --- /dev/null +++ b/designer/client/src/components/modals/ActivityHeader.tsx @@ -0,0 +1,33 @@ +import { useSelector } from "react-redux"; +import { getProcessName } from "../../reducers/selectors/graph"; +import { Box, Typography } from "@mui/material"; +import React from "react"; +import ProcessDialogWarnings from "./ProcessDialogWarnings"; + +interface Props { + title: string; + displayWarnings?: boolean; +} + +export function ActivityHeader(props: Props): JSX.Element { + const processName = useSelector(getProcessName); + return ( + + + {props.title} + + + {processName} + + {props.displayWarnings && } + + ); +} diff --git a/designer/client/src/components/modals/ActivityProperty.tsx b/designer/client/src/components/modals/ActivityProperty.tsx new file mode 100644 index 00000000000..c401d025e16 --- /dev/null +++ b/designer/client/src/components/modals/ActivityProperty.tsx @@ -0,0 +1,49 @@ +import { ExpressionLang } from "../graph/node-modal/editors/expression/types"; +import React, { useCallback } from "react"; +import { FieldLabel } from "../graph/node-modal/FieldLabel"; +import { getValidationErrorsForField } from "../graph/node-modal/editors/Validators"; +import { ActivityNodeParameters, ActivityParameterConfig } from "../../types/activity"; +import { NodesDeploymentData } from "../../http/HttpService"; +import { NodeValidationError } from "../../types"; +import { default as EditableEditor } from "../graph/node-modal/editors/EditableEditor"; + +interface Props { + nodeName: string; + propertyName: string; + propertyConfig: ActivityParameterConfig; + nodesData: NodesDeploymentData; + onChange: ( + nodeId: string, + property: K, + newValue: ActivityNodeParameters["parameters"][K], + defaultValue?: ActivityNodeParameters["parameters"][K], + ) => void; + errors: NodeValidationError[]; +} + +export function ActivityProperty(props: Props): JSX.Element { + const { nodeName, propertyName, propertyConfig, errors, nodesData, onChange } = props; + + const current = nodesData[nodeName][propertyName] || ""; + const expressionObj = { expression: current, value: current, language: ExpressionLang.String }; + const onValueChange = useCallback((newValue) => onChange(nodeName, propertyName, newValue), [onChange, nodeName, propertyName]); + + return ( + ( + + )} + readOnly={false} + showSwitch={false} + showValidation={true} + //ScenarioProperties do not use any variables + variableTypes={{}} + fieldErrors={getValidationErrorsForField(errors, propertyName)} + /> + ); +} diff --git a/designer/client/src/components/modals/AdvancedParametersSection.tsx b/designer/client/src/components/modals/AdvancedParametersSection.tsx new file mode 100644 index 00000000000..74cb8897190 --- /dev/null +++ b/designer/client/src/components/modals/AdvancedParametersSection.tsx @@ -0,0 +1,26 @@ +import React, { PropsWithChildren } from "react"; +import Accordion from "@mui/material/Accordion"; +import AccordionSummary from "@mui/material/AccordionSummary"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; +import { Typography } from "@mui/material"; +import AccordionDetails from "@mui/material/AccordionDetails"; + +interface Props { + nodeId: string; +} + +export function AdvancedParametersSection({ children, nodeId }: PropsWithChildren): JSX.Element { + return ( + + } + aria-controls={`${nodeId}-content`} + id={`${nodeId}-header`} + sx={{ flexDirection: "row-reverse", px: 0, border: 0 }} + > + {nodeId} + + {children} + + ); +} diff --git a/designer/client/src/components/modals/CalculateCounts/CalculateCountsDialog.tsx b/designer/client/src/components/modals/CalculateCounts/CalculateCountsDialog.tsx index 7391493f8c5..3a2f045e048 100644 --- a/designer/client/src/components/modals/CalculateCounts/CalculateCountsDialog.tsx +++ b/designer/client/src/components/modals/CalculateCounts/CalculateCountsDialog.tsx @@ -59,7 +59,7 @@ export function CountsDialog({ children, ...props }: PropsWithChildren { await confirm(); diff --git a/designer/client/src/components/modals/CustomActionDialog.tsx b/designer/client/src/components/modals/CustomActionDialog.tsx index 679f2a0afb6..7818d1f87d4 100644 --- a/designer/client/src/components/modals/CustomActionDialog.tsx +++ b/designer/client/src/components/modals/CustomActionDialog.tsx @@ -7,7 +7,7 @@ import { loadProcessState } from "../../actions/nk"; import HttpService, { CustomActionValidationRequest } from "../../http/HttpService"; import { CustomAction, NodeValidationError } from "../../types"; import { UnknownRecord } from "../../types/common"; -import { WindowContent, WindowKind } from "../../windowManager"; +import { PromptContent, WindowKind } from "../../windowManager"; import { ChangeableValue } from "../ChangeableValue"; import { editors, ExtendedEditor, SimpleEditor } from "../graph/node-modal/editors/expression/Editor"; import { ExpressionLang } from "../graph/node-modal/editors/expression/types"; @@ -18,7 +18,8 @@ import { LoadingButtonTypes } from "../../windowManager/LoadingButton"; import { nodeValue } from "../graph/node-modal/NodeDetailsContent/NodeTableStyled"; import { getValidationErrorsForField } from "../graph/node-modal/editors/Validators"; import { getFeatureSettings } from "../../reducers/selectors/settings"; -import CommentInput from "../comment/CommentInput"; +import { ActivityCommentTextField } from "./ActivityCommentTextField"; +import { ActivityHeader } from "./ActivityHeader"; interface CustomActionFormProps extends ChangeableValue { action: CustomAction; @@ -123,32 +124,25 @@ export function CustomActionDialog(props: WindowContentProps [ { title: t("dialog.button.cancel", "Cancel"), action: () => props.close(), classname: LoadingButtonTypes.secondaryButton }, - { title: t("dialog.button.confirm", "Ok"), action: () => confirmAction() }, + { title: t("dialog.button.confirm", "Apply"), action: () => confirmAction() }, ], [confirmAction, props, t], ); return ( - +
- + setComment(e.target.value)} - value={comment} - defaultValue={deploymentCommentSettings?.exampleComment} - className={cx( - css({ - minWidth: 600, - minHeight: 80, - }), - )} autoFocus /> - - {validationError} -
- + ); } diff --git a/designer/client/src/components/modals/DeployProcessDialog.tsx b/designer/client/src/components/modals/DeployProcessDialog.tsx index a9c468f2b10..2eae9482915 100644 --- a/designer/client/src/components/modals/DeployProcessDialog.tsx +++ b/designer/client/src/components/modals/DeployProcessDialog.tsx @@ -7,26 +7,43 @@ import { getActivityParameters, getProcessName } from "../../reducers/selectors/ import { getFeatureSettings } from "../../reducers/selectors/settings"; import { ProcessName } from "../Process/types"; import { PromptContent, WindowKind } from "../../windowManager"; -import CommentInput from "../comment/CommentInput"; -import ProcessDialogWarnings from "./ProcessDialogWarnings"; -import { FormHelperText, Typography } from "@mui/material"; import { LoadingButtonTypes } from "../../windowManager/LoadingButton"; +import { ActivityNodeParameters } from "../../types/activity"; +import { AdvancedParametersSection } from "./AdvancedParametersSection"; +import { mapValues } from "lodash"; +import { NodesDeploymentData } from "../../http/HttpService"; +import { ActivityProperty } from "./ActivityProperty"; +import { ActivityCommentTextField } from "./ActivityCommentTextField"; +import { ActivityHeader } from "./ActivityHeader"; +import { NodeTable } from "../graph/node-modal/NodeDetailsContent/NodeTable"; export type ToggleProcessActionModalData = { - action: (processName: ProcessName, comment: string) => Promise; + action: (processName: ProcessName, comment: string, nodeData: NodesDeploymentData) => Promise; + activityName: string; displayWarnings?: boolean; }; +function initialNodesData(params: ActivityNodeParameters[]) { + return params.reduce( + (paramObj, { nodeId, parameters }) => ({ + ...paramObj, + [nodeId]: mapValues(parameters, (value) => value.defaultValue || ""), + }), + {}, + ); +} + export function DeployProcessDialog(props: WindowContentProps): JSX.Element { // TODO: get rid of meta const { - meta: { action, displayWarnings }, + meta: { action, activityName, displayWarnings }, } = props.data; const processName = useSelector(getProcessName); const activityParameters = useSelector(getActivityParameters); - const activityNodeParameters = activityParameters["DEPLOY"] || []; - console.log(activityNodeParameters); + const activityNodeParameters = activityParameters[activityName] || ([] as ActivityNodeParameters[]); + const initialValues = useMemo(() => initialNodesData(activityNodeParameters), [activityNodeParameters]); + const [values, setValues] = useState(initialValues); const [comment, setComment] = useState(""); const [validationError, setValidationError] = useState(""); @@ -37,7 +54,7 @@ export function DeployProcessDialog(props: WindowContentProps { try { - await action(processName, comment); + await action(processName, comment, values); props.close(); } catch (error) { setValidationError(error?.response?.data); @@ -47,32 +64,54 @@ export function DeployProcessDialog(props: WindowContentProps [ - { title: t("dialog.button.cancel", "Cancel"), action: () => props.close(), classname: LoadingButtonTypes.secondaryButton }, - { title: t("dialog.button.ok", "Ok"), action: () => confirmAction() }, + { + title: t("dialog.button.cancel", "Cancel"), + action: () => props.close(), + classname: LoadingButtonTypes.secondaryButton, + }, + { title: t("dialog.button.ok", "Apply"), action: () => confirmAction() }, ], [confirmAction, props, t], ); return ( -
- {props.data.title} - {displayWarnings && } - + + setComment(e.target.value)} - value={comment} - defaultValue={deploymentCommentSettings?.exampleComment} - className={cx( - css({ - minWidth: 600, - minHeight: 80, - }), - )} autoFocus /> - - {validationError} - + {activityNodeParameters.map((anp: ActivityNodeParameters) => ( + + + {Object.entries(anp.parameters).map(([paramName, paramConfig]) => { + return ( + { + setValues({ + ...values, + [nodeId]: { + ...values[nodeId], + [paramName]: newValue, + }, + }); + }} + nodesData={values} + /> + ); + })} + + + ))}
); diff --git a/designer/client/src/components/modals/GenerateTestDataDialog.tsx b/designer/client/src/components/modals/GenerateTestDataDialog.tsx index 03d80ea3905..a9ae5ab792e 100644 --- a/designer/client/src/components/modals/GenerateTestDataDialog.tsx +++ b/designer/client/src/components/modals/GenerateTestDataDialog.tsx @@ -46,7 +46,7 @@ function GenerateTestDataDialog(props: WindowContentProps): JSX.Element { const buttons: WindowButtonProps[] = useMemo( () => [ { title: t("dialog.button.cancel", "Cancel"), action: () => props.close(), classname: LoadingButtonTypes.secondaryButton }, - { title: t("dialog.button.ok", "Ok"), disabled: !isValid, action: () => confirmAction() }, + { title: t("dialog.button.ok", "Apply"), disabled: !isValid, action: () => confirmAction() }, ], [t, confirmAction, props, isValid], ); diff --git a/designer/client/src/components/modals/SaveProcessDialog.tsx b/designer/client/src/components/modals/SaveProcessDialog.tsx index d77f2222c57..8853392368e 100644 --- a/designer/client/src/components/modals/SaveProcessDialog.tsx +++ b/designer/client/src/components/modals/SaveProcessDialog.tsx @@ -5,7 +5,6 @@ import { useTranslation } from "react-i18next"; import { useDispatch } from "react-redux"; import { displayCurrentProcessVersion, displayProcessActivity, loadProcessToolbarsConfiguration } from "../../actions/nk"; import { PromptContent } from "../../windowManager"; -import { CommentInput } from "../comment/CommentInput"; import { ThunkAction } from "../../actions/reduxTypes"; import { getScenarioGraph, @@ -18,8 +17,9 @@ import HttpService from "../../http/HttpService"; import { ActionCreators as UndoActionCreators } from "redux-undo"; import { visualizationUrl } from "../../common/VisualizationUrl"; import { useLocation, useNavigate } from "react-router-dom"; -import { Typography } from "@mui/material"; import { LoadingButtonTypes } from "../../windowManager/LoadingButton"; +import { ActivityCommentTextField } from "./ActivityCommentTextField"; +import { ActivityHeader } from "./ActivityHeader"; export function SaveProcessDialog(props: WindowContentProps): JSX.Element { const location = useLocation(); @@ -70,7 +70,7 @@ export function SaveProcessDialog(props: WindowContentProps): JSX.Element { const buttons: WindowButtonProps[] = useMemo( () => [ { title: t("dialog.button.cancel", "Cancel"), action: () => props.close(), classname: LoadingButtonTypes.secondaryButton }, - { title: t("dialog.button.ok", "Ok"), action: () => confirmAction() }, + { title: t("dialog.button.ok", "Apply"), action: () => confirmAction() }, ], [confirmAction, props, t], ); @@ -78,16 +78,8 @@ export function SaveProcessDialog(props: WindowContentProps): JSX.Element { return (
- {props.data.title} - setState(e.target.value)} - value={comment} - className={css({ - minWidth: 600, - minHeight: 80, - })} - autoFocus - /> + + setState(e.target.value)} autoFocus />
); diff --git a/designer/client/src/components/toolbars/process/buttons/SaveButton.tsx b/designer/client/src/components/toolbars/process/buttons/SaveButton.tsx index 44e25d994ee..19812e0277a 100644 --- a/designer/client/src/components/toolbars/process/buttons/SaveButton.tsx +++ b/designer/client/src/components/toolbars/process/buttons/SaveButton.tsx @@ -17,8 +17,8 @@ function SaveButton(props: ToolbarButtonProps): JSX.Element { const unsavedNewName = useSelector(getProcessUnsavedNewName); const isRenamed = useSelector(isProcessRenamed); const title = isRenamed - ? t("saveProcess.renameTitle", "Save scenario as {{name}}", { name: unsavedNewName }) - : t("saveProcess.title", "Save scenario {{name}}", { name: processName }); + ? t("saveProcess.renameTitle", "Save scenario as", { name: unsavedNewName }) + : t("saveProcess.title", "Save scenario", { name: processName }); const { open } = useWindows(); const onClick = () => diff --git a/designer/client/src/components/toolbars/scenarioActions/buttons/CancelDeployButton.tsx b/designer/client/src/components/toolbars/scenarioActions/buttons/CancelDeployButton.tsx index cc14a764dcb..1bb36088e26 100644 --- a/designer/client/src/components/toolbars/scenarioActions/buttons/CancelDeployButton.tsx +++ b/designer/client/src/components/toolbars/scenarioActions/buttons/CancelDeployButton.tsx @@ -23,7 +23,7 @@ export default function CancelDeployButton(props: ToolbarButtonProps) { const { open } = useWindows(); const action = (p, c) => HttpService.cancel(p, c).finally(() => dispatch(loadProcessState(processName))); - const message = t("panels.actions.deploy-canel.dialog", "Cancel scenario {{name}}", { name: processName }); + const message = t("panels.actions.deploy-canel.dialog", "Cancel scenario"); return ( HttpService.deploy(p, c).finally(() => dispatch(loadProcessState(processName))); + const message = t("panels.actions.deploy.dialog", "Deploy scenario"); + const action = (p, c, d) => HttpService.deploy(p, c, d).finally(() => dispatch(loadProcessState(processName))); return ( ; + parameters: { [key: ActivityParameterName]: ActivityParameterConfig }; } export type ActivityName = string; -export type ActivityParameters = Record; +export type ActivityParameters = { [key: ActivityName]: ActivityNodeParameters[] }; diff --git a/designer/client/src/windowManager/PromptContent.tsx b/designer/client/src/windowManager/PromptContent.tsx index 37a76f38740..785d9d6137d 100644 --- a/designer/client/src/windowManager/PromptContent.tsx +++ b/designer/client/src/windowManager/PromptContent.tsx @@ -16,7 +16,13 @@ export function PromptContent(props: PropsWithChildren): JS paddingLeft: theme.custom.spacing.baseUnit * 6, paddingRight: theme.custom.spacing.baseUnit * 6, }); - return { ...props.classnames, content }; + return { + footer: css({ + justifyContent: "flex-end", + }), + ...props.classnames, + content, + }; }, [props.classnames, theme.custom.spacing.baseUnit]); const components = useMemo( diff --git a/nussknacker-dist/src/universal/conf/dev-application.conf b/nussknacker-dist/src/universal/conf/dev-application.conf index ddf3c4c27aa..3082ff0a08c 100644 --- a/nussknacker-dist/src/universal/conf/dev-application.conf +++ b/nussknacker-dist/src/universal/conf/dev-application.conf @@ -359,8 +359,8 @@ commentSettings: { } deploymentCommentSettings: { - validationPattern: "(.*)" - exampleComment: "issues/1234" + validationPattern: "issues/\\d+" + exampleComment: "issues/1234 Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec ultricies nec sapien id bibendum. Ut in mollis risus. Curabitur efficitur maximus interdum. Vivamus convallis eu nibh ut rhoncus. Quisque finibus maximus dui vel finibus." } countsSettings { From 69b8ade6c312a5ae26e44caae39a11fecd064c04 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Wed, 2 Oct 2024 10:28:50 +0200 Subject: [PATCH 16/20] minor styling --- designer/client/src/windowManager/PromptContent.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/designer/client/src/windowManager/PromptContent.tsx b/designer/client/src/windowManager/PromptContent.tsx index 785d9d6137d..511f5adfc08 100644 --- a/designer/client/src/windowManager/PromptContent.tsx +++ b/designer/client/src/windowManager/PromptContent.tsx @@ -19,6 +19,7 @@ export function PromptContent(props: PropsWithChildren): JS return { footer: css({ justifyContent: "flex-end", + backgroundColor: "#1D2734", }), ...props.classnames, content, From b5a133a6d8b2ce6ae7a019d2971c5e249c7f7c17 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Wed, 2 Oct 2024 14:19:34 +0200 Subject: [PATCH 17/20] two modes for fixed list --- .../api/definition/FixedExpressionValue.scala | 5 ++-- .../api/editor/FixedValuesEditorMode.java | 15 ++++++++++ .../api/definition/ParameterEditor.scala | 19 ++++++++++-- .../api/description/NodesApiEndpoints.scala | 3 +- .../aggregate/AggregateHelper.java | 29 +++++++++++-------- .../sample/source/BoundedSource.scala | 12 ++++---- .../EditorBasedLanguageDeterminer.scala | 2 +- ...bleValuesBasedDefaultValueDeterminer.scala | 6 ++-- .../EditorBasedValidatorExtractor.scala | 2 +- 9 files changed, 66 insertions(+), 27 deletions(-) create mode 100644 components-api/src/main/java/pl/touk/nussknacker/engine/api/editor/FixedValuesEditorMode.java diff --git a/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala b/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala index dbfce58b797..6f999e1f8bf 100644 --- a/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala +++ b/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala @@ -2,8 +2,9 @@ package pl.touk.nussknacker.engine.api.definition import io.circe.generic.JsonCodec -@JsonCodec case class FixedExpressionValue(expression: String, label: String) +@JsonCodec case class FixedExpressionValue(expression: String, label: String, hint: Option[String] = None) object FixedExpressionValue { - val nullFixedValue: FixedExpressionValue = FixedExpressionValue("", "") + def apply(expression: String, label: String): FixedExpressionValue = FixedExpressionValue(expression, label, None) + val nullFixedValue: FixedExpressionValue = FixedExpressionValue("", "") } diff --git a/components-api/src/main/java/pl/touk/nussknacker/engine/api/editor/FixedValuesEditorMode.java b/components-api/src/main/java/pl/touk/nussknacker/engine/api/editor/FixedValuesEditorMode.java new file mode 100644 index 00000000000..99ab6778589 --- /dev/null +++ b/components-api/src/main/java/pl/touk/nussknacker/engine/api/editor/FixedValuesEditorMode.java @@ -0,0 +1,15 @@ +package pl.touk.nussknacker.engine.api.editor; + +public enum FixedValuesEditorMode { + LIST, RADIO; + + public static FixedValuesEditorMode fromName(String name) { + switch (name) { + case "LIST": + return LIST; + case "RADIO": + default: + return RADIO; + } + } +} diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala index 83cc7259019..7c0a641defc 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala @@ -4,6 +4,7 @@ import io.circe.generic.JsonCodec import io.circe.generic.extras.ConfiguredJsonCodec import io.circe.{Decoder, Encoder, Json} import pl.touk.nussknacker.engine.api.CirceUtil._ +import pl.touk.nussknacker.engine.api.editor.FixedValuesEditorMode import pl.touk.nussknacker.engine.api.editor.DualEditorMode import java.time.temporal.ChronoUnit @@ -73,8 +74,10 @@ object PeriodParameterEditor { */ case object CronParameterEditor extends SimpleParameterEditor -@JsonCodec case class FixedValuesParameterEditor(possibleValues: List[FixedExpressionValue]) - extends SimpleParameterEditor +@JsonCodec case class FixedValuesParameterEditor( + possibleValues: List[FixedExpressionValue], + mode: FixedValuesEditorMode = FixedValuesEditorMode.LIST +) extends SimpleParameterEditor @JsonCodec case class FixedValuesWithIconParameterEditor(possibleValues: List[FixedExpressionValueWithIcon]) extends SimpleParameterEditor @@ -100,3 +103,15 @@ object DualParameterEditor { } } + +object FixedValuesParameterEditor { + def apply(possibleValues: List[FixedExpressionValue]): FixedValuesParameterEditor = + FixedValuesParameterEditor(possibleValues, mode = FixedValuesEditorMode.LIST) + + implicit val fixedValuesEditorModeEncoder: Encoder[FixedValuesEditorMode] = new Encoder[FixedValuesEditorMode] { + override def apply(a: FixedValuesEditorMode): Json = Encoder.encodeString(a.name()) + } + + implicit val fixedValuesEditorModeDecoder: Decoder[FixedValuesEditorMode] = + Decoder.decodeString.emapTry(name => Try(FixedValuesEditorMode.fromName(name))) +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala index c00e18d7ed9..eb401d122f6 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala @@ -18,7 +18,7 @@ import pl.touk.nussknacker.engine.api.definition.{ ParameterEditor, SimpleParameterEditor } -import pl.touk.nussknacker.engine.api.editor.DualEditorMode +import pl.touk.nussknacker.engine.api.editor.{DualEditorMode, FixedValuesEditorMode} import pl.touk.nussknacker.engine.api.generics.ExpressionParseError.{CellError, ColumnDefinition, ErrorDetails} import pl.touk.nussknacker.engine.api.graph.{Edge, ProcessProperties, ScenarioGraph} import pl.touk.nussknacker.engine.api.parameter.{ @@ -1299,6 +1299,7 @@ object NodesApiEndpoints { implicit lazy val simpleParameterEditorSchema: Schema[SimpleParameterEditor] = Schema.derived implicit lazy val parameterEditorSchema: Schema[ParameterEditor] = Schema.derived implicit lazy val dualEditorSchema: Schema[DualEditorMode] = Schema.string + implicit lazy val fixedValuesEditorMode: Schema[FixedValuesEditorMode] = Schema.string implicit lazy val durationSchema: Schema[Duration] = Schema.schemaForJavaDuration implicit lazy val uiParameterSchema: Schema[UIParameter] = Schema.derived diff --git a/engine/flink/components/base-unbounded/src/main/java/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/AggregateHelper.java b/engine/flink/components/base-unbounded/src/main/java/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/AggregateHelper.java index a6329c746e5..f81ee47c769 100644 --- a/engine/flink/components/base-unbounded/src/main/java/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/AggregateHelper.java +++ b/engine/flink/components/base-unbounded/src/main/java/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/AggregateHelper.java @@ -6,10 +6,11 @@ import pl.touk.nussknacker.engine.api.Hidden; import pl.touk.nussknacker.engine.api.ParamName; import pl.touk.nussknacker.engine.api.definition.DualParameterEditor; -import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue; +import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue$; import pl.touk.nussknacker.engine.api.definition.FixedValuesParameterEditor; import pl.touk.nussknacker.engine.api.definition.SimpleParameterEditor; import pl.touk.nussknacker.engine.api.editor.DualEditorMode; +import pl.touk.nussknacker.engine.api.editor.FixedValuesEditorMode; import scala.collection.JavaConverters; /** @@ -22,17 +23,21 @@ */ public class AggregateHelper implements Serializable { - public static final SimpleParameterEditor SIMPLE_EDITOR = new FixedValuesParameterEditor(JavaConverters.collectionAsScalaIterableConverter(Arrays.asList( - new FixedExpressionValue("#AGG.first", "First"), - new FixedExpressionValue("#AGG.last", "Last"), - new FixedExpressionValue("#AGG.countWhen", "CountWhen"), - new FixedExpressionValue("#AGG.average", "Average"), - new FixedExpressionValue("#AGG.min", "Min"), - new FixedExpressionValue("#AGG.max", "Max"), - new FixedExpressionValue("#AGG.sum", "Sum"), - new FixedExpressionValue("#AGG.list", "List"), - new FixedExpressionValue("#AGG.set", "Set"), - new FixedExpressionValue("#AGG.approxCardinality", "ApproximateSetCardinality"))).asScala().toList()); + public static final SimpleParameterEditor SIMPLE_EDITOR = new FixedValuesParameterEditor( + JavaConverters.collectionAsScalaIterableConverter(Arrays.asList( + FixedExpressionValue$.MODULE$.apply("#AGG.first", "First"), + FixedExpressionValue$.MODULE$.apply("#AGG.last", "Last"), + FixedExpressionValue$.MODULE$.apply("#AGG.countWhen", "CountWhen"), + FixedExpressionValue$.MODULE$.apply("#AGG.average", "Average"), + FixedExpressionValue$.MODULE$.apply("#AGG.min", "Min"), + FixedExpressionValue$.MODULE$.apply("#AGG.max", "Max"), + FixedExpressionValue$.MODULE$.apply("#AGG.sum", "Sum"), + FixedExpressionValue$.MODULE$.apply("#AGG.list", "List"), + FixedExpressionValue$.MODULE$.apply("#AGG.set", "Set"), + FixedExpressionValue$.MODULE$.apply("#AGG.approxCardinality", "ApproximateSetCardinality") + )).asScala().toList(), + FixedValuesEditorMode.LIST + ); @Hidden public static final DualParameterEditor DUAL_EDITOR = new DualParameterEditor(SIMPLE_EDITOR, DualEditorMode.SIMPLE); diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala index ecd083b9ab5..b225fd2be75 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala @@ -10,6 +10,7 @@ import pl.touk.nussknacker.engine.api.definition.{ RawParameterEditor } import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName +import pl.touk.nussknacker.engine.api.editor.FixedValuesEditorMode import pl.touk.nussknacker.engine.api.process.{SourceFactory, WithActivityParameters} import pl.touk.nussknacker.engine.api.typed.typing.Unknown import pl.touk.nussknacker.engine.api.{MethodToInvoke, ParamName} @@ -36,10 +37,11 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone val fixedValuesEditor = Some( FixedValuesParameterEditor( List( - FixedExpressionValue("LATEST", "LATEST"), - FixedExpressionValue("EARLIEST", "EARLIEST"), - FixedExpressionValue("NONE", "NONE"), - ) + FixedExpressionValue("Continue", "Continue", Some("Resumes reading data where it previously stopped.")), + FixedExpressionValue("Reset", "Reset", Some("Starts reading new events only.")), + FixedExpressionValue("Restart", "Restart", Some("Rewinds reading from the earliest event.")), + ), + FixedValuesEditorMode.RADIO ) ) Map( @@ -55,7 +57,7 @@ object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamCompone ), // TODO: remove offsetResetStrategy "offsetResetStrategy" -> ParameterConfig( - defaultValue = Some("EARLIEST"), + defaultValue = Some("Restart"), editor = fixedValuesEditor, validators = None, label = Some("Starting point strategy"), diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorBasedLanguageDeterminer.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorBasedLanguageDeterminer.scala index ae97173b9a6..c5b81c6f965 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorBasedLanguageDeterminer.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorBasedLanguageDeterminer.scala @@ -16,7 +16,7 @@ object EditorBasedLanguageDeterminer { editor match { case BoolParameterEditor | StringParameterEditor | DateParameterEditor | TimeParameterEditor | DateTimeParameterEditor | TextareaParameterEditor | JsonParameterEditor | DurationParameterEditor(_) | - PeriodParameterEditor(_) | CronParameterEditor | FixedValuesParameterEditor(_) | + PeriodParameterEditor(_) | CronParameterEditor | FixedValuesParameterEditor(_, _) | FixedValuesWithIconParameterEditor(_) => Expression.Language.Spel case SqlParameterEditor | SpelTemplateParameterEditor => diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorPossibleValuesBasedDefaultValueDeterminer.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorPossibleValuesBasedDefaultValueDeterminer.scala index 9add251b317..b00c2f00d95 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorPossibleValuesBasedDefaultValueDeterminer.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorPossibleValuesBasedDefaultValueDeterminer.scala @@ -14,10 +14,10 @@ protected object EditorPossibleValuesBasedDefaultValueDeterminer extends Paramet override def determineParameterDefaultValue(parameters: DefaultValueDeterminerParameters): Option[Expression] = { parameters.determinedEditor .flatMap { - case FixedValuesParameterEditor(firstValue :: _) => Some(Expression.spel(firstValue.expression)) + case FixedValuesParameterEditor(firstValue :: _, _) => Some(Expression.spel(firstValue.expression)) // it is better to see error that field is not filled instead of strange default value like '' for String - case FixedValuesParameterEditor(Nil) => Some(Expression.spel("")) - case DualParameterEditor(FixedValuesParameterEditor(firstValue :: _), _) => + case FixedValuesParameterEditor(Nil, _) => Some(Expression.spel("")) + case DualParameterEditor(FixedValuesParameterEditor(firstValue :: _, _), _) => Some(Expression.spel(firstValue.expression)) case TabularTypedDataEditor => Some(Expression.tabularDataDefinition(TabularTypedData.empty.stringify)) diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/validator/EditorBasedValidatorExtractor.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/validator/EditorBasedValidatorExtractor.scala index 8a7230026b5..b1096f613b1 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/validator/EditorBasedValidatorExtractor.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/validator/EditorBasedValidatorExtractor.scala @@ -5,7 +5,7 @@ import pl.touk.nussknacker.engine.api.definition._ object EditorBasedValidatorExtractor extends ValidatorExtractor { override def extract(params: ValidatorExtractorParameters): Option[ParameterValidator] = { - params.extractedEditor.collect { case FixedValuesParameterEditor(possibleValues) => + params.extractedEditor.collect { case FixedValuesParameterEditor(possibleValues, _) => FixedValuesValidator(possibleValues) } } From a0686ee95f3a318c176333ab808c298503f7c4ff Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Wed, 2 Oct 2024 18:40:25 +0200 Subject: [PATCH 18/20] two modes for fixed list fixed decoder --- .../api/definition/ParameterEditor.scala | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala index 7c0a641defc..4170766d0d3 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala @@ -2,7 +2,8 @@ package pl.touk.nussknacker.engine.api.definition import io.circe.generic.JsonCodec import io.circe.generic.extras.ConfiguredJsonCodec -import io.circe.{Decoder, Encoder, Json} +import io.circe.generic.semiauto.deriveEncoder +import io.circe.{Decoder, Encoder, HCursor, Json} import pl.touk.nussknacker.engine.api.CirceUtil._ import pl.touk.nussknacker.engine.api.editor.FixedValuesEditorMode import pl.touk.nussknacker.engine.api.editor.DualEditorMode @@ -74,7 +75,7 @@ object PeriodParameterEditor { */ case object CronParameterEditor extends SimpleParameterEditor -@JsonCodec case class FixedValuesParameterEditor( +case class FixedValuesParameterEditor( possibleValues: List[FixedExpressionValue], mode: FixedValuesEditorMode = FixedValuesEditorMode.LIST ) extends SimpleParameterEditor @@ -114,4 +115,22 @@ object FixedValuesParameterEditor { implicit val fixedValuesEditorModeDecoder: Decoder[FixedValuesEditorMode] = Decoder.decodeString.emapTry(name => Try(FixedValuesEditorMode.fromName(name))) + + implicit val fixedValuesParameterEditorEncoder: Encoder[FixedValuesParameterEditor] = + deriveEncoder[FixedValuesParameterEditor] + + implicit val fixedValuesParameterEditorDecoder: Decoder[FixedValuesParameterEditor] = { (c: HCursor) => + { + for { + possibleValues <- c.downField("possibleValues").as[List[FixedExpressionValue]] + modeOpt <- c.downField("mode").as[Option[String]] + } yield { + FixedValuesParameterEditor( + possibleValues, + modeOpt.map(FixedValuesEditorMode.fromName).getOrElse(FixedValuesEditorMode.LIST) + ) + } + } + } + } From 178a56fa9e75fed2961d52c477d2f15d046d1fed Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Wed, 2 Oct 2024 22:49:42 +0200 Subject: [PATCH 19/20] hint text for fixed value option --- .../api/definition/FixedExpressionValue.scala | 2 +- .../editors/expression/FixedValuesEditor.tsx | 24 ++++++++++++++++--- .../fragment-input-definition/item/types.ts | 1 + 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala b/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala index 6f999e1f8bf..a595e8b88ea 100644 --- a/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala +++ b/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala @@ -2,7 +2,7 @@ package pl.touk.nussknacker.engine.api.definition import io.circe.generic.JsonCodec -@JsonCodec case class FixedExpressionValue(expression: String, label: String, hint: Option[String] = None) +@JsonCodec case class FixedExpressionValue(expression: String, label: String, hintText: Option[String] = None) object FixedExpressionValue { def apply(expression: String, label: String): FixedExpressionValue = FixedExpressionValue(expression, label, None) diff --git a/designer/client/src/components/graph/node-modal/editors/expression/FixedValuesEditor.tsx b/designer/client/src/components/graph/node-modal/editors/expression/FixedValuesEditor.tsx index 1499b50cee6..e8e98113f33 100644 --- a/designer/client/src/components/graph/node-modal/editors/expression/FixedValuesEditor.tsx +++ b/designer/client/src/components/graph/node-modal/editors/expression/FixedValuesEditor.tsx @@ -5,7 +5,7 @@ import { ExpressionObj } from "./types"; import { isEmpty } from "lodash"; import { cx } from "@emotion/css"; import { selectStyled } from "../../../../../stylesheets/SelectStyled"; -import { Stack, styled, Typography, useTheme } from "@mui/material"; +import { FormControlLabel, Radio, RadioGroup, Stack, styled, Typography, useTheme } from "@mui/material"; import { ExtendedEditor } from "./Editor"; import { FieldError } from "../Validators"; import { FixedValuesOption } from "../../fragment-input-definition/item"; @@ -26,6 +26,7 @@ interface Option { label: string; value: string; icon: string | null; + hintText: string | null; } function getOptions(values: FixedValuesOption[]): Option[] { @@ -33,19 +34,26 @@ function getOptions(values: FixedValuesOption[]): Option[] { value: value.expression, label: value.label, icon: value.icon, + hintText: value.hintText, })); } +enum FixedValuesEditorMode { + LIST = "LIST", + RADIO = "RADIO", +} + export const FixedValuesEditor: ExtendedEditor = (props: Props) => { const handleCurrentOption = (expressionObj: ExpressionObj, options: Option[]): Option => { return ( (expressionObj && options.find((option) => option.value === expressionObj.expression)) || // current value with label taken from options - (expressionObj && { value: expressionObj.expression, label: expressionObj.expression, icon: null }) || // current value is no longer valid option? Show it anyway, let user know. Validation should take care + (expressionObj && { value: expressionObj.expression, label: expressionObj.expression, icon: null, hintText: null }) || // current value is no longer valid option? Show it anyway, let user know. Validation should take care null ); // just leave undefined and let the user explicitly select one }; const { expressionObj, readOnly, onValueChange, className, showValidation, editorConfig, fieldErrors } = props; + const mode = FixedValuesEditorMode[editorConfig.mode || "LIST"]; const options = getOptions(editorConfig.possibleValues); const currentOption = handleCurrentOption(expressionObj, options); const theme = useTheme(); @@ -58,7 +66,17 @@ export const FixedValuesEditor: ExtendedEditor = (props: Props) => { const { control, input, valueContainer, singleValue, menuPortal, menu, menuList, menuOption, indicatorSeparator, dropdownIndicator } = selectStyled(theme); - return ( + return mode == FixedValuesEditorMode.RADIO ? ( +
+ onValueChange(event.target.value)}> + {options.map((option: Option) => { + const label = option.value === "" ? `${option.value} (default)` : option.value; + return } label={label} />; + })} + + {currentOption.hintText ? {currentOption.hintText} : null} +
+ ) : (
Date: Wed, 2 Oct 2024 22:51:37 +0200 Subject: [PATCH 20/20] upd --- .../src/components/modals/ActivityHeader.tsx | 21 +++++++------------ .../modals/AdvancedParametersSection.tsx | 4 ++-- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/designer/client/src/components/modals/ActivityHeader.tsx b/designer/client/src/components/modals/ActivityHeader.tsx index 4c02dcf8f43..0829d9a22e9 100644 --- a/designer/client/src/components/modals/ActivityHeader.tsx +++ b/designer/client/src/components/modals/ActivityHeader.tsx @@ -1,6 +1,6 @@ import { useSelector } from "react-redux"; import { getProcessName } from "../../reducers/selectors/graph"; -import { Box, Typography } from "@mui/material"; +import { Typography } from "@mui/material"; import React from "react"; import ProcessDialogWarnings from "./ProcessDialogWarnings"; @@ -12,22 +12,17 @@ interface Props { export function ActivityHeader(props: Props): JSX.Element { const processName = useSelector(getProcessName); return ( - - - {props.title} - + <> + {props.title} + + {processName} {props.displayWarnings && } - + ); } diff --git a/designer/client/src/components/modals/AdvancedParametersSection.tsx b/designer/client/src/components/modals/AdvancedParametersSection.tsx index 74cb8897190..4a5a3ebc0a5 100644 --- a/designer/client/src/components/modals/AdvancedParametersSection.tsx +++ b/designer/client/src/components/modals/AdvancedParametersSection.tsx @@ -13,10 +13,10 @@ export function AdvancedParametersSection({ children, nodeId }: PropsWithChildre return ( } + expandIcon={} aria-controls={`${nodeId}-content`} id={`${nodeId}-header`} - sx={{ flexDirection: "row-reverse", px: 0, border: 0 }} + sx={{ flexDirection: "row-reverse", border: 0 }} > {nodeId}