From b72dc6df2b9f6b6bd6b996b6120d3493932f914b Mon Sep 17 00:00:00 2001 From: Maciej Cichanowicz Date: Wed, 6 Nov 2024 13:52:04 +0100 Subject: [PATCH] Tests --- .../test/utils/domain/TestFactory.scala | 6 +- .../NotificationServiceTest.scala | 3 +- .../deployment/DeploymentServiceSpec.scala | 3 +- .../newdeployment/DeploymentServiceTest.scala | 2 +- .../FlinkProcessCompilerDataFactory.scala | 1 - .../engine/process/runner/FlinkTestMain.scala | 9 +- .../process/runner/FlinkTestMainSpec.scala | 68 ++++++++++++- ...nalDictComponentConfigsExtractorTest.scala | 99 +++++++++++++++++++ 8 files changed, 182 insertions(+), 9 deletions(-) create mode 100644 engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/AdditionalDictComponentConfigsExtractorTest.scala diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala index 6cb0419e829..c95746e8536 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala @@ -6,7 +6,7 @@ import cats.effect.unsafe.IORuntime import cats.instances.future._ import com.typesafe.config.ConfigFactory import db.util.DBIOActionInstances._ -import pl.touk.nussknacker.engine.api.component.{DesignerWideComponentId, ProcessingMode} +import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId, ProcessingMode} import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue import pl.touk.nussknacker.engine.api.deployment.{ NoOpScenarioActivityManager, @@ -120,6 +120,10 @@ object TestFactory { Streaming.stringify -> new ScenarioResolver(sampleResolver, Streaming.stringify) ) + def additionalComponentConfigsByProcessingType + : ProcessingTypeDataProvider[Map[DesignerWideComponentId, ComponentAdditionalConfig], _] = + mapProcessingTypeDataProvider() + val modelDependencies: ModelDependencies = ModelDependencies( TestAdditionalUIConfigProvider.componentAdditionalConfigMap, diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/notifications/NotificationServiceTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/notifications/NotificationServiceTest.scala index 8d67d78e469..0433d1314bd 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/notifications/NotificationServiceTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/notifications/NotificationServiceTest.scala @@ -16,7 +16,6 @@ import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.{DeploymentData, DeploymentId, ExternalDeploymentId} import pl.touk.nussknacker.test.base.db.WithHsqlDbTesting -import pl.touk.nussknacker.test.utils.domain.TestFactory.mapProcessingTypeDataProvider import pl.touk.nussknacker.test.utils.domain.{ProcessTestData, TestFactory} import pl.touk.nussknacker.test.utils.scalas.DBIOActionValues import pl.touk.nussknacker.test.{EitherValuesDetailedMessage, PatientScalaFutures} @@ -196,7 +195,7 @@ class NotificationServiceTest mock[ProcessChangeListener], None, None, - mapProcessingTypeDataProvider(), + TestFactory.additionalComponentConfigsByProcessingType, clock ) { override protected def validateBeforeDeploy( diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentServiceSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentServiceSpec.scala index 7883acf103f..1acdabd897c 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentServiceSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentServiceSpec.scala @@ -20,6 +20,7 @@ import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.deployment.{CustomActionResult, DeploymentId, ExternalDeploymentId} import pl.touk.nussknacker.test.base.db.WithHsqlDbTesting import pl.touk.nussknacker.test.base.it.WithClock +import pl.touk.nussknacker.test.config.WithSimplifiedDesignerConfig.TestProcessingType.Streaming import pl.touk.nussknacker.test.mock.{MockDeploymentManager, TestProcessChangeListener} import pl.touk.nussknacker.test.utils.domain.TestFactory._ import pl.touk.nussknacker.test.utils.domain.{ProcessTestData, TestFactory} @@ -115,7 +116,7 @@ class DeploymentServiceSpec listener, scenarioStateTimeout, deploymentCommentSettings, - mapProcessingTypeDataProvider() + additionalComponentConfigsByProcessingType ) } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/newdeployment/DeploymentServiceTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/newdeployment/DeploymentServiceTest.scala index 6e04fd20fd2..19aa10ce864 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/newdeployment/DeploymentServiceTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/newdeployment/DeploymentServiceTest.scala @@ -55,7 +55,7 @@ class DeploymentServiceTest ), dbioRunner, clock, - mapProcessingTypeDataProvider() + TestFactory.additionalComponentConfigsByProcessingType ) } diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkProcessCompilerDataFactory.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkProcessCompilerDataFactory.scala index 82eefcff80d..dd56b848c8f 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkProcessCompilerDataFactory.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkProcessCompilerDataFactory.scala @@ -121,7 +121,6 @@ class FlinkProcessCompilerDataFactory( ): (ModelDefinitionWithClasses, EngineDictRegistry) = { val dictRegistryFactory = loadDictRegistry(userCodeClassLoader) val modelDefinitionWithTypes = ModelDefinitionWithClasses( - // additionalConfigsFromProvider aren't provided, as it's not needed to run the process on flink extractModelDefinition( userCodeClassLoader, modelDependencies, diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/runner/FlinkTestMain.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/runner/FlinkTestMain.scala index bdbce7f3d83..c7cc037196c 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/runner/FlinkTestMain.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/runner/FlinkTestMain.scala @@ -32,7 +32,14 @@ object FlinkTestMain extends FlinkRunner { val processVersion = ProcessVersion.empty.copy(processName = ProcessName("snapshot version") ) // testing process may be unreleased, so it has no version - new FlinkTestMain(modelData, process, scenarioTestData, processVersion, DeploymentData.empty, configuration).runTest + new FlinkTestMain( + modelData, + process, + scenarioTestData, + processVersion, + DeploymentData.empty.copy(configsFromProviderWithDictionaryEditor = modelData.additionalConfigsFromProvider), + configuration + ).runTest } } diff --git a/engine/flink/executor/src/test/scala/pl/touk/nussknacker/engine/process/runner/FlinkTestMainSpec.scala b/engine/flink/executor/src/test/scala/pl/touk/nussknacker/engine/process/runner/FlinkTestMainSpec.scala index c2c336fd22a..8d378cb6a97 100644 --- a/engine/flink/executor/src/test/scala/pl/touk/nussknacker/engine/process/runner/FlinkTestMainSpec.scala +++ b/engine/flink/executor/src/test/scala/pl/touk/nussknacker/engine/process/runner/FlinkTestMainSpec.scala @@ -21,6 +21,13 @@ import pl.touk.nussknacker.engine.process.helpers.SampleNodes._ import pl.touk.nussknacker.engine.testmode.TestProcess._ import pl.touk.nussknacker.engine.util.ThreadUtils import pl.touk.nussknacker.engine.ModelData +import pl.touk.nussknacker.engine.api.component.{ + ComponentAdditionalConfig, + DesignerWideComponentId, + ParameterAdditionalUIConfig +} +import pl.touk.nussknacker.engine.api.parameter.{ParameterName, ValueInputWithDictEditor} +import pl.touk.nussknacker.engine.graph.expression.Expression import java.util.{Date, UUID} import scala.concurrent.ExecutionContext.Implicits.global @@ -654,6 +661,62 @@ class FlinkTestMainSpec extends AnyWordSpec with Matchers with Inside with Befor variable(List(ComponentUseCase.TestRuntime, ComponentUseCase.TestRuntime)) ) } + + "should throw exception when parameter was modified by AdditionalUiConfigProvider with dict editor and flink wasn't provided with additional config" in { + val process = + ScenarioBuilder + .streaming(scenarioName) + .source(sourceNodeId, "input") + .processor( + "eager1", + "collectingEager", + "static" -> Expression.dictKeyWithLabel("'s'", Some("s")), + "dynamic" -> "#input.id".spel + ) + .emptySink("out", "valueMonitor", "Value" -> "#input.value1".spel) + + val run = Future { + runFlinkTest(process, ScenarioTestData(List(createTestRecord(id = "2", value1 = 2))), useIOMonadInInterpreter) + } + val dictEditorException = intercept[IllegalStateException](Await.result(run, 10 seconds)) + dictEditorException.getMessage shouldBe "DictKeyWithLabel expression can only be used with DictParameterEditor, got Some(DualParameterEditor(StringParameterEditor,RAW))" + } + + "should run correctly when parameter was modified by AdditionalUiConfigProvider with dict editor and flink was provided with additional config" in { + val modifiedComponentName = "collectingEager" + val modifiedParameterName = "static" + val process = + ScenarioBuilder + .streaming(scenarioName) + .source(sourceNodeId, "input") + .processor( + "eager1", + modifiedComponentName, + modifiedParameterName -> Expression.dictKeyWithLabel("'s'", Some("s")), + "dynamic" -> "#input.id".spel + ) + .emptySink("out", "valueMonitor", "Value" -> "#input.value1".spel) + + val results = runFlinkTest( + process, + ScenarioTestData(List(createTestRecord(id = "2", value1 = 2))), + useIOMonadInInterpreter, + additionalConfigsFromProvider = Map( + DesignerWideComponentId("service-" + modifiedComponentName) -> ComponentAdditionalConfig( + parameterConfigs = Map( + ParameterName(modifiedParameterName) -> ParameterAdditionalUIConfig( + required = false, + initialValue = None, + hintText = None, + valueEditor = Some(ValueInputWithDictEditor("someDictId", allowOtherValue = false)), + valueCompileTimeValidation = None + ) + ) + ) + ) + ) + results.exceptions should have length 0 + } } private def createTestRecord( @@ -667,13 +730,14 @@ class FlinkTestMainSpec extends AnyWordSpec with Matchers with Inside with Befor process: CanonicalProcess, scenarioTestData: ScenarioTestData, useIOMonadInInterpreter: Boolean, - enrichDefaultConfig: Config => Config = identity + enrichDefaultConfig: Config => Config = identity, + additionalConfigsFromProvider: Map[DesignerWideComponentId, ComponentAdditionalConfig] = Map.empty ): TestResults[_] = { val config = enrichDefaultConfig(ConfigFactory.load("application.conf")) .withValue("globalParameters.useIOMonadInInterpreter", ConfigValueFactory.fromAnyRef(useIOMonadInInterpreter)) // We need to set context loader to avoid forking in sbt - val modelData = ModelData.duringFlinkExecution(config, Map.empty) + val modelData = ModelData.duringFlinkExecution(config, additionalConfigsFromProvider) ThreadUtils.withThisAsContextClassLoader(getClass.getClassLoader) { FlinkTestMain.run(modelData, process, scenarioTestData, FlinkTestConfiguration.configuration()) } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/AdditionalDictComponentConfigsExtractorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/AdditionalDictComponentConfigsExtractorTest.scala new file mode 100644 index 00000000000..dc653309720 --- /dev/null +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/AdditionalDictComponentConfigsExtractorTest.scala @@ -0,0 +1,99 @@ +package pl.touk.nussknacker.engine.management.periodic + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import pl.touk.nussknacker.engine.api.component.{ + ComponentAdditionalConfig, + ComponentGroupName, + DesignerWideComponentId, + ParameterAdditionalUIConfig +} +import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue +import pl.touk.nussknacker.engine.api.parameter.{ParameterName, ValueInputWithDictEditor} +import pl.touk.nussknacker.engine.management.periodic.AdditionalDictComponentConfigsExtractorTest.{ + componentConfigWithDictionaryEditorInParameter, + componentConfigWithOnlyDictEditorParameters, + componentConfigWithoutDictionaryEditorInParameter +} + +class AdditionalDictComponentConfigsExtractorTest extends AnyFunSuite with Matchers { + + test("should filter only components and parameters with dictionary editors") { + val additionalConfig = Map( + DesignerWideComponentId("componentA") -> componentConfigWithDictionaryEditorInParameter, + DesignerWideComponentId("componentB") -> componentConfigWithoutDictionaryEditorInParameter, + ) + val filteredResult = + AdditionalDictComponentConfigsExtractor.getAdditionalConfigsWithDictParametersEditors(additionalConfig) + + filteredResult shouldBe Map( + DesignerWideComponentId("componentA") -> componentConfigWithOnlyDictEditorParameters + ) + } + +} + +object AdditionalDictComponentConfigsExtractorTest { + + private val parameterAWithDictEditor = ( + ParameterName("parameterA"), + ParameterAdditionalUIConfig( + required = true, + initialValue = Some(FixedExpressionValue("'someInitialValueExpression'", "someInitialValueLabel")), + hintText = None, + valueEditor = Some(ValueInputWithDictEditor("someDictA", allowOtherValue = true)), + valueCompileTimeValidation = None + ) + ) + + private val parameterBWithDictEditor = ( + ParameterName("parameterB"), + ParameterAdditionalUIConfig( + required = false, + initialValue = None, + hintText = Some("someHint"), + valueEditor = Some(ValueInputWithDictEditor("someDictB", allowOtherValue = false)), + valueCompileTimeValidation = None + ) + ) + + private val parameterWithoutDictEditor = ( + ParameterName("parameterC"), + ParameterAdditionalUIConfig( + required = true, + initialValue = None, + hintText = None, + valueEditor = None, + valueCompileTimeValidation = None + ) + ) + + private val componentConfigWithDictionaryEditorInParameter = ComponentAdditionalConfig( + parameterConfigs = Map( + parameterAWithDictEditor, + parameterBWithDictEditor, + parameterWithoutDictEditor + ), + icon = Some("someIcon"), + docsUrl = Some("someDocUrl"), + componentGroup = Some(ComponentGroupName("Service")) + ) + + private val componentConfigWithoutDictionaryEditorInParameter = ComponentAdditionalConfig( + parameterConfigs = Map(parameterWithoutDictEditor), + icon = Some("someOtherIcon"), + docsUrl = Some("someOtherDocUrl"), + componentGroup = Some(ComponentGroupName("Service")) + ) + + private val componentConfigWithOnlyDictEditorParameters = ComponentAdditionalConfig( + parameterConfigs = Map( + parameterAWithDictEditor, + parameterBWithDictEditor + ), + icon = Some("someIcon"), + docsUrl = Some("someDocUrl"), + componentGroup = Some(ComponentGroupName("Service")) + ) + +}