diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/cast.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/cast.kt index 4d0245b159..378612717c 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/cast.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/cast.kt @@ -33,6 +33,8 @@ else cast() public fun AnyRow.cast(): DataRow = this as DataRow +public fun GroupBy<*, *>.cast(): GroupBy = this as GroupBy + public inline fun AnyRow.cast(verify: Boolean = true): DataRow = df().cast(verify)[0] public fun AnyCol.cast(): DataColumn = this as DataColumn diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator.kt index 4ec4202584..a34545fe16 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator.kt @@ -7,6 +7,7 @@ import org.jetbrains.kotlinx.dataframe.codeGen.Marker import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility import org.jetbrains.kotlinx.dataframe.codeGen.MarkersExtractor import org.jetbrains.kotlinx.dataframe.codeGen.NameNormalizer +import org.jetbrains.kotlinx.dataframe.codeGen.ProvidedCodeConverter import org.jetbrains.kotlinx.dataframe.impl.codeGen.CodeGeneratorImpl import org.jetbrains.kotlinx.dataframe.impl.codeGen.FullyQualifiedNames import org.jetbrains.kotlinx.dataframe.impl.codeGen.ShortNames @@ -22,7 +23,7 @@ public enum class InterfaceGenerationMode { None; } -public data class CodeGenResult(val code: CodeWithConverter, val newMarkers: List) +public data class CodeGenResult(val code: CodeWithConverter, val newMarkers: List) public interface CodeGenerator : ExtensionsCodeGenerator { @@ -43,7 +44,7 @@ public interface CodeGenerator : ExtensionsCodeGenerator { interfaceMode: InterfaceGenerationMode, extensionProperties: Boolean, readDfMethod: DefaultReadDfMethod? = null, - ): CodeWithConverter + ): CodeWithConverter public companion object { public fun create(useFqNames: Boolean = true): CodeGenerator { @@ -61,7 +62,7 @@ internal fun CodeGenerator.generate( markerClass: KClass<*>, interfaceMode: InterfaceGenerationMode, extensionProperties: Boolean, -): CodeWithConverter = generate( +): CodeWithConverter = generate( MarkersExtractor.get(markerClass), interfaceMode, extensionProperties @@ -70,4 +71,4 @@ internal fun CodeGenerator.generate( public inline fun CodeGenerator.generate( interfaceMode: InterfaceGenerationMode, extensionProperties: Boolean, -): CodeWithConverter = generate(T::class, interfaceMode, extensionProperties) +): CodeWithConverter = generate(T::class, interfaceMode, extensionProperties) diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt index dd48b739f4..9bbe416142 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt @@ -7,14 +7,19 @@ import org.jetbrains.kotlinx.jupyter.api.VariableName * Class representing generated code declarations for a [Marker]. * * @param declarations The generated code. + * @param converter Needs to provide additional info (name) from org.jetbrains.dataframe.impl.codeGen.CodeGenerator to its callers + * But at the same time name doesn't make sense for GroupBy where code to be executed contains two declarations * @param converter Optional converter for the [Marker], such as a [org.jetbrains.kotlinx.dataframe.api.cast], often used for Jupyter. */ -public data class CodeWithConverter(val declarations: Code, val converter: (VariableName) -> Code = EmptyConverter) { +public data class CodeWithConverter( + val declarations: Code, + val converter: T +) { public companion object { public const val EmptyDeclarations: Code = "" - public val EmptyConverter: (VariableName) -> Code = { it } - public val Empty: CodeWithConverter = CodeWithConverter(EmptyDeclarations, EmptyConverter) + public val EmptyConverter: CodeConverter = CodeConverter { it } + public val Empty: CodeWithConverter = CodeWithConverter(EmptyDeclarations, EmptyConverter) } val hasDeclarations: Boolean get() = declarations.isNotBlank() @@ -27,3 +32,19 @@ public data class CodeWithConverter(val declarations: Code, val converter: (Vari else -> declarations + "\n" + converter(name) } } + +public sealed interface CodeConverter : (VariableName) -> Code + +public class CodeConverterImpl(private val f: (VariableName) -> Code) : CodeConverter { + override fun invoke(p1: VariableName): Code { + return f(p1) + } +} + +public fun CodeConverter(f: (VariableName) -> Code): CodeConverter = CodeConverterImpl(f) + +public class ProvidedCodeConverter(public val markerName: String) : CodeConverter { + override fun invoke(p1: VariableName): Code { + return "$p1.cast<$markerName>()" + } +} diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ExtensionsCodeGenerator.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ExtensionsCodeGenerator.kt index de874282f1..fc67c23ff2 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ExtensionsCodeGenerator.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ExtensionsCodeGenerator.kt @@ -4,7 +4,7 @@ import org.jetbrains.kotlinx.dataframe.impl.codeGen.ExtensionsCodeGeneratorImpl import org.jetbrains.kotlinx.dataframe.impl.codeGen.ShortNames public interface ExtensionsCodeGenerator { - public fun generate(marker: IsolatedMarker): CodeWithConverter + public fun generate(marker: IsolatedMarker): CodeWithConverter<*> public companion object { public fun create(): ExtensionsCodeGenerator = ExtensionsCodeGeneratorImpl(ShortNames) diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt index 63ceb4ffd5..001c48f65e 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt @@ -2,6 +2,7 @@ package org.jetbrains.dataframe.impl.codeGen import org.jetbrains.kotlinx.dataframe.AnyFrame import org.jetbrains.kotlinx.dataframe.AnyRow +import org.jetbrains.kotlinx.dataframe.api.GroupBy import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter import org.jetbrains.kotlinx.dataframe.impl.codeGen.ReplCodeGeneratorImpl import org.jetbrains.kotlinx.jupyter.api.Code @@ -13,15 +14,17 @@ internal interface ReplCodeGenerator { fun process( df: AnyFrame, property: KProperty<*>? = null, - ): CodeWithConverter + ): CodeWithConverter<*> fun process( row: AnyRow, property: KProperty<*>? = null, - ): CodeWithConverter + ): CodeWithConverter<*> fun process(markerClass: KClass<*>): Code + fun process(groupBy: GroupBy<*, *>): CodeWithConverter<*> + companion object { fun create(): ReplCodeGenerator = ReplCodeGeneratorImpl() } diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt index 73baaf8f01..c7729f26e8 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt @@ -27,6 +27,7 @@ import org.jetbrains.kotlinx.dataframe.codeGen.IsolatedMarker import org.jetbrains.kotlinx.dataframe.codeGen.Marker import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility import org.jetbrains.kotlinx.dataframe.codeGen.NameNormalizer +import org.jetbrains.kotlinx.dataframe.codeGen.ProvidedCodeConverter import org.jetbrains.kotlinx.dataframe.codeGen.SchemaProcessor import org.jetbrains.kotlinx.dataframe.codeGen.ValidFieldName import org.jetbrains.kotlinx.dataframe.codeGen.toNullable @@ -43,7 +44,7 @@ internal fun Iterable.filterRequiredForSchema(schema: DataFrameSchema) = internal val charsToQuote = """[ `(){}\[\].<>'"/|\\!?@:;%^&*#$-]""".toRegex() internal fun createCodeWithConverter(code: String, markerName: String) = - CodeWithConverter(code) { "$it.cast<$markerName>()" } + CodeWithConverter(code, ProvidedCodeConverter(markerName)) private val letterCategories = setOf( CharCategory.UPPERCASE_LETTER, @@ -345,7 +346,7 @@ internal open class ExtensionsCodeGeneratorImpl( return declarations.joinToString("\n") } - override fun generate(marker: IsolatedMarker): CodeWithConverter { + override fun generate(marker: IsolatedMarker): CodeWithConverter<*> { val code = generateExtensionProperties(marker) return createCodeWithConverter(code, marker.name) } @@ -370,7 +371,7 @@ internal class CodeGeneratorImpl(typeRendering: TypeRenderingStrategy = FullyQua interfaceMode: InterfaceGenerationMode, extensionProperties: Boolean, readDfMethod: DefaultReadDfMethod?, - ): CodeWithConverter { + ): CodeWithConverter { val code = when (interfaceMode) { NoFields, WithFields -> generateInterface( @@ -505,7 +506,7 @@ internal class CodeGeneratorImpl(typeRendering: TypeRenderingStrategy = FullyQua } } -public fun CodeWithConverter.toStandaloneSnippet(packageName: String, additionalImports: List): String = +public fun CodeWithConverter<*>.toStandaloneSnippet(packageName: String, additionalImports: List): String = declarations.toStandaloneSnippet(packageName, additionalImports) public fun Code.toStandaloneSnippet(packageName: String, additionalImports: List): String = diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt index ee32c8e231..ea31d88e84 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt @@ -8,11 +8,14 @@ import org.jetbrains.kotlinx.dataframe.AnyRow import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.DataRow import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.api.GroupBy import org.jetbrains.kotlinx.dataframe.api.schema +import org.jetbrains.kotlinx.dataframe.codeGen.CodeConverter import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter import org.jetbrains.kotlinx.dataframe.codeGen.Marker import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility import org.jetbrains.kotlinx.dataframe.codeGen.MarkersExtractor +import org.jetbrains.kotlinx.dataframe.codeGen.ProvidedCodeConverter import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.jetbrains.kotlinx.jupyter.api.Code import kotlin.reflect.KClass @@ -44,9 +47,9 @@ internal class ReplCodeGeneratorImpl : ReplCodeGenerator { else -> null } - override fun process(row: AnyRow, property: KProperty<*>?): CodeWithConverter = process(row.df(), property) + override fun process(row: AnyRow, property: KProperty<*>?): CodeWithConverter<*> = process(row.df(), property) - override fun process(df: AnyFrame, property: KProperty<*>?): CodeWithConverter { + override fun process(df: AnyFrame, property: KProperty<*>?): CodeWithConverter<*> { var targetSchema = df.schema() if (property != null) { @@ -78,11 +81,21 @@ internal class ReplCodeGeneratorImpl : ReplCodeGenerator { return generate(schema = targetSchema, name = markerInterfacePrefix, isOpen = true) } + override fun process(groupBy: GroupBy<*, *>): CodeWithConverter<*> { + val key = generate(groupBy.keys.schema(), markerInterfacePrefix + "Keys", isOpen = false) + val group = generate(groupBy.groups.schema.value, markerInterfacePrefix + "Groups", isOpen = false) + + return CodeWithConverter( + key.declarations.plus("\n").plus(group.declarations), + converter = CodeConverter { "$it.cast<${key.converter.markerName}, ${group.converter.markerName}>()" } + ) + } + fun generate( schema: DataFrameSchema, name: String, isOpen: Boolean, - ): CodeWithConverter { + ): CodeWithConverter { val result = generator.generate( schema = schema, name = name, diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt index af4ebd902d..4a937ff80f 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt @@ -33,6 +33,7 @@ import org.jetbrains.kotlinx.dataframe.api.asDataFrame import org.jetbrains.kotlinx.dataframe.api.columnsCount import org.jetbrains.kotlinx.dataframe.api.isColumnGroup import org.jetbrains.kotlinx.dataframe.api.name +import org.jetbrains.kotlinx.dataframe.codeGen.CodeConverter import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup import org.jetbrains.kotlinx.dataframe.columns.ColumnReference @@ -64,7 +65,7 @@ internal class Integration( val version = options["v"] - private fun KotlinKernelHost.execute(codeWithConverter: CodeWithConverter, argument: String): VariableName? { + private fun KotlinKernelHost.execute(codeWithConverter: CodeWithConverter<*>, argument: String): VariableName? { val code = codeWithConverter.with(argument) return if (code.isNotBlank()) { val result = execute(code) @@ -75,7 +76,7 @@ internal class Integration( } private fun KotlinKernelHost.execute( - codeWithConverter: CodeWithConverter, + codeWithConverter: CodeWithConverter<*>, property: KProperty<*>, type: KType, ): VariableName? { @@ -148,7 +149,7 @@ internal class Integration( codeGen: ReplCodeGenerator, ): VariableName? = if (col.isColumnGroup()) { val codeWithConverter = codeGen.process(col.asColumnGroup().asDataFrame(), property).let { c -> - CodeWithConverter(c.declarations) { c.converter("$it.asColumnGroup()") } + CodeWithConverter(c.declarations, converter = CodeConverter { c.converter("$it.asColumnGroup()") }) } execute( codeWithConverter = codeWithConverter, @@ -282,6 +283,7 @@ internal class Integration( is AnyRow -> updateAnyRowVariable(instance, property, codeGen) is AnyFrame -> updateAnyFrameVariable(instance, property, codeGen) is ImportDataSchema -> updateImportDataSchemaVariable(instance, property) + is GroupBy<*, *> -> execute(codeGen.process(instance), property, GroupBy::class.createStarProjectedType(false)) else -> error("${instance::class} should not be handled by Dataframe field handler") } } @@ -290,7 +292,8 @@ internal class Integration( value is ColumnGroup<*> || value is AnyRow || value is AnyFrame || - value is ImportDataSchema + value is ImportDataSchema || + value is GroupBy<*, *> } }) diff --git a/core/generated-sources/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/CodeGenerationTests.kt b/core/generated-sources/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/CodeGenerationTests.kt index a7f1d28e1c..47fc06fb49 100644 --- a/core/generated-sources/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/CodeGenerationTests.kt +++ b/core/generated-sources/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/CodeGenerationTests.kt @@ -52,6 +52,22 @@ class CodeGenerationTests : DataFrameJupyterTest() { """.checkCompilation() } + @Test + fun `groupBy`() { + """ + val groupBy = dataFrameOf("a")("1", "11", "2", "22").groupBy { expr { "a"().length } named "k" } + groupBy.keys.k + """.checkCompilation() + } + + @Test + fun `groupBy add`() { + """ + val groupBy = dataFrameOf("a")("1", "11", "2", "22").groupBy { expr { "a"().length } named "k" }.add("newCol") { 42 } + groupBy.aggregate { newCol into "newCol" } + """.checkCompilation() + } + @Test fun `interface without body compiled correctly`() { """ diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/cast.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/cast.kt index 4d0245b159..378612717c 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/cast.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/cast.kt @@ -33,6 +33,8 @@ else cast() public fun AnyRow.cast(): DataRow = this as DataRow +public fun GroupBy<*, *>.cast(): GroupBy = this as GroupBy + public inline fun AnyRow.cast(verify: Boolean = true): DataRow = df().cast(verify)[0] public fun AnyCol.cast(): DataColumn = this as DataColumn diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator.kt index 4ec4202584..a34545fe16 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator.kt @@ -7,6 +7,7 @@ import org.jetbrains.kotlinx.dataframe.codeGen.Marker import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility import org.jetbrains.kotlinx.dataframe.codeGen.MarkersExtractor import org.jetbrains.kotlinx.dataframe.codeGen.NameNormalizer +import org.jetbrains.kotlinx.dataframe.codeGen.ProvidedCodeConverter import org.jetbrains.kotlinx.dataframe.impl.codeGen.CodeGeneratorImpl import org.jetbrains.kotlinx.dataframe.impl.codeGen.FullyQualifiedNames import org.jetbrains.kotlinx.dataframe.impl.codeGen.ShortNames @@ -22,7 +23,7 @@ public enum class InterfaceGenerationMode { None; } -public data class CodeGenResult(val code: CodeWithConverter, val newMarkers: List) +public data class CodeGenResult(val code: CodeWithConverter, val newMarkers: List) public interface CodeGenerator : ExtensionsCodeGenerator { @@ -43,7 +44,7 @@ public interface CodeGenerator : ExtensionsCodeGenerator { interfaceMode: InterfaceGenerationMode, extensionProperties: Boolean, readDfMethod: DefaultReadDfMethod? = null, - ): CodeWithConverter + ): CodeWithConverter public companion object { public fun create(useFqNames: Boolean = true): CodeGenerator { @@ -61,7 +62,7 @@ internal fun CodeGenerator.generate( markerClass: KClass<*>, interfaceMode: InterfaceGenerationMode, extensionProperties: Boolean, -): CodeWithConverter = generate( +): CodeWithConverter = generate( MarkersExtractor.get(markerClass), interfaceMode, extensionProperties @@ -70,4 +71,4 @@ internal fun CodeGenerator.generate( public inline fun CodeGenerator.generate( interfaceMode: InterfaceGenerationMode, extensionProperties: Boolean, -): CodeWithConverter = generate(T::class, interfaceMode, extensionProperties) +): CodeWithConverter = generate(T::class, interfaceMode, extensionProperties) diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt index dd48b739f4..9bbe416142 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt @@ -7,14 +7,19 @@ import org.jetbrains.kotlinx.jupyter.api.VariableName * Class representing generated code declarations for a [Marker]. * * @param declarations The generated code. + * @param converter Needs to provide additional info (name) from org.jetbrains.dataframe.impl.codeGen.CodeGenerator to its callers + * But at the same time name doesn't make sense for GroupBy where code to be executed contains two declarations * @param converter Optional converter for the [Marker], such as a [org.jetbrains.kotlinx.dataframe.api.cast], often used for Jupyter. */ -public data class CodeWithConverter(val declarations: Code, val converter: (VariableName) -> Code = EmptyConverter) { +public data class CodeWithConverter( + val declarations: Code, + val converter: T +) { public companion object { public const val EmptyDeclarations: Code = "" - public val EmptyConverter: (VariableName) -> Code = { it } - public val Empty: CodeWithConverter = CodeWithConverter(EmptyDeclarations, EmptyConverter) + public val EmptyConverter: CodeConverter = CodeConverter { it } + public val Empty: CodeWithConverter = CodeWithConverter(EmptyDeclarations, EmptyConverter) } val hasDeclarations: Boolean get() = declarations.isNotBlank() @@ -27,3 +32,19 @@ public data class CodeWithConverter(val declarations: Code, val converter: (Vari else -> declarations + "\n" + converter(name) } } + +public sealed interface CodeConverter : (VariableName) -> Code + +public class CodeConverterImpl(private val f: (VariableName) -> Code) : CodeConverter { + override fun invoke(p1: VariableName): Code { + return f(p1) + } +} + +public fun CodeConverter(f: (VariableName) -> Code): CodeConverter = CodeConverterImpl(f) + +public class ProvidedCodeConverter(public val markerName: String) : CodeConverter { + override fun invoke(p1: VariableName): Code { + return "$p1.cast<$markerName>()" + } +} diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ExtensionsCodeGenerator.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ExtensionsCodeGenerator.kt index de874282f1..fc67c23ff2 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ExtensionsCodeGenerator.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ExtensionsCodeGenerator.kt @@ -4,7 +4,7 @@ import org.jetbrains.kotlinx.dataframe.impl.codeGen.ExtensionsCodeGeneratorImpl import org.jetbrains.kotlinx.dataframe.impl.codeGen.ShortNames public interface ExtensionsCodeGenerator { - public fun generate(marker: IsolatedMarker): CodeWithConverter + public fun generate(marker: IsolatedMarker): CodeWithConverter<*> public companion object { public fun create(): ExtensionsCodeGenerator = ExtensionsCodeGeneratorImpl(ShortNames) diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt index 63ceb4ffd5..001c48f65e 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt @@ -2,6 +2,7 @@ package org.jetbrains.dataframe.impl.codeGen import org.jetbrains.kotlinx.dataframe.AnyFrame import org.jetbrains.kotlinx.dataframe.AnyRow +import org.jetbrains.kotlinx.dataframe.api.GroupBy import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter import org.jetbrains.kotlinx.dataframe.impl.codeGen.ReplCodeGeneratorImpl import org.jetbrains.kotlinx.jupyter.api.Code @@ -13,15 +14,17 @@ internal interface ReplCodeGenerator { fun process( df: AnyFrame, property: KProperty<*>? = null, - ): CodeWithConverter + ): CodeWithConverter<*> fun process( row: AnyRow, property: KProperty<*>? = null, - ): CodeWithConverter + ): CodeWithConverter<*> fun process(markerClass: KClass<*>): Code + fun process(groupBy: GroupBy<*, *>): CodeWithConverter<*> + companion object { fun create(): ReplCodeGenerator = ReplCodeGeneratorImpl() } diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt index 73baaf8f01..c7729f26e8 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt @@ -27,6 +27,7 @@ import org.jetbrains.kotlinx.dataframe.codeGen.IsolatedMarker import org.jetbrains.kotlinx.dataframe.codeGen.Marker import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility import org.jetbrains.kotlinx.dataframe.codeGen.NameNormalizer +import org.jetbrains.kotlinx.dataframe.codeGen.ProvidedCodeConverter import org.jetbrains.kotlinx.dataframe.codeGen.SchemaProcessor import org.jetbrains.kotlinx.dataframe.codeGen.ValidFieldName import org.jetbrains.kotlinx.dataframe.codeGen.toNullable @@ -43,7 +44,7 @@ internal fun Iterable.filterRequiredForSchema(schema: DataFrameSchema) = internal val charsToQuote = """[ `(){}\[\].<>'"/|\\!?@:;%^&*#$-]""".toRegex() internal fun createCodeWithConverter(code: String, markerName: String) = - CodeWithConverter(code) { "$it.cast<$markerName>()" } + CodeWithConverter(code, ProvidedCodeConverter(markerName)) private val letterCategories = setOf( CharCategory.UPPERCASE_LETTER, @@ -345,7 +346,7 @@ internal open class ExtensionsCodeGeneratorImpl( return declarations.joinToString("\n") } - override fun generate(marker: IsolatedMarker): CodeWithConverter { + override fun generate(marker: IsolatedMarker): CodeWithConverter<*> { val code = generateExtensionProperties(marker) return createCodeWithConverter(code, marker.name) } @@ -370,7 +371,7 @@ internal class CodeGeneratorImpl(typeRendering: TypeRenderingStrategy = FullyQua interfaceMode: InterfaceGenerationMode, extensionProperties: Boolean, readDfMethod: DefaultReadDfMethod?, - ): CodeWithConverter { + ): CodeWithConverter { val code = when (interfaceMode) { NoFields, WithFields -> generateInterface( @@ -505,7 +506,7 @@ internal class CodeGeneratorImpl(typeRendering: TypeRenderingStrategy = FullyQua } } -public fun CodeWithConverter.toStandaloneSnippet(packageName: String, additionalImports: List): String = +public fun CodeWithConverter<*>.toStandaloneSnippet(packageName: String, additionalImports: List): String = declarations.toStandaloneSnippet(packageName, additionalImports) public fun Code.toStandaloneSnippet(packageName: String, additionalImports: List): String = diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt index ee32c8e231..ea31d88e84 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt @@ -8,11 +8,14 @@ import org.jetbrains.kotlinx.dataframe.AnyRow import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.DataRow import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.api.GroupBy import org.jetbrains.kotlinx.dataframe.api.schema +import org.jetbrains.kotlinx.dataframe.codeGen.CodeConverter import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter import org.jetbrains.kotlinx.dataframe.codeGen.Marker import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility import org.jetbrains.kotlinx.dataframe.codeGen.MarkersExtractor +import org.jetbrains.kotlinx.dataframe.codeGen.ProvidedCodeConverter import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.jetbrains.kotlinx.jupyter.api.Code import kotlin.reflect.KClass @@ -44,9 +47,9 @@ internal class ReplCodeGeneratorImpl : ReplCodeGenerator { else -> null } - override fun process(row: AnyRow, property: KProperty<*>?): CodeWithConverter = process(row.df(), property) + override fun process(row: AnyRow, property: KProperty<*>?): CodeWithConverter<*> = process(row.df(), property) - override fun process(df: AnyFrame, property: KProperty<*>?): CodeWithConverter { + override fun process(df: AnyFrame, property: KProperty<*>?): CodeWithConverter<*> { var targetSchema = df.schema() if (property != null) { @@ -78,11 +81,21 @@ internal class ReplCodeGeneratorImpl : ReplCodeGenerator { return generate(schema = targetSchema, name = markerInterfacePrefix, isOpen = true) } + override fun process(groupBy: GroupBy<*, *>): CodeWithConverter<*> { + val key = generate(groupBy.keys.schema(), markerInterfacePrefix + "Keys", isOpen = false) + val group = generate(groupBy.groups.schema.value, markerInterfacePrefix + "Groups", isOpen = false) + + return CodeWithConverter( + key.declarations.plus("\n").plus(group.declarations), + converter = CodeConverter { "$it.cast<${key.converter.markerName}, ${group.converter.markerName}>()" } + ) + } + fun generate( schema: DataFrameSchema, name: String, isOpen: Boolean, - ): CodeWithConverter { + ): CodeWithConverter { val result = generator.generate( schema = schema, name = name, diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt index af4ebd902d..4a937ff80f 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt @@ -33,6 +33,7 @@ import org.jetbrains.kotlinx.dataframe.api.asDataFrame import org.jetbrains.kotlinx.dataframe.api.columnsCount import org.jetbrains.kotlinx.dataframe.api.isColumnGroup import org.jetbrains.kotlinx.dataframe.api.name +import org.jetbrains.kotlinx.dataframe.codeGen.CodeConverter import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup import org.jetbrains.kotlinx.dataframe.columns.ColumnReference @@ -64,7 +65,7 @@ internal class Integration( val version = options["v"] - private fun KotlinKernelHost.execute(codeWithConverter: CodeWithConverter, argument: String): VariableName? { + private fun KotlinKernelHost.execute(codeWithConverter: CodeWithConverter<*>, argument: String): VariableName? { val code = codeWithConverter.with(argument) return if (code.isNotBlank()) { val result = execute(code) @@ -75,7 +76,7 @@ internal class Integration( } private fun KotlinKernelHost.execute( - codeWithConverter: CodeWithConverter, + codeWithConverter: CodeWithConverter<*>, property: KProperty<*>, type: KType, ): VariableName? { @@ -148,7 +149,7 @@ internal class Integration( codeGen: ReplCodeGenerator, ): VariableName? = if (col.isColumnGroup()) { val codeWithConverter = codeGen.process(col.asColumnGroup().asDataFrame(), property).let { c -> - CodeWithConverter(c.declarations) { c.converter("$it.asColumnGroup()") } + CodeWithConverter(c.declarations, converter = CodeConverter { c.converter("$it.asColumnGroup()") }) } execute( codeWithConverter = codeWithConverter, @@ -282,6 +283,7 @@ internal class Integration( is AnyRow -> updateAnyRowVariable(instance, property, codeGen) is AnyFrame -> updateAnyFrameVariable(instance, property, codeGen) is ImportDataSchema -> updateImportDataSchemaVariable(instance, property) + is GroupBy<*, *> -> execute(codeGen.process(instance), property, GroupBy::class.createStarProjectedType(false)) else -> error("${instance::class} should not be handled by Dataframe field handler") } } @@ -290,7 +292,8 @@ internal class Integration( value is ColumnGroup<*> || value is AnyRow || value is AnyFrame || - value is ImportDataSchema + value is ImportDataSchema || + value is GroupBy<*, *> } }) diff --git a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/CodeGenerationTests.kt b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/CodeGenerationTests.kt index a7f1d28e1c..47fc06fb49 100644 --- a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/CodeGenerationTests.kt +++ b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/CodeGenerationTests.kt @@ -52,6 +52,22 @@ class CodeGenerationTests : DataFrameJupyterTest() { """.checkCompilation() } + @Test + fun `groupBy`() { + """ + val groupBy = dataFrameOf("a")("1", "11", "2", "22").groupBy { expr { "a"().length } named "k" } + groupBy.keys.k + """.checkCompilation() + } + + @Test + fun `groupBy add`() { + """ + val groupBy = dataFrameOf("a")("1", "11", "2", "22").groupBy { expr { "a"().length } named "k" }.add("newCol") { 42 } + groupBy.aggregate { newCol into "newCol" } + """.checkCompilation() + } + @Test fun `interface without body compiled correctly`() { """