diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1d1dc8..ca571e35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each ## [Unreleased] +## [1.3.4](https://github.com/quintoandar/butterfree/releases/tag/1.3.4) +* Fix Cassandra Config and tests ([#366](https://github.com/quintoandar/butterfree/pull/366)) + ## [1.3.3](https://github.com/quintoandar/butterfree/releases/tag/1.3.3) * Fix Cassandra Config and Numpy version ([#364](https://github.com/quintoandar/butterfree/pull/364)) diff --git a/butterfree/configs/db/cassandra_config.py b/butterfree/configs/db/cassandra_config.py index d576359c..6d7f9a20 100644 --- a/butterfree/configs/db/cassandra_config.py +++ b/butterfree/configs/db/cassandra_config.py @@ -238,9 +238,9 @@ def translate(self, schema: List[Dict[str, Any]]) -> List[Dict[str, Any]]: "integertype": "int", "longtype": "bigint", "stringtype": "text", - "arraytype(longtype,true)": "frozen>", - "arraytype(stringtype,true)": "frozen>", - "arraytype(floattype,true)": "frozen>", + "arraytype(longtype, true)": "frozen>", + "arraytype(stringtype, true)": "frozen>", + "arraytype(floattype, true)": "frozen>", } cassandra_schema = [] for features in schema: diff --git a/docs/source/butterfree.configs.rst b/docs/source/butterfree.configs.rst index 771600f6..616bc4f8 100644 --- a/docs/source/butterfree.configs.rst +++ b/docs/source/butterfree.configs.rst @@ -38,6 +38,11 @@ butterfree.configs.logger module :undoc-members: :show-inheritance: +.. automodule:: butterfree.configs.logger + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/butterfree.constants.rst b/docs/source/butterfree.constants.rst index cfbfe725..a1f7a391 100644 --- a/docs/source/butterfree.constants.rst +++ b/docs/source/butterfree.constants.rst @@ -43,6 +43,12 @@ butterfree.constants.spark\_constants module :show-inheritance: +.. automodule:: butterfree.constants.migrations + :members: + :undoc-members: + :show-inheritance: + + .. automodule:: butterfree.constants.spark_constants :members: :undoc-members: @@ -66,6 +72,11 @@ butterfree.constants.window\_definitions module :undoc-members: :show-inheritance: +.. automodule:: butterfree.constants.window_definitions + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/butterfree.dataframe_service.rst b/docs/source/butterfree.dataframe_service.rst index 395e99ab..4343305b 100644 --- a/docs/source/butterfree.dataframe_service.rst +++ b/docs/source/butterfree.dataframe_service.rst @@ -4,29 +4,12 @@ butterfree.dataframe\_service package Submodules ---------- -butterfree.dataframe\_service.incremental\_strategy module ----------------------------------------------------------- .. automodule:: butterfree.dataframe_service.incremental_strategy :members: :undoc-members: :show-inheritance: -butterfree.dataframe\_service.partitioning module -------------------------------------------------- - -.. automodule:: butterfree.dataframe_service.partitioning - :members: - :undoc-members: - :show-inheritance: - -butterfree.dataframe\_service.repartition module ------------------------------------------------- - -.. automodule:: butterfree.dataframe_service.incremental_strategy - :members: - :undoc-members: - :show-inheritance: .. automodule:: butterfree.dataframe_service.partitioning :members: diff --git a/docs/source/butterfree.hooks.rst b/docs/source/butterfree.hooks.rst index c633cade..72f13223 100644 --- a/docs/source/butterfree.hooks.rst +++ b/docs/source/butterfree.hooks.rst @@ -12,16 +12,12 @@ Subpackages Submodules ---------- -butterfree.hooks.hook module ----------------------------- .. automodule:: butterfree.hooks.hook :members: :undoc-members: :show-inheritance: -butterfree.hooks.hookable\_component module -------------------------------------------- .. automodule:: butterfree.hooks.hookable_component :members: diff --git a/docs/source/butterfree.hooks.schema_compatibility.rst b/docs/source/butterfree.hooks.schema_compatibility.rst index 2d3de66c..a39c5b93 100644 --- a/docs/source/butterfree.hooks.schema_compatibility.rst +++ b/docs/source/butterfree.hooks.schema_compatibility.rst @@ -4,16 +4,12 @@ butterfree.hooks.schema\_compatibility package Submodules ---------- -butterfree.hooks.schema\_compatibility.cassandra\_table\_schema\_compatibility\_hook module -------------------------------------------------------------------------------------------- .. automodule:: butterfree.hooks.schema_compatibility.cassandra_table_schema_compatibility_hook :members: :undoc-members: :show-inheritance: -butterfree.hooks.schema\_compatibility.spark\_table\_schema\_compatibility\_hook module ---------------------------------------------------------------------------------------- .. automodule:: butterfree.hooks.schema_compatibility.spark_table_schema_compatibility_hook :members: diff --git a/docs/source/butterfree.migrations.database_migration.rst b/docs/source/butterfree.migrations.database_migration.rst index e015b0ea..892165df 100644 --- a/docs/source/butterfree.migrations.database_migration.rst +++ b/docs/source/butterfree.migrations.database_migration.rst @@ -4,23 +4,18 @@ butterfree.migrations.database\_migration package Submodules ---------- ---------------------------------------------------------------------- .. automodule:: butterfree.migrations.database_migration.cassandra_migration :members: :undoc-members: :show-inheritance: -butterfree.migrations.database\_migration.database\_migration module --------------------------------------------------------------------- .. automodule:: butterfree.migrations.database_migration.database_migration :members: :undoc-members: :show-inheritance: -butterfree.migrations.database\_migration.metastore\_migration module ---------------------------------------------------------------------- .. automodule:: butterfree.migrations.database_migration.metastore_migration :members: diff --git a/setup.py b/setup.py index f12b2681..2e64b472 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages, setup __package_name__ = "butterfree" -__version__ = "1.3.3" +__version__ = "1.3.4" __repository_url__ = "https://github.com/quintoandar/butterfree" with open("requirements.txt") as f: diff --git a/tests/unit/butterfree/migrations/database_migration/conftest.py b/tests/unit/butterfree/migrations/database_migration/conftest.py index 237158b7..3d3662d8 100644 --- a/tests/unit/butterfree/migrations/database_migration/conftest.py +++ b/tests/unit/butterfree/migrations/database_migration/conftest.py @@ -1,4 +1,11 @@ -from pyspark.sql.types import DoubleType, FloatType, LongType, TimestampType +from pyspark.sql.types import ( + ArrayType, + DoubleType, + FloatType, + LongType, + StringType, + TimestampType, +) from pytest import fixture from butterfree.constants import DataType @@ -30,6 +37,11 @@ def fs_schema(): {"column_name": "id", "type": LongType(), "primary_key": True}, {"column_name": "timestamp", "type": TimestampType(), "primary_key": True}, {"column_name": "new_feature", "type": FloatType(), "primary_key": False}, + { + "column_name": "array_feature", + "type": ArrayType(StringType(), True), + "primary_key": False, + }, { "column_name": "feature1__avg_over_1_week_rolling_windows", "type": FloatType(), diff --git a/tests/unit/butterfree/migrations/database_migration/test_cassandra_migration.py b/tests/unit/butterfree/migrations/database_migration/test_cassandra_migration.py index 5666cc47..5e89b65b 100644 --- a/tests/unit/butterfree/migrations/database_migration/test_cassandra_migration.py +++ b/tests/unit/butterfree/migrations/database_migration/test_cassandra_migration.py @@ -33,9 +33,11 @@ def test_create_table_query(self, fs_schema): expected_query = [ "CREATE TABLE test.table_name " "(id LongType, timestamp TimestampType, new_feature FloatType, " + "array_feature ArrayType(StringType(), True), " "feature1__avg_over_1_week_rolling_windows FloatType, " "PRIMARY KEY (id, timestamp));" ] + query = cassandra_migration.create_query(fs_schema, "table_name") assert query, expected_query