Release 1.3.4 (#367)

* fix: Cassandra config keys (#366)

CHANGELOG.md

@@ -5,6 +5,9 @@ Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each
 
 ## [Unreleased]
 
+## [1.3.4](https://github.com/quintoandar/butterfree/releases/tag/1.3.4)
+* Fix Cassandra Config and tests ([#366](https://github.com/quintoandar/butterfree/pull/366))
+
 ## [1.3.3](https://github.com/quintoandar/butterfree/releases/tag/1.3.3)
 * Fix Cassandra Config and Numpy version ([#364](https://github.com/quintoandar/butterfree/pull/364))
 

butterfree/configs/db/cassandra_config.py

@@ -238,9 +238,9 @@ def translate(self, schema: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
             "integertype": "int",
             "longtype": "bigint",
             "stringtype": "text",
-            "arraytype(longtype,true)": "frozen<list<bigint>>",
-            "arraytype(stringtype,true)": "frozen<list<text>>",
-            "arraytype(floattype,true)": "frozen<list<float>>",
+            "arraytype(longtype, true)": "frozen<list<bigint>>",
+            "arraytype(stringtype, true)": "frozen<list<text>>",
+            "arraytype(floattype, true)": "frozen<list<float>>",
         }
         cassandra_schema = []
         for features in schema:

docs/source/butterfree.configs.rst

@@ -38,6 +38,11 @@ butterfree.configs.logger module
    :undoc-members:
    :show-inheritance:
 
+.. automodule:: butterfree.configs.logger
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

docs/source/butterfree.constants.rst

@@ -43,6 +43,12 @@ butterfree.constants.spark\_constants module
    :show-inheritance:
 
 
+.. automodule:: butterfree.constants.migrations
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
 .. automodule:: butterfree.constants.spark_constants
    :members:
    :undoc-members:
@@ -66,6 +72,11 @@ butterfree.constants.window\_definitions module
    :undoc-members:
    :show-inheritance:
 
+.. automodule:: butterfree.constants.window_definitions
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

docs/source/butterfree.dataframe_service.rst

@@ -4,29 +4,12 @@ butterfree.dataframe\_service package
 Submodules
 ----------
 
-butterfree.dataframe\_service.incremental\_strategy module
-----------------------------------------------------------
 
 .. automodule:: butterfree.dataframe_service.incremental_strategy
    :members:
    :undoc-members:
    :show-inheritance:
 
-butterfree.dataframe\_service.partitioning module
--------------------------------------------------
-
-.. automodule:: butterfree.dataframe_service.partitioning
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-butterfree.dataframe\_service.repartition module
-------------------------------------------------
-
-.. automodule:: butterfree.dataframe_service.incremental_strategy
-   :members:
-   :undoc-members:
-   :show-inheritance:
 
 .. automodule:: butterfree.dataframe_service.partitioning
    :members:

docs/source/butterfree.hooks.rst

@@ -12,16 +12,12 @@ Subpackages
 Submodules
 ----------
 
-butterfree.hooks.hook module
-----------------------------
 
 .. automodule:: butterfree.hooks.hook
    :members:
    :undoc-members:
    :show-inheritance:
 
-butterfree.hooks.hookable\_component module
--------------------------------------------
 
 .. automodule:: butterfree.hooks.hookable_component
    :members:

docs/source/butterfree.hooks.schema_compatibility.rst

@@ -4,16 +4,12 @@ butterfree.hooks.schema\_compatibility package
 Submodules
 ----------
 
-butterfree.hooks.schema\_compatibility.cassandra\_table\_schema\_compatibility\_hook module
--------------------------------------------------------------------------------------------
 
 .. automodule:: butterfree.hooks.schema_compatibility.cassandra_table_schema_compatibility_hook
    :members:
    :undoc-members:
    :show-inheritance:
 
-butterfree.hooks.schema\_compatibility.spark\_table\_schema\_compatibility\_hook module
----------------------------------------------------------------------------------------
 
 .. automodule:: butterfree.hooks.schema_compatibility.spark_table_schema_compatibility_hook
    :members:

docs/source/butterfree.migrations.database_migration.rst

@@ -4,23 +4,18 @@ butterfree.migrations.database\_migration package
 Submodules
 ----------
 
----------------------------------------------------------------------
 
 .. automodule:: butterfree.migrations.database_migration.cassandra_migration
    :members:
    :undoc-members:
    :show-inheritance:
 
-butterfree.migrations.database\_migration.database\_migration module
---------------------------------------------------------------------
 
 .. automodule:: butterfree.migrations.database_migration.database_migration
    :members:
    :undoc-members:
    :show-inheritance:
 
-butterfree.migrations.database\_migration.metastore\_migration module
----------------------------------------------------------------------
 
 .. automodule:: butterfree.migrations.database_migration.metastore_migration
    :members:

setup.py

@@ -1,7 +1,7 @@
 from setuptools import find_packages, setup
 
 __package_name__ = "butterfree"
-__version__ = "1.3.3"
+__version__ = "1.3.4"
 __repository_url__ = "https://github.com/quintoandar/butterfree"
 
 with open("requirements.txt") as f:

tests/unit/butterfree/migrations/database_migration/conftest.py

@@ -1,4 +1,11 @@
-from pyspark.sql.types import DoubleType, FloatType, LongType, TimestampType
+from pyspark.sql.types import (
+    ArrayType,
+    DoubleType,
+    FloatType,
+    LongType,
+    StringType,
+    TimestampType,
+)
 from pytest import fixture
 
 from butterfree.constants import DataType
@@ -30,6 +37,11 @@ def fs_schema():
         {"column_name": "id", "type": LongType(), "primary_key": True},
         {"column_name": "timestamp", "type": TimestampType(), "primary_key": True},
         {"column_name": "new_feature", "type": FloatType(), "primary_key": False},
+        {
+            "column_name": "array_feature",
+            "type": ArrayType(StringType(), True),
+            "primary_key": False,
+        },
         {
             "column_name": "feature1__avg_over_1_week_rolling_windows",
             "type": FloatType(),

tests/unit/butterfree/migrations/database_migration/test_cassandra_migration.py

@@ -33,9 +33,11 @@ def test_create_table_query(self, fs_schema):
         expected_query = [
             "CREATE TABLE test.table_name "
             "(id LongType, timestamp TimestampType, new_feature FloatType, "
+            "array_feature ArrayType(StringType(), True), "
             "feature1__avg_over_1_week_rolling_windows FloatType, "
             "PRIMARY KEY (id, timestamp));"
         ]
+
         query = cassandra_migration.create_query(fs_schema, "table_name")
 
         assert query, expected_query