Add binary type (#140)

* Add binary type

* Update project files

* Update schema version to v1.4.0

CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## v7.4.0
+
+### Change
+- Add support for `binary` type
+
+## v7.3.0
+
+### Change
+- Set the default AWS Glue version to 2.0
+
 ## v7.2.0
 
 ### Change

etl_manager/meta.py

@@ -57,7 +57,7 @@
 )
 
 _web_link_to_table_json_schema = (
-    "https://moj-analytical-services.github.io/metadata_schema/table/v1.3.0.json"
+    "https://moj-analytical-services.github.io/metadata_schema/table/v1.4.0.json"
 )
 
 try:

etl_manager/specs/glue_spark_dict.json

@@ -31,6 +31,10 @@
       "glue":"timestamp",
       "spark":"TimestampType"
     },
+    "binary": {
+      "glue":"binary",
+      "spark":"BinaryType"
+    },
     "boolean":{
       "glue":"boolean",
       "spark":"BooleanType"

etl_manager/specs/table_schema.json

@@ -41,6 +41,7 @@
               "decimal",
               "date",
               "datetime",
+              "binary",
               "boolean",
               "struct",
               "array"

etl_manager/utils.py

@@ -158,7 +158,7 @@ def _unnest_github_zipfile_and_return_new_zip_path(zip_path):
 # Note the ?R recursive.  We only allow 'character' (the agnostic type), in a non-complex type, but must allow string within complex types.
 # User will still get an error for string as non-complex type from the schema.
 COL_TYPE_REGEX = regex.compile(
-    r"(character|int|long|float|double|date|datetime|boolean|decimal\(\d+,\d+\)|struct<(([a-zA-Z_]+):((?R)(,?)))+>|array<(?R)>)"
+    r"(character|int|long|float|double|date|datetime|boolean|binary|decimal\(\d+,\d+\)|struct<(([a-zA-Z_]+):((?R)(,?)))+>|array<(?R)>)"
 )
 
 

example/meta_data/db1/employees.json

@@ -1,5 +1,5 @@
 {
-    "$schema": "https://moj-analytical-services.github.io/metadata_schema/table/v1.3.0.json",
+    "$schema": "https://moj-analytical-services.github.io/metadata_schema/table/v1.4.0.json",
     "name": "employees",
     "description": "table containing employee information",
     "data_format": "parquet",

example/meta_data/db1/pay.json

@@ -1,5 +1,5 @@
 {
-    "$schema": "https://moj-analytical-services.github.io/metadata_schema/table/v1.3.0.json",
+    "$schema": "https://moj-analytical-services.github.io/metadata_schema/table/v1.4.0.json",
     "name": "pay",
     "description": "Check glue specific works",
     "data_format": "csv",

example/meta_data/db1/teams.json

@@ -1,5 +1,5 @@
 {
-    "$schema": "https://moj-analytical-services.github.io/metadata_schema/table/v1.3.0.json",
+    "$schema": "https://moj-analytical-services.github.io/metadata_schema/table/v1.4.0.json",
     "name": "teams",
     "description": "month snapshot of which employee with working in what team",
     "data_format": "parquet",

pyproject.toml

@@ -1,7 +1,7 @@
 [tool]
 [tool.poetry]
 name = "etl_manager"
-version = "7.3.0"
+version = "7.4.0"
 description = "A python package to manage etl processes on AWS"
 license = "MIT"
 authors = ["Karik Isichei <[email protected]>"]

tests/data/data_types/test_table.json

@@ -1,5 +1,5 @@
 {
-    "$schema": "https://moj-analytical-services.github.io/metadata_schema/table/v1.3.0.json",
+    "$schema": "https://moj-analytical-services.github.io/metadata_schema/table/v1.4.0.json",
     "name": "test_table",
     "description": "table containing columns with each of the data types for testing database creation",
     "data_format": "json",

tests/test_column_types.py

@@ -59,6 +59,7 @@ def test_create_tables_using_etl_manager_api(self, mock_client_create_table):
             ("decimal(38,0)", True),
             ("date", True),
             ("datetime", True),
+            ("binary", True),
             ("boolean", True),
             ("struct", False),
             ("array", False),