diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 7e2003d..3d1304e 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,9 @@ +**v0.8.2** +1. Added fundamental concept of TableMetaModel - class that unifies metadata parsed from different classes/ORM models types/DDLs to one standard to allow easy way convert one models to another +in next releases it will be used for converter from one type of models to another. +2. Fixed issue: https://github.com/xnuinside/omymodels/issues/18 "NOW() not recognized as now()" +3. Fixed issue: https://github.com/xnuinside/omymodels/issues/19 "Default value of now() always returns same time, use field for dataclass" + **v0.8.1** 1. Parser version is updated (fixed several issues with generation) 2. Fixed issue with Unique Constraint after schema in SQLAlchemy Core diff --git a/README.md b/README.md index 981accf..e9580ce 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,7 @@ And result will be this: 1. Add Sequence generation in Models (Gino, SQLAlchemy) 2. Generate Tortoise ORM models (https://tortoise-orm.readthedocs.io/en/latest/) +3. Convert SQLAlchemy models to DjangoORM, Pydantic, SQLAlchemy Tables, Dataclasses (?) ## How to contribute @@ -277,6 +278,12 @@ Please describe issue that you want to solve and open the PR, I will review it a Any questions? Ping me in Telegram: https://t.me/xnuinside ## Changelog +**v0.8.2** +1. Added fundamental concept of TableMetaModel - class that unifies metadata parsed from different classes/ORM models types/DDLs to one standard to allow easy way convert one models to another +in next releases it will be used for converter from one type of models to another. +2. Fixed issue: https://github.com/xnuinside/omymodels/issues/18 "NOW() not recognized as now()" +3. Fixed issue: https://github.com/xnuinside/omymodels/issues/19 "Default value of now() always returns same time, use field for dataclass" + **v0.8.1** 1. Parser version is updated (fixed several issues with generation) 2. Fixed issue with Unique Constraint after schema in SQLAlchemy Core diff --git a/docs/README.rst b/docs/README.rst index 4ed47c7..4c6727f 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -281,6 +281,7 @@ TODO in next releases #. Add Sequence generation in Models (Gino, SQLAlchemy) #. Generate Tortoise ORM models (https://tortoise-orm.readthedocs.io/en/latest/) +#. Convert SQLAlchemy models to DjangoORM, Pydantic, SQLAlchemy Tables, Dataclasses (?) How to contribute ----------------- @@ -292,6 +293,14 @@ Any questions? Ping me in Telegram: https://t.me/xnuinside Changelog --------- +**v0.8.2** + + +#. Added fundamental concept of TableMetaModel - class that unifies metadata parsed from different classes/ORM models types/DDLs to one standard to allow easy way convert one models to another + in next releases it will be used for converter from one type of models to another. +#. Fixed issue: https://github.com/xnuinside/omymodels/issues/18 "NOW() not recognized as now()" +#. Fixed issue: https://github.com/xnuinside/omymodels/issues/19 "Default value of now() always returns same time, use field for dataclass" + **v0.8.1** diff --git a/omymodels/common.py b/omymodels/common.py index 9017cda..14dec19 100644 --- a/omymodels/common.py +++ b/omymodels/common.py @@ -4,12 +4,15 @@ from jinja2 import Template from typing import Optional, List, Dict + +from typing_extensions import final from simple_ddl_parser import DDLParser, parse_from_file from omymodels.gino import core as g from omymodels.pydantic import core as p from omymodels.dataclass import core as d from omymodels.sqlalchemy import core as s from omymodels.sqlalchemy_core import core as sc +from omymodels.meta_model import TableMeta, Column, Type def get_tables_information( @@ -41,11 +44,10 @@ def create_models( """ models_type can be: "gino", "dataclass", "pydantic" """ # extract data from ddl file data = get_tables_information(ddl, ddl_path) - print(data) - if not data['tables'] or data['tables'][0].get("schema", 'NOT EXIST') == 'NOT EXIST': - print("No tables found in DDL. Exit.") - sys.exit(0) data = remove_quotes_from_strings(data) + data = convert_ddl_to_models(data) + if not data['tables']: + sys.exit(0) # generate code output = generate_models_file( data, singular, naming_exceptions, models_type, schema_global, defaults_off @@ -57,6 +59,19 @@ def create_models( return {"metadata": data, "code": output} +def convert_ddl_to_models(data): + final_data = {'tables': [], 'types': []} + tables = [] + for table in data['tables']: + tables.append(TableMeta(**table)) + final_data['tables'] = tables + _types = [] + for _type in data['types']: + _types.append(Type(**_type)) + final_data['types'] = _types + return final_data + + def save_models_to_file(models: str, dump_path: str) -> None: folder = os.path.dirname(dump_path) if folder: @@ -98,7 +113,8 @@ def generate_models_file( schema_global=schema_global, defaults_off=defaults_off, ) - header = model_generator.create_header(data["tables"], schema=schema_global) + header = model_generator.create_header( + data["tables"], schema=schema_global) output = render_jinja2_template(models_type, models_str, header) return output diff --git a/omymodels/dataclass/core.py b/omymodels/dataclass/core.py index fc0b0a3..97aaefa 100644 --- a/omymodels/dataclass/core.py +++ b/omymodels/dataclass/core.py @@ -1,5 +1,5 @@ from typing import Optional, List, Dict -from omymodels.dataclass import templates as pt +from omymodels.dataclass import templates as dt from omymodels.utils import create_class_name, enum_number_name_list from omymodels.dataclass.types import types_mapping, datetime_types @@ -13,6 +13,7 @@ def __init__(self): self.enum_imports = set() self.custom_types = {} self.uuid_import = False + self.additional_imports = set() def add_custom_type(self, _type: str) -> str: column_type = self.custom_types.get(_type, _type) @@ -22,12 +23,12 @@ def add_custom_type(self, _type: str) -> str: return _type def generate_attr(self, column: Dict, defaults_off: bool) -> str: - column_str = pt.dataclass_attr + column_str = dt.dataclass_attr - if "." in column["type"]: - _type = column["type"].split(".")[1] + if "." in column.type: + _type = column.type.split(".")[1] else: - _type = column["type"].lower().split("[")[0] + _type = column.type.lower().split("[")[0] if self.custom_types: _type = self.add_custom_type(_type) if _type == _type: @@ -36,31 +37,32 @@ def generate_attr(self, column: Dict, defaults_off: bool) -> str: self.typing_imports.add(_type.split("[")[0]) elif "datetime" in _type: self.datetime_import = True - elif "[" in column["type"]: + self.additional_imports.add('field') + elif "[" in column.type: self.typing_imports.add("List") _type = f"List[{_type}]" if _type == "UUID": self.uuid_import = True - column_str = column_str.format(arg_name=column["name"], type=_type) - if column["default"] and defaults_off is False: + column_str = column_str.format(arg_name=column.name, type=_type) + if column.default and defaults_off is False: column_str = self.add_column_default(column_str, column) if ( - column["nullable"] - and not (column["default"] and not defaults_off) + column.nullable + and not (column.default and not defaults_off) and not defaults_off ): - column_str += pt.dataclass_default_attr.format(default=None) + column_str += dt.dataclass_default_attr.format(default=None) return column_str @staticmethod def add_column_default(column_str: str, column: Dict) -> str: - if column["type"].upper() in datetime_types: - if "now" in column["default"]: + if column.type.upper() in datetime_types: + if "now" in column.default.lower(): # todo: need to add other popular PostgreSQL & MySQL functions - column["default"] = "datetime.datetime.now()" - elif "'" not in column["default"]: - column["default"] = f"'{column['default']}'" - column_str += pt.dataclass_default_attr.format(default=column["default"]) + column.default = dt.field_datetime_now + elif "'" not in column.default: + column.default = f"'{column['default']}'" + column_str += dt.dataclass_default_attr.format(default=column.default) return column_str def generate_model( @@ -73,43 +75,47 @@ def generate_model( **kwargs, ) -> str: model = "" - if table.get("table_name"): - # mean one model one table - model += "\n\n" - model += ( - pt.dataclass_class.format( - class_name=create_class_name( - table["table_name"], singular, exceptions - ), - table_name=table["table_name"], - ) - ) + "\n\n" - columns = {"default": [], "non_default": []} - for column in table["columns"]: - column_str = self.generate_attr(column, defaults_off) + "\n" - if "=" in column_str: - columns["default"].append(column_str) - else: - columns["non_default"].append(column_str) - for column in columns["non_default"]: - model += column - for column in columns["default"]: - model += column + + # mean one model one table + model += "\n\n" + model += ( + dt.dataclass_class.format( + class_name=create_class_name( + table.name, singular, exceptions + ), + table_name=table.name, + ) + ) + "\n\n" + columns = {"default": [], "non_default": []} + for column in table.columns: + column_str = self.generate_attr(column, defaults_off) + "\n" + if "=" in column_str: + columns["default"].append(column_str) + else: + columns["non_default"].append(column_str) + for column in columns["non_default"]: + model += column + for column in columns["default"]: + model += column return model def create_header(self, *args, **kwargs) -> str: header = "" if self.enum_imports: - header += pt.enum_import.format(enums=",".join(self.enum_imports)) + "\n" + header += dt.enum_import.format(enums=",".join(self.enum_imports)) + "\n" if self.uuid_import: - header += pt.uuid_import + "\n" + header += dt.uuid_import + "\n" if self.datetime_import: - header += pt.datetime_import + "\n" + header += dt.datetime_import + "\n" if self.typing_imports: _imports = list(self.typing_imports) _imports.sort() - header += pt.typing_imports.format(typing_types=", ".join(_imports)) + "\n" - header += pt.dataclass_imports + header += dt.typing_imports.format(typing_types=", ".join(_imports)) + "\n" + if self.additional_imports: + self.additional_imports = f', {",".join(self.additional_imports)}' + else: + self.additional_imports = '' + header += dt.dataclass_imports.format(additional_imports=self.additional_imports) return header def generate_type( @@ -117,35 +123,35 @@ def generate_type( ) -> str: """ method to prepare one Model defention - name & tablename & columns """ type_class = "" - if _type["properties"].get("values"): + if _type.properties.get("values"): # mean this is a Enum - _type["properties"]["values"].sort() - for num, value in enumerate(_type["properties"]["values"]): + _type.properties["values"].sort() + for num, value in enumerate(_type.properties["values"]): _value = value.replace("'", "") if not _value.isnumeric(): type_class += ( - pt.enum_value.format(name=value.replace("'", ""), value=value) + dt.enum_value.format(name=value.replace("'", ""), value=value) + "\n" ) sub_type = "str, Enum" self.enum_imports.add("Enum") else: type_class += ( - pt.enum_value.format( + dt.enum_value.format( name=enum_number_name_list.get(num), value=_value ) + "\n" ) sub_type = "IntEnum" self.enum_imports.add("IntEnum") - class_name = create_class_name(_type["type_name"], singular, exceptions) + class_name = create_class_name(_type.name, singular, exceptions) type_class = ( "\n\n" + ( - pt.enum_class.format(class_name=class_name, sub_type=sub_type) + dt.enum_class.format(class_name=class_name, sub_type=sub_type) + "\n\n" ) + type_class ) - self.custom_types[_type["type_name"]] = ("db.Enum", class_name) + self.custom_types[_type.name] = ("db.Enum", class_name) return type_class diff --git a/omymodels/dataclass/templates.py b/omymodels/dataclass/templates.py index 637adb2..28cf991 100644 --- a/omymodels/dataclass/templates.py +++ b/omymodels/dataclass/templates.py @@ -4,7 +4,7 @@ base_model = "BaseModel" -dataclass_imports = """from dataclasses import dataclass""" +dataclass_imports = """from dataclasses import dataclass{additional_imports}""" dataclass_class = """@dataclass class {class_name}:""" @@ -16,3 +16,4 @@ class {class_name}:""" enum_import = "from enum import {enums}" uuid_import = "from uuid import UUID" +field_datetime_now = "field(default_factory=datetime.datetime.now)" \ No newline at end of file diff --git a/omymodels/gino/core.py b/omymodels/gino/core.py index 0424247..f6f1cb8 100644 --- a/omymodels/gino/core.py +++ b/omymodels/gino/core.py @@ -26,10 +26,10 @@ def add_custom_type(self, column_data_type: str, column_type: str) -> str: def prepare_column_type(self, column_data: Dict) -> str: self.no_need_par = False column_type = type_not_found - if "." in column_data["type"]: - column_data_type = column_data["type"].split(".")[1] + if "." in column_data.type: + column_data_type = column_data.type.split(".")[1] else: - column_data_type = column_data["type"].lower().split("[")[0] + column_data_type = column_data.type.lower().split("[")[0] if self.custom_types: column_type = self.add_custom_type(column_data_type, column_type) if column_type == type_not_found: @@ -40,43 +40,43 @@ def prepare_column_type(self, column_data: Dict) -> str: if column_type in postgresql_dialect: self.postgresql_dialect_cols.add(column_type) - if column_data["size"]: + if column_data.size: column_type = self.set_column_size(column_type, column_data) elif self.no_need_par is False: column_type += "()" - if "[" in column_data["type"]: + if "[" in column_data.type: self.postgresql_dialect_cols.add("ARRAY") column_type = f"ARRAY({column_type})" column = gt.column_template.format( - column_name=column_data["name"], column_type=column_type + column_name=column_data.name, column_type=column_type ) return column @staticmethod def set_column_size(column_type, column_data): - if isinstance(column_data["size"], int): - column_type += f"({column_data['size']})" - elif isinstance(column_data["size"], tuple): - column_type += f"({','.join([str(x) for x in column_data['size']])})" + if isinstance(column_data.size, int): + column_type += f"({column_data.size})" + elif isinstance(column_data.size, tuple): + column_type += f"({','.join([str(x) for x in column_data.size])})" return column_type def prepare_column_default(self, column_data: Dict, column: str) -> str: - if isinstance(column_data["default"], str): - if column_data["type"].upper() in datetime_types: - if "now" in column_data["default"]: + if isinstance(column_data.default, str): + if column_data.type.upper() in datetime_types: + if "now" in column_data.default.lower(): # todo: need to add other popular PostgreSQL & MySQL functions - column_data["default"] = "func.now()" + column_data.default = "func.now()" self.state.add("func") - elif "'" not in column_data["default"]: - column_data["default"] = f"'{column_data['default']}'" + elif "'" not in column_data.default: + column_data.default = f"'{column_data.default}'" else: - if "'" not in column_data["default"]: - column_data["default"] = f"'{column_data['default']}'" + if "'" not in column_data.default: + column_data.default = f"'{column_data.default}'" else: - column_data["default"] = f"'{str(column_data['default'])}'" - column += gt.default.format(default=column_data["default"]) + column_data.default = f"'{str(column_data.default)}'" + column += gt.default.format(default=column_data.default) return column def setup_column_attributes( @@ -84,33 +84,33 @@ def setup_column_attributes( ) -> str: if ( - column_data["type"].lower() == "serial" - or column_data["type"].lower() == "bigserial" + column_data.type.lower() == "serial" + or column_data.type.lower() == "bigserial" ): column += gt.autoincrement - if column_data["references"]: + if column_data.references: column = self.add_reference_to_the_column( - column_data["name"], column, column_data["references"] + column_data.name, column, column_data.references ) - if not column_data["nullable"] and not column_data["name"] in table_pk: + if not column_data.nullable and not column_data.name in table_pk: column += gt.required - if column_data["default"] is not None: + if column_data.default is not None: column = self.prepare_column_default(column_data, column) - if column_data["name"] in table_pk: + if column_data.name in table_pk: column += gt.pk_template - if column_data["unique"]: + if column_data.unique: column += gt.unique - if "columns" in table_data["alter"]: - for alter_column in table_data["alter"]["columns"]: + if "columns" in table_data.alter: + for alter_column in table_data.alter["columns"]: if ( - alter_column["name"] == column_data["name"] + alter_column['name'] == column_data.name and not alter_column["constraint_name"] and alter_column["references"] ): column = self.add_reference_to_the_column( - alter_column["name"], column, alter_column["references"] + alter_column['name'], column, alter_column["references"] ) return column @@ -143,8 +143,8 @@ def add_table_args( self, model: str, table: Dict, schema_global: bool = True ) -> str: statements = [] - if table.get("index"): - for index in table["index"]: + if table.indexes: + for index in table.indexes: if not index["unique"]: self.im_index = True @@ -162,8 +162,8 @@ def add_table_args( name=f"'{index['index_name']}'", ) ) - if not schema_global and table["schema"]: - statements.append(gt.schema.format(schema_name=table["schema"])) + if not schema_global and table.table_schema: + statements.append(gt.schema.format(schema_name=table.table_schema)) if statements: model += gt.table_args.format(statements=",".join(statements)) return model @@ -174,10 +174,10 @@ def generate_type( """ method to prepare one Model defention - name & tablename & columns """ type_class = "" - if _type["properties"].get("values"): + if _type.properties.get("values"): # mean this is a Enum - _type["properties"]["values"].sort() - for num, value in enumerate(_type["properties"]["values"]): + _type.properties["values"].sort() + for num, value in enumerate(_type.properties["values"]): _value = value.replace("'", "") if not _value.isnumeric(): type_class += ( @@ -195,14 +195,14 @@ def generate_type( ) sub_type = "IntEnum" self.enum_imports.add("IntEnum") - class_name = create_class_name(_type["type_name"], singular, exceptions) + class_name = create_class_name(_type.name, singular, exceptions) type_class = ( "\n\n" + (gt.enum_class.format(class_name=class_name, type=sub_type) + "\n") + "\n" + type_class ) - self.custom_types[_type["type_name"]] = ("db.Enum", class_name) + self.custom_types[_type.name] = ("db.Enum", class_name) return type_class def generate_model( @@ -216,23 +216,20 @@ def generate_model( ) -> str: """ method to prepare one Model defention - name & tablename & columns """ model = "" - if table.get("table_name"): - model = gt.model_template.format( - model_name=create_class_name(table["table_name"], singular, exceptions), - table_name=table["table_name"], - ) - for column in table["columns"]: - model += self.generate_column(column, table["primary_key"], table) + model = gt.model_template.format( + model_name=create_class_name(table.name, singular, exceptions), + table_name=table.name, + ) + for column in table.columns: + model += self.generate_column(column, table.primary_key, table) if ( - table.get("index") - or table.get("alter") - or table.get("checks") + table.indexes + or table.alter + or table.checks or not schema_global ): model = self.add_table_args(model, table, schema_global) - elif table.get("sequence_name"): - # create sequence - ... + # create sequence return model def create_header(self, tables: List[Dict], schema: bool = False) -> str: @@ -253,8 +250,8 @@ def create_header(self, tables: List[Dict], schema: bool = False) -> str: header += gt.unique_cons_import + "\n" if self.im_index: header += gt.index_import + "\n" - if schema and tables[0]["schema"]: - schema = tables[0]["schema"].replace('"', "") + if schema and tables[0].table_schema: + schema = tables[0].table_schema.replace('"', "") header += "\n" + gt.gino_init_schema.format(schema=schema) else: header += "\n" + gt.gino_init diff --git a/omymodels/meta_model.py b/omymodels/meta_model.py new file mode 100644 index 0000000..872d784 --- /dev/null +++ b/omymodels/meta_model.py @@ -0,0 +1,47 @@ +from pydantic import BaseModel, Field, validator +from typing import List, Optional, Union + + +class TableProperties(BaseModel): + indexes: List + + +class Column(BaseModel): + + name: str + type: str + size: Optional[Union[str, int, tuple]] + primary_key: bool = False + unique: bool = False + default: Optional[str] + nullable: bool = True + identifier: Optional[bool] + generated_as: Optional[str] + other_properties: Optional[dict] + references: Optional[dict] + + @validator('size') + def size_must_contain_space(cls, v): + if isinstance(v, str) and v.isnumeric(): + return int(v) + return v + + +class TableMeta(BaseModel): + name: str = Field(alias='table_name') + table_schema: Optional[str] = Field(alias='schema') + columns: List[Column] + indexes: Optional[List[dict]] = Field(alias='index') + alter: Optional[dict] + checks: Optional[List[dict]] + properties: Optional[TableProperties] + primary_key: List + + class Config: + arbitrary_types_allowed = True + + +class Type(BaseModel): + name: str = Field(alias='type_name') + base_type: str + properties: Optional[dict] \ No newline at end of file diff --git a/omymodels/pydantic/core.py b/omymodels/pydantic/core.py index 3f067e6..e135a32 100644 --- a/omymodels/pydantic/core.py +++ b/omymodels/pydantic/core.py @@ -23,15 +23,15 @@ def add_custom_type(self, _type): return _type def generate_attr(self, column: Dict, defaults_off: bool) -> str: - if column["nullable"]: + if column.nullable: self.typing_imports.add("Optional") column_str = pt.pydantic_optional_attr else: column_str = pt.pydantic_attr - if "." in column["type"]: - _type = column["type"].split(".")[1] + if "." in column.type: + _type = column.type.split(".")[1] else: - _type = column["type"].lower().split("[")[0] + _type = column.type.lower().split("[")[0] if self.custom_types: _type = self.add_custom_type(_type) if _type == _type: @@ -40,28 +40,28 @@ def generate_attr(self, column: Dict, defaults_off: bool) -> str: self.imports.add(_type) elif "datetime" in _type: self.datetime_import = True - elif "[" in column["type"]: + elif "[" in column.type: self.typing_imports.add("List") _type = f"List[{_type}]" if _type == "UUID": self.uuid_import = True - column_str = column_str.format(arg_name=column["name"], type=_type) + column_str = column_str.format(arg_name=column.name, type=_type) - if column["default"] and defaults_off is False: + if column.default and defaults_off is False: column_str = self.add_default_values(column_str, column) return column_str @staticmethod def add_default_values(column_str: str, column: Dict) -> str: - if column["type"].upper() in datetime_types: - if "now" in column["default"]: + if column.type.upper() in datetime_types: + if "now" in column.default.lower(): # todo: need to add other popular PostgreSQL & MySQL functions - column["default"] = "datetime.datetime.now()" - elif "'" not in column["default"]: - column["default"] = f"'{column['default']}'" - column_str += pt.pydantic_default_attr.format(default=column["default"]) + column.default = "datetime.datetime.now()" + elif "'" not in column.default: + column.default = f"'{column['default']}'" + column_str += pt.pydantic_default_attr.format(default=column.default) return column_str def generate_model( @@ -74,20 +74,19 @@ def generate_model( **kwargs, ) -> str: model = "" - if table.get("table_name"): - # mean one model one table - model += "\n\n" - model += ( - pt.pydantic_class.format( - class_name=create_class_name( - table["table_name"], singular, exceptions - ), - table_name=table["table_name"], - ) - ) + "\n\n" + # mean one model one table + model += "\n\n" + model += ( + pt.pydantic_class.format( + class_name=create_class_name( + table.name, singular, exceptions + ), + table_name=table.name, + ) + ) + "\n\n" - for column in table["columns"]: - model += self.generate_attr(column, defaults_off) + "\n" + for column in table.columns: + model += self.generate_attr(column, defaults_off) + "\n" return model @@ -115,10 +114,10 @@ def generate_type( ) -> str: """ method to prepare one Model defention - name & tablename & columns """ type_class = "" - if _type["properties"].get("values"): + if _type.properties.get("values"): # mean this is a Enum - _type["properties"]["values"].sort() - for num, value in enumerate(_type["properties"]["values"]): + _type.properties["values"].sort() + for num, value in enumerate(_type.properties["values"]): _value = value.replace("'", "") if not _value.isnumeric(): type_class += ( @@ -136,7 +135,7 @@ def generate_type( ) sub_type = "IntEnum" self.enum_imports.add("IntEnum") - class_name = create_class_name(_type["type_name"], singular, exceptions) + class_name = create_class_name(_type.name, singular, exceptions) type_class = ( "\n\n" + ( @@ -145,5 +144,5 @@ def generate_type( ) + type_class ) - self.custom_types[_type["type_name"]] = ("db.Enum", class_name) + self.custom_types[_type.name] = ("db.Enum", class_name) return type_class diff --git a/omymodels/sqlalchemy/core.py b/omymodels/sqlalchemy/core.py index 14bbc2f..0c76905 100644 --- a/omymodels/sqlalchemy/core.py +++ b/omymodels/sqlalchemy/core.py @@ -26,10 +26,10 @@ def add_custom_type(self, column_data_type: str, column_type: str) -> str: def prepare_column_type(self, column_data: Dict) -> str: self.no_need_par = False column_type = type_not_found - if "." in column_data["type"]: - column_data_type = column_data["type"].split(".")[1] + if "." in column_data.type: + column_data_type = column_data.type.split(".")[1] else: - column_data_type = column_data["type"].lower().split("[")[0] + column_data_type = column_data.type.lower().split("[")[0] if self.custom_types: column_type = self.add_custom_type(column_data_type, column_type) @@ -41,42 +41,42 @@ def prepare_column_type(self, column_data: Dict) -> str: if column_type in postgresql_dialect: self.postgresql_dialect_cols.add(column_type) - if column_data["size"]: + if column_data.size: column_type = self.set_column_size(column_type, column_data) elif self.no_need_par is False: column_type += "()" - if "[" in column_data["type"]: + if "[" in column_data.type: self.postgresql_dialect_cols.add("ARRAY") column_type = f"ARRAY({column_type})" column = st.column_template.format( - column_name=column_data["name"], column_type=column_type + column_name=column_data.name, column_type=column_type ) return column @staticmethod def set_column_size(column_type: str, column_data: Dict) -> str: - if isinstance(column_data["size"], int): - column_type += f"({column_data['size']})" - elif isinstance(column_data["size"], tuple): - column_type += f"({','.join([str(x) for x in column_data['size']])})" + if isinstance(column_data.size, int): + column_type += f"({column_data.size})" + elif isinstance(column_data.size, tuple): + column_type += f"({','.join([str(x) for x in column_data.size])})" return column_type def prepare_column_default(self, column_data: Dict, column: str) -> str: - if isinstance(column_data["default"], str): - if column_data["type"].upper() in datetime_types: - if "now" in column_data["default"]: + if isinstance(column_data.default, str): + if column_data.type.upper() in datetime_types: + if "now" in column_data.default.lower(): # todo: need to add other popular PostgreSQL & MySQL functions - column_data["default"] = "func.now()" + column_data.default = "func.now()" self.state.add("func") - elif "'" not in column_data["default"]: - column_data["default"] = f"'{column_data['default']}'" + elif "'" not in column_data.default: + column_data.default = f"'{column_data.default}'" else: - if "'" not in column_data["default"]: - column_data["default"] = f"'{column_data['default']}'" + if "'" not in column_data.default: + column_data.default = f"'{column_data.default}'" else: - column_data["default"] = f"'{str(column_data['default'])}'" - column += st.default.format(default=column_data["default"]) + column_data.default = f"'{str(column_data.default)}'" + column += st.default.format(default=column_data.default) return column def setup_column_attributes( @@ -84,33 +84,33 @@ def setup_column_attributes( ) -> str: if ( - column_data["type"].lower() == "serial" - or column_data["type"].lower() == "bigserial" + column_data.type.lower() == "serial" + or column_data.type.lower() == "bigserial" ): column += st.autoincrement - if column_data["references"]: + if column_data.references: column = self.add_reference_to_the_column( - column_data["name"], column, column_data["references"] + column_data.name, column, column_data.references ) - if not column_data["nullable"] and not column_data["name"] in table_pk: + if not column_data.nullable and not column_data.name in table_pk: column += st.required - if column_data["default"] is not None: + if column_data.default is not None: column = self.prepare_column_default(column_data, column) - if column_data["name"] in table_pk: + if column_data.name in table_pk: column += st.pk_template - if column_data["unique"]: + if column_data.unique: column += st.unique - if "columns" in table_data["alter"]: - for alter_column in table_data["alter"]["columns"]: + if "columns" in table_data.alter: + for alter_column in table_data.alter["columns"]: if ( - alter_column["name"] == column_data["name"] + alter_column['name'] == column_data.name and not alter_column["constraint_name"] and alter_column["references"] ): column = self.add_reference_to_the_column( - alter_column["name"], column, alter_column["references"] + alter_column['name'], column, alter_column["references"] ) return column @@ -141,8 +141,8 @@ def add_table_args( self, model: str, table: Dict, schema_global: bool = True ) -> str: statements = [] - if table.get("index"): - for index in table["index"]: + if table.indexes: + for index in table.indexes: if not index["unique"]: self.im_index = True @@ -160,8 +160,8 @@ def add_table_args( name=f"'{index['index_name']}'", ) ) - if not schema_global and table["schema"]: - statements.append(st.schema.format(schema_name=table["schema"])) + if not schema_global and table.table_schema: + statements.append(st.schema.format(schema_name=table.table_schema)) if statements: model += st.table_args.format(statements=",".join(statements)) return model @@ -172,10 +172,10 @@ def generate_type( """ method to prepare one Model defention - name & tablename & columns """ type_class = "" - if _type["properties"].get("values"): + if _type.properties.get("values"): # mean this is a Enum - _type["properties"]["values"].sort() - for num, value in enumerate(_type["properties"]["values"]): + _type.properties["values"].sort() + for num, value in enumerate(_type.properties["values"]): _value = value.replace("'", "") if not _value.isnumeric(): type_class += ( @@ -193,14 +193,14 @@ def generate_type( ) sub_type = "IntEnum" self.enum_imports.add("IntEnum") - class_name = create_class_name(_type["type_name"], singular, exceptions) + class_name = create_class_name(_type.name, singular, exceptions) type_class = ( "\n\n" + (st.enum_class.format(class_name=class_name, type=sub_type) + "\n") + "\n" + type_class ) - self.custom_types[_type["type_name"]] = ("sa.Enum", class_name) + self.custom_types[_type.name] = ("sa.Enum", class_name) return type_class def generate_model( @@ -214,17 +214,17 @@ def generate_model( ) -> str: """ method to prepare one Model defention - name & tablename & columns """ model = "" - if table.get("table_name"): - model = st.model_template.format( - model_name=create_class_name(table["table_name"], singular, exceptions), - table_name=table["table_name"], - ) - for column in table["columns"]: - model += self.generate_column(column, table["primary_key"], table) + + model = st.model_template.format( + model_name=create_class_name(table.name, singular, exceptions), + table_name=table.name, + ) + for column in table.columns: + model += self.generate_column(column, table.primary_key, table) if ( - table.get("index") - or table.get("alter") - or table.get("checks") + table.indexes + or table.alter + or table.checks or not schema_global ): model = self.add_table_args(model, table, schema_global) diff --git a/omymodels/sqlalchemy_core/core.py b/omymodels/sqlalchemy_core/core.py index 37702c8..f5b3692 100644 --- a/omymodels/sqlalchemy_core/core.py +++ b/omymodels/sqlalchemy_core/core.py @@ -6,6 +6,7 @@ datetime_types, ) from omymodels.utils import create_class_name, type_not_found, enum_number_name_list +from omymodels.meta_model import Column class ModelGenerator: @@ -31,10 +32,10 @@ def prepare_column_type(self, column_data: Dict) -> str: """ extract and map column type """ self.no_need_par = False column_type = type_not_found - if "." in column_data["type"]: - column_data_type = column_data["type"].split(".")[1] + if "." in column_data.type: + column_data_type = column_data.type.split(".")[1] else: - column_data_type = column_data["type"].lower().split("[")[0] + column_data_type = column_data.type.lower().split("[")[0] if self.custom_types: column_type = self.add_custom_type(column_data_type, column_type) if column_type == type_not_found: @@ -43,12 +44,12 @@ def prepare_column_type(self, column_data: Dict) -> str: self.postgresql_dialect_cols.add(column_type) if column_type == "UUID": self.no_need_par = True - if column_data["size"]: - column_type = self.add_size_to_column_type(column_data["size"]) + if column_data.size: + column_type = self.add_size_to_column_type(column_data.size) elif self.no_need_par is False: column_type += "()" - if "[" in column_data["type"]: + if "[" in column_data.type: self.postgresql_dialect_cols.add("ARRAY") column_type = f"ARRAY({column_type})" return column_type @@ -62,20 +63,20 @@ def add_size_to_column_type(size): def column_default(self, column_data: Dict) -> str: """ extract & format column default values """ - if isinstance(column_data["default"], str): - if column_data["type"].upper() in datetime_types: - if "now" in column_data["default"]: + if isinstance(column_data.default, str): + if column_data.type.upper() in datetime_types: + if "now" in column_data.default.lower(): # todo: need to add other popular PostgreSQL & MySQL functions - column_data["default"] = "func.now()" + column_data.default = "func.now()" self.state.add("func") - elif "'" not in column_data["default"]: - column_data["default"] = f"'{column_data['default']}'" + elif "'" not in column_data.default: + column_data.default = f"'{column_data.default}'" else: - if "'" not in column_data["default"]: - column_data["default"] = f"'{column_data['default']}'" + if "'" not in column_data.default: + column_data.default = f"'{column_data.default}'" else: - column_data["default"] = f"'{str(column_data['default'])}'" - default_property = st.default.format(default=column_data["default"]) + column_data.default = f"'{str(column_data.default)}'" + default_property = st.default.format(default=column_data.default) return default_property def get_column_attributes( @@ -83,33 +84,33 @@ def get_column_attributes( ) -> List[str]: properties = [] if ( - column_data["type"].lower() == "serial" - or column_data["type"].lower() == "bigserial" + column_data.type.lower() == "serial" + or column_data.type.lower() == "bigserial" ): properties.append(st.autoincrement) - if column_data["references"]: + if column_data.references: properties.append( - self.column_reference(column_data["name"], column_data["references"]) + self.column_reference(column_data.name, column_data.references) ) - if not column_data["nullable"] and not column_data["name"] in table_pk: + if not column_data.nullable and not column_data.name in table_pk: properties.append(st.required) - if column_data["default"] is not None: + if column_data.default is not None: properties.append(self.column_default(column_data)) - if column_data["name"] in table_pk: + if column_data.name in table_pk: properties.append(st.pk_template) - if column_data["unique"]: + if column_data.unique: properties.append(st.unique) - if "columns" in table_data["alter"]: - for alter_column in table_data["alter"]["columns"]: + if "columns" in table_data.alter: + for alter_column in table_data.alter["columns"]: if ( - alter_column["name"] == column_data["name"] + alter_column['name'] == column_data.name and not alter_column["constraint_name"] and alter_column["references"] - and not column_data["references"] + and not column_data.references ): properties.append( self.column_reference( - alter_column["name"], alter_column["references"] + alter_column['name'], alter_column["references"] ) ) return properties @@ -127,7 +128,7 @@ def column_reference(column_name: str, reference: Dict[str, str]) -> str: return ref_property def generate_column( - self, column_data: Dict, table_pk: List[str], table_data: Dict + self, column_data: Column, table_pk: List[str], table_data: Dict ) -> str: """ method to generate full column defention """ column_type = self.prepare_column_type(column_data) @@ -136,7 +137,7 @@ def generate_column( ) column = st.column_template.format( - column_name=column_data["name"], + column_name=column_data.name, column_type=column_type, properties=properties, ) @@ -147,8 +148,8 @@ def get_indexes_and_unique( ) -> str: indexes = [] unique_constr = [] - if table.get("index"): - for index in table["index"]: + if table.indexes: + for index in table.indexes: if not index["unique"]: self.im_index = True indexes.append( @@ -180,10 +181,10 @@ def generate_type( """ method to prepare one Model defention - name & tablename & columns """ type_class = "" - if _type["properties"].get("values"): + if _type.properties.get("values"): # mean this is a Enum - _type["properties"]["values"].sort() - for num, value in enumerate(_type["properties"]["values"]): + _type.properties["values"].sort() + for num, value in enumerate(_type.properties["values"]): _value = value.replace("'", "") if not _value.isnumeric(): type_class += ( @@ -201,48 +202,47 @@ def generate_type( ) sub_type = "IntEnum" self.enum_imports.add("IntEnum") - class_name = create_class_name(_type["type_name"], singular, exceptions) + class_name = create_class_name(_type.name, singular, exceptions) type_class = ( "\n\n" + (st.enum_class.format(class_name=class_name, type=sub_type) + "\n") + "\n" + type_class ) - self.custom_types[_type["type_name"]] = ("sa.Enum", class_name) + self.custom_types[_type.name] = ("sa.Enum", class_name) return type_class def generate_model(self, data: Dict, *args, **kwargs) -> str: """ method to prepare one Model defention - name & tablename & columns """ model = "" - if data.get("table_name"): - # mean this is a table - table = data - columns = "" - - for column in table["columns"]: - columns += self.generate_column(column, table["primary_key"], table) + # mean this is a table + table = data + columns = "" - table_var_name = table["table_name"].replace("-", "_") + for column in table.columns: + columns += self.generate_column(column, table.primary_key, table) - indexes = [] - constraints = None + table_var_name = table.name.replace("-", "_") - if table.get("index") or table.get("alter") or table.get("checks"): - indexes, constraints = self.get_indexes_and_unique( - model, table, table_var_name - ) + indexes = [] + constraints = None - model = st.table_template.format( - table_var=table_var_name, - table_name=table["table_name"], - columns=columns, - schema="" - if not table.get("schema") - else st.schema.format(schema_name=table["schema"]), - constraints=", ".join(constraints) if constraints else "", + if table.indexes or table.alter or table.checks: + indexes, constraints = self.get_indexes_and_unique( + model, table, table_var_name ) - for index in indexes: - model += index + + model = st.table_template.format( + table_var=table_var_name, + table_name=table.name, + columns=columns, + schema="" + if not table.table_schema + else st.schema.format(schema_name=table.table_schema), + constraints=", ".join(constraints) if constraints else "", + ) + for index in indexes: + model += index return model def create_header(self, tables: List[Dict], schema: bool = False) -> str: diff --git a/poetry.lock b/poetry.lock index 44a7643..35f6867 100644 --- a/poetry.lock +++ b/poetry.lock @@ -268,6 +268,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyparsing = ">=2.0.2" +[[package]] +name = "parsimonious" +version = "0.8.1" +description = "(Soon to be) the fastest pure-Python PEG parser I could muster" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.9.0" + [[package]] name = "pathspec" version = "0.8.1" @@ -317,6 +328,17 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "py-models-parser" +version = "0.5.0" +description = "Parser for Different Python Models (Pydantic, Enums, ORMs: Tortoise, SqlAlchemy, GinoORM, PonyORM, Pydal tables) to extract information about columns(attrs), model, table args,etc in one format." +category = "main" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +parsimonious = ">=0.8.1,<0.9.0" + [[package]] name = "pycparser" version = "2.20" @@ -464,7 +486,7 @@ ply = ">=3.11,<4.0" name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" @@ -568,7 +590,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "21e28996290e001142b67c171a11717aed59376f24b9d3cb5e213ef0b030fff9" +content-hash = "83789278aa9f30256298d8423bf834d02c1b9653ca93dcc8db6c07d75bde3466" [metadata.files] appdirs = [ @@ -742,6 +764,9 @@ packaging = [ {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"}, {file = "packaging-20.9.tar.gz", hash = "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5"}, ] +parsimonious = [ + {file = "parsimonious-0.8.1.tar.gz", hash = "sha256:3add338892d580e0cb3b1a39e4a1b427ff9f687858fdd61097053742391a9f6b"}, +] pathspec = [ {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, @@ -762,6 +787,10 @@ py = [ {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] +py-models-parser = [ + {file = "py-models-parser-0.5.0.tar.gz", hash = "sha256:e6e0553463aa0560eb4d849a25d852d7a926427b7e70698dcde1661b0e758ac0"}, + {file = "py_models_parser-0.5.0-py3-none-any.whl", hash = "sha256:7319b05598288a8588dcd57497572f51ea11a4dae57ed5597a749b2e63127525"}, +] pycparser = [ {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, diff --git a/pyproject.toml b/pyproject.toml index 68ddb4f..815a498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "omymodels" -version = "0.8.1" -description = "O! My Models (omymodels) is a library to generate Python Models for SQLAlchemy (ORM & Core), GinoORM, Pydantic & Python Dataclasses from SQL DDL ." +version = "0.8.2" +description = "O! My Models (omymodels) is a library to generate Python Models for SQLAlchemy (ORM & Core), GinoORM, Pydantic, Pydal tables & Python Dataclasses from SQL DDL. OMyModels also allow you to covert one type of Models to another. Right supported conversion from SQLAlchemy to Pydal tables." authors = ["Iuliia Volkova "] license = "MIT" readme = "docs/README.rst" @@ -21,6 +21,7 @@ classifiers = [ python = "^3.6" simple-ddl-parser = "0.16.0" Jinja2 = "^3.0.1" +py-models-parser = "^0.5.0" [tool.poetry.dev-dependencies] pytest = "^5.2" diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 28624c4..cd178ac 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -50,7 +50,7 @@ def test_defaults_datetime(): """ result = create_models(ddl, models_type="dataclass") expected = """import datetime -from dataclasses import dataclass +from dataclasses import dataclass, field @dataclass @@ -61,7 +61,7 @@ class UserHistory: status: str runid: int = None job_id: int = None - event_time: datetime.datetime = datetime.datetime.now() + event_time: datetime.datetime = field(default_factory=datetime.datetime.now) comment: str = 'none' """ assert expected == result["code"] @@ -71,7 +71,7 @@ def test_enums_in_dataclasses(): expected = """from enum import Enum import datetime from typing import Union -from dataclasses import dataclass +from dataclasses import dataclass, field class MaterialType(str, Enum): @@ -89,7 +89,7 @@ class Material: description: str = None type: MaterialType = None additional_properties: Union[dict, list] = None - created_at: datetime.datetime = datetime.datetime.now() + created_at: datetime.datetime = field(default_factory=datetime.datetime.now) updated_at: datetime.datetime = None """ ddl = """ @@ -117,7 +117,7 @@ def test_defaults_off(): expected = """from enum import Enum import datetime from typing import Union -from dataclasses import dataclass +from dataclasses import dataclass, field class MaterialType(str, Enum): @@ -157,3 +157,49 @@ class Material: """ result = create_models(ddl, models_type="dataclass", defaults_off=True) assert expected == result["code"] + + +def test_upper_now_produces_same_result(): + expected = """from enum import Enum +import datetime +from typing import Union +from dataclasses import dataclass, field + + +class MaterialType(str, Enum): + + article = 'article' + video = 'video' + + +@dataclass +class Material: + + id: int + title: str + link: str + description: str = None + type: MaterialType = None + additional_properties: Union[dict, list] = None + created_at: datetime.datetime = field(default_factory=datetime.datetime.now) + updated_at: datetime.datetime = None +""" + ddl = """ +CREATE TYPE "material_type" AS ENUM ( + 'video', + 'article' +); + +CREATE TABLE "material" ( + "id" SERIAL PRIMARY KEY, + "title" varchar NOT NULL, + "description" text, + "link" varchar NOT NULL, + "type" material_type, + "additional_properties" json, + "created_at" timestamp DEFAULT (NOW()), + "updated_at" timestamp +); +""" + result = create_models(ddl, models_type="dataclass")["code"] + assert expected == result diff --git a/tests/test_sqlalchemy.py b/tests/test_sqlalchemy.py index aedf247..6303882 100644 --- a/tests/test_sqlalchemy.py +++ b/tests/test_sqlalchemy.py @@ -122,3 +122,54 @@ class Attachments(Base): """ result = create_models(ddl, models_type="sqlalchemy")["code"] assert result == expected + + +def test_upper_name_produces_the_same_result(): + expected = """import sqlalchemy as sa +from sqlalchemy.ext.declarative import declarative_base +from enum import Enum +from sqlalchemy.sql import func +from sqlalchemy.dialects.postgresql import JSON + + +Base = declarative_base() + + +class MaterialType(Enum): + + article = 'article' + video = 'video' + + +class Material(Base): + + __tablename__ = 'material' + + id = sa.Column(sa.Integer(), autoincrement=True, primary_key=True) + title = sa.Column(sa.String(), nullable=False) + description = sa.Column(sa.Text()) + link = sa.Column(sa.String(), nullable=False) + type = sa.Column(sa.Enum(MaterialType)) + additional_properties = sa.Column(JSON(), server_default='{"key": "value"}') + created_at = sa.Column(sa.TIMESTAMP(), server_default=func.now()) + updated_at = sa.Column(sa.TIMESTAMP()) +""" + ddl = """ +CREATE TYPE "material_type" AS ENUM ( + 'video', + 'article' +); + +CREATE TABLE "material" ( + "id" SERIAL PRIMARY KEY, + "title" varchar NOT NULL, + "description" text, + "link" varchar NOT NULL, + "type" material_type, + "additional_properties" json DEFAULT '{"key": "value"}', + "created_at" timestamp DEFAULT (NOW()), + "updated_at" timestamp +); +""" + result = create_models(ddl, models_type="sqlalchemy") + assert expected == result["code"] \ No newline at end of file