From 762fea7e6e516fa6afa559b6ff5cbb3f99fc3f70 Mon Sep 17 00:00:00 2001 From: xnuinside Date: Mon, 31 Oct 2022 20:19:04 +0300 Subject: [PATCH] new fixes for bunch of issues --- CHANGELOG.txt | 8 ++ README.md | 8 ++ docs/README.rst | 11 ++ pyproject.toml | 2 +- simple_ddl_parser/parser.py | 9 +- tests/test_simple_ddl_parser.py | 193 ++++++++++++++++++++++++++++++++ 6 files changed, 227 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index f317a64..4ce1733 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +**v0.28.1** +Imporvements: +1. Lines started with INSERT INTO statement now successfully ignored by parser (so you can keep them in ddl - they will be just skiped) + +Fixes: +1. Important fix for multiline comments + + **v0.28.0** Important Changes (Pay attention): diff --git a/README.md b/README.md index add2b10..c0a76cb 100644 --- a/README.md +++ b/README.md @@ -484,6 +484,14 @@ https://github.com/swiatek25 ## Changelog +**v0.28.1** +Imporvements: +1. Lines started with INSERT INTO statement now successfully ignored by parser (so you can keep them in ddl - they will be just skiped) + +Fixes: +1. Important fix for multiline comments + + **v0.28.0** Important Changes (Pay attention): diff --git a/docs/README.rst b/docs/README.rst index 7f34d18..0c6486b 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -552,6 +552,17 @@ https://github.com/swiatek25 Changelog --------- +**v0.28.1** +Imporvements: + + +#. Lines started with INSERT INTO statement now successfully ignored by parser (so you can keep them in ddl - they will be just skiped) + +Fixes: + + +#. Important fix for multiline comments + **v0.28.0** Important Changes (Pay attention): diff --git a/pyproject.toml b/pyproject.toml index 224b66b..11f056e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "simple-ddl-parser" -version = "0.28.0" +version = "0.28.1" description = "Simple DDL Parser to parse SQL & dialects like HQL, TSQL (MSSQL), Oracle, AWS Redshift, Snowflake, MySQL, PostgreSQL, etc ddl files to json/python dict with full information about columns: types, defaults, primary keys, etc.; sequences, alters, custom types & other entities from ddl." authors = ["Iuliia Volkova "] license = "MIT" diff --git a/simple_ddl_parser/parser.py b/simple_ddl_parser/parser.py index 313750d..5de9c8d 100755 --- a/simple_ddl_parser/parser.py +++ b/simple_ddl_parser/parser.py @@ -95,9 +95,12 @@ def __init__( self.comments = [] def catch_comment_or_process_line(self, code_line: str) -> str: - if self.multi_line_comment and CL_COM not in self.line: + if self.multi_line_comment: self.comments.append(self.line) + if CL_COM in self.line: + self.multi_line_comment = False return '' + elif not ( self.line.strip().startswith(MYSQL_COM) or self.line.strip().startswith(IN_COM) @@ -218,7 +221,7 @@ def check_new_statement_start(self, line: str) -> bool: return self.new_statement def check_line_on_skip_words(self) -> bool: - skip_regex = r"^(GO|USE)\b" + skip_regex = r"^(GO|USE|INSERT)\b" self.skip = False @@ -272,7 +275,7 @@ def process_line( final_line = self.line.endswith(";") and not self.set_was_in_line self.add_line_to_statement() - if final_line or self.new_statement: + if (final_line or self.new_statement) and self.statement: # end of sql operation, remove ; from end of line self.statement = self.statement[:-1] elif last_line and not self.skip: diff --git a/tests/test_simple_ddl_parser.py b/tests/test_simple_ddl_parser.py index 094a799..8fd67fa 100644 --- a/tests/test_simple_ddl_parser.py +++ b/tests/test_simple_ddl_parser.py @@ -2897,3 +2897,196 @@ def test_floats(): 'types': []} assert expected == results + + +def test_fix_multiline_comments_not_joined_with_table(): + result = DDLParser( +"""/************************ +@Author: Azat Erol +Always happy coding! +************************/ + +CREATE TABLE Kunde ( + KundenID INT PRIMARY KEY, + KundenName VARCHAR(40), + AbteilungID INT + FOREIGN KEY(AbteilungID) REFERENCES Abteilung(AbteilungID) ON DELETE SET NULL +); +""", normalize_names=True).run(group_by_type=True) + expected = {'comments': ['***********************', + '@Author: Azat Erol', + 'Always happy coding!', + '************************/'], + 'ddl_properties': [], + 'domains': [], + 'schemas': [], + 'sequences': [], + 'tables': [{'alter': {}, + 'checks': [], + 'columns': [{'check': None, + 'default': None, + 'name': 'KundenID', + 'nullable': False, + 'references': None, + 'size': None, + 'type': 'INT', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'KundenName', + 'nullable': True, + 'references': None, + 'size': 40, + 'type': 'VARCHAR', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'AbteilungID', + 'nullable': True, + 'references': {'column': 'AbteilungID', + 'deferrable_initially': None, + 'on_delete': 'SET', + 'on_update': None, + 'schema': None, + 'table': 'Abteilung'}, + 'size': None, + 'type': 'INT', + 'unique': False}], + 'index': [], + 'partitioned_by': [], + 'primary_key': ['KundenID'], + 'schema': None, + 'table_name': 'Kunde', + 'tablespace': None}], + 'types': []} + assert expected == result + + +def test_inserts_skipped_validly(): + result = DDLParser( +""" +INSERT INTO "autofill_profiles" VALUES('Jim Johnson, 789 4th Street',1,'Jim','','Johnson','jim@acme.com','Acme Inc.','789 4th Street','Apt. #4','San Francisco','CA','94102','USA','4155512255','4155512233',1287508123); +INSERT INTO "autofill_profiles" VALUES('Billy Jean, 1 Ghost Blvd.',3,'Billy','','Jean','billy@thriller.com','Thriller Inc.','1 Ghost Blvd.','','Santa Monica','CA','98990','USA','4431110000','',1287508123); +CREATE TABLE credit_cards ( label VARCHAR, unique_id INTEGER PRIMARY KEY, name_on_card VARCHAR, type VARCHAR, card_number VARCHAR, expiration_month INTEGER, expiration_year INTEGER, verification_code VARCHAR, billing_address VARCHAR, shipping_address VARCHAR, card_number_encrypted BLOB, verification_code_encrypted BLOB, date_modified INTEGER NOT NULL DEFAULT 0); + +COMMIT; +""", normalize_names=True).run(group_by_type=True) + expected = {'ddl_properties': [], + 'domains': [], + 'schemas': [], + 'sequences': [], + 'tables': [{'alter': {}, + 'checks': [], + 'columns': [{'check': None, + 'default': None, + 'name': 'label', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'VARCHAR', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'unique_id', + 'nullable': False, + 'references': None, + 'size': None, + 'type': 'INTEGER', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'name_on_card', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'VARCHAR', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'type', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'VARCHAR', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'card_number', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'VARCHAR', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'expiration_month', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'INTEGER', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'expiration_year', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'INTEGER', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'verification_code', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'VARCHAR', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'billing_address', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'VARCHAR', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'shipping_address', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'VARCHAR', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'card_number_encrypted', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'BLOB', + 'unique': False}, + {'check': None, + 'default': None, + 'name': 'verification_code_encrypted', + 'nullable': True, + 'references': None, + 'size': None, + 'type': 'BLOB', + 'unique': False}, + {'check': None, + 'default': 0, + 'name': 'date_modified', + 'nullable': False, + 'references': None, + 'size': None, + 'type': 'INTEGER', + 'unique': False}], + 'index': [], + 'partitioned_by': [], + 'primary_key': ['unique_id'], + 'schema': None, + 'table_name': 'credit_cards', + 'tablespace': None}], + 'types': []} + assert expected == result + \ No newline at end of file