From a8160cd682580f7a2096c536a8371cbdc64d7b4b Mon Sep 17 00:00:00 2001 From: James Kent Date: Tue, 11 Jul 2023 13:12:45 -0400 Subject: [PATCH 1/2] Find foreign keys through relationship field and model --- pgsync/querybuilder.py | 47 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/pgsync/querybuilder.py b/pgsync/querybuilder.py index 07d920d3..e02fed2a 100644 --- a/pgsync/querybuilder.py +++ b/pgsync/querybuilder.py @@ -108,11 +108,22 @@ def get_foreign_keys(self, node_a: Node, node_b: Node) -> dict: node_b.relationship.foreign_key.child ) - else: - fkeys: dict = defaultdict(list) - if node_a.model.foreign_keys: - for key in node_a.model.original.foreign_keys: - if key._table_key() == str(node_b.model.original): + fkeys: dict = defaultdict(list) + if node_a.model.foreign_keys: + for key in node_a.model.original.foreign_keys: + if key._table_key() == str(node_b.model.original): + fkeys[ + f"{key.parent.table.schema}." + f"{key.parent.table.name}" + ].append(str(key.parent.name)) + fkeys[ + f"{key.column.table.schema}." + f"{key.column.table.name}" + ].append(str(key.column.name)) + if not fkeys: + if node_b.model.original.foreign_keys: + for key in node_b.model.original.foreign_keys: + if key._table_key() == str(node_a.model.original): fkeys[ f"{key.parent.table.schema}." f"{key.parent.table.name}" @@ -121,26 +132,14 @@ def get_foreign_keys(self, node_a: Node, node_b: Node) -> dict: f"{key.column.table.schema}." f"{key.column.table.name}" ].append(str(key.column.name)) - if not fkeys: - if node_b.model.original.foreign_keys: - for key in node_b.model.original.foreign_keys: - if key._table_key() == str(node_a.model.original): - fkeys[ - f"{key.parent.table.schema}." - f"{key.parent.table.name}" - ].append(str(key.parent.name)) - fkeys[ - f"{key.column.table.schema}." - f"{key.column.table.name}" - ].append(str(key.column.name)) - if not fkeys: - raise ForeignKeyError( - f"No foreign key relationship between " - f"{node_a.model.original} and {node_b.model.original}" - ) + if not fkeys and not foreign_keys: + raise ForeignKeyError( + f"No foreign key relationship between " + f"{node_a.model.original} and {node_b.model.original}" + ) - for table, columns in fkeys.items(): - foreign_keys[table] = columns + for table, columns in fkeys.items(): + foreign_keys[table] = columns self._cache[(node_a, node_b)] = foreign_keys From 083af2a8cb43d0516d70ca7790fb055e855f1620 Mon Sep 17 00:00:00 2001 From: James Kent Date: Fri, 14 Jul 2023 15:52:05 -0400 Subject: [PATCH 2/2] Create tests for through_tables for grandchildren --- pgsync/querybuilder.py | 62 ++++++++------- tests/test_unique_behaviour.py | 135 ++++++++++++++++++++++++++++----- 2 files changed, 149 insertions(+), 48 deletions(-) diff --git a/pgsync/querybuilder.py b/pgsync/querybuilder.py index e02fed2a..8cfbfd8d 100644 --- a/pgsync/querybuilder.py +++ b/pgsync/querybuilder.py @@ -84,30 +84,8 @@ def _json_build_object( return expression - # this is for handling non-through tables - def get_foreign_keys(self, node_a: Node, node_b: Node) -> dict: + def get_foreign_keys_through_model(self, node_a: Node, node_b: Node) -> dict: if (node_a, node_b) not in self._cache: - foreign_keys: dict = {} - # if either offers a foreign_key via relationship, use it! - if ( - node_a.relationship.foreign_key.parent - or node_b.relationship.foreign_key.parent - ): - if node_a.relationship.foreign_key.parent: - foreign_keys[node_a.parent.name] = sorted( - node_a.relationship.foreign_key.parent - ) - foreign_keys[node_a.name] = sorted( - node_a.relationship.foreign_key.child - ) - if node_b.relationship.foreign_key.parent: - foreign_keys[node_b.parent.name] = sorted( - node_b.relationship.foreign_key.parent - ) - foreign_keys[node_b.name] = sorted( - node_b.relationship.foreign_key.child - ) - fkeys: dict = defaultdict(list) if node_a.model.foreign_keys: for key in node_a.model.original.foreign_keys: @@ -132,12 +110,13 @@ def get_foreign_keys(self, node_a: Node, node_b: Node) -> dict: f"{key.column.table.schema}." f"{key.column.table.name}" ].append(str(key.column.name)) - if not fkeys and not foreign_keys: + if not fkeys: raise ForeignKeyError( f"No foreign key relationship between " f"{node_a.model.original} and {node_b.model.original}" ) + foreign_keys: dict = {} for table, columns in fkeys.items(): foreign_keys[table] = columns @@ -145,6 +124,35 @@ def get_foreign_keys(self, node_a: Node, node_b: Node) -> dict: return self._cache[(node_a, node_b)] + # this is for handling non-through tables + def get_foreign_keys(self, node_a: Node, node_b: Node) -> dict: + if (node_a, node_b) not in self._cache: + foreign_keys: dict = {} + # if either offers a foreign_key via relationship, use it! + if ( + node_a.relationship.foreign_key.parent + or node_b.relationship.foreign_key.parent + ): + if node_a.relationship.foreign_key.parent: + foreign_keys[node_a.parent.name] = sorted( + node_a.relationship.foreign_key.parent + ) + foreign_keys[node_a.name] = sorted( + node_a.relationship.foreign_key.child + ) + if node_b.relationship.foreign_key.parent: + foreign_keys[node_b.parent.name] = sorted( + node_b.relationship.foreign_key.parent + ) + foreign_keys[node_b.name] = sorted( + node_b.relationship.foreign_key.child + ) + self._cache[(node_a, node_b)] = foreign_keys + else: + self.get_foreign_keys_through_model(node_a, node_b) + + return self._cache[(node_a, node_b)] + def _get_foreign_keys(self, node_a: Node, node_b: Node) -> dict: """This is for handling through nodes.""" if (node_a, node_b) not in self._cache: @@ -437,9 +445,9 @@ def _children(self, node: Node) -> None: def _through(self, node: Node) -> None: # noqa: C901 through: Node = node.relationship.throughs[0] - foreign_keys: dict = self.get_foreign_keys(node, through) + foreign_keys: dict = self.get_foreign_keys_through_model(node, through) - for key, values in self.get_foreign_keys(through, node.parent).items(): + for key, values in self.get_foreign_keys_through_model(through, node.parent).items(): if key in foreign_keys: for value in values: if value not in foreign_keys[key]: @@ -648,7 +656,7 @@ def _through(self, node: Node) -> None: # noqa: C901 sa.func.JSON_AGG(outer_subquery.c.anon).label(node.label), ] - foreign_keys: dict = self.get_foreign_keys(node.parent, through) + foreign_keys: dict = self.get_foreign_keys_through_model(node.parent, through) for column in foreign_keys[through.name]: columns.append(through.model.c[str(column)]) diff --git a/tests/test_unique_behaviour.py b/tests/test_unique_behaviour.py index 84bb6c65..28cdcf18 100644 --- a/tests/test_unique_behaviour.py +++ b/tests/test_unique_behaviour.py @@ -20,6 +20,8 @@ def data( user_cls, contact_cls, contact_item_cls, + author_cls, + book_author_cls, ): session = sync.session contacts = [ @@ -31,8 +33,8 @@ def data( # contact_item_cls(name="Contact Item 2", contact=contacts[1]), ] users = [ - user_cls(name="Fonzy Bear", contact=contacts[0]), - user_cls(name="Jack Jones", contact=contacts[1]), + user_cls(id=1, name="Fonzy Bear", contact=contacts[0]), + user_cls(id=2, name="Jack Jones", contact=contacts[1]), ] books = [ book_cls( @@ -43,6 +45,22 @@ def data( seller=users[1], ), ] + authors = [ + author_cls( + id=1, + name="Roald Dahl", + birth_year=1916, + ), + author_cls( + id=2, + name="Haruki Murakami", + birth_year=1949, + ), + ] + book_authors = [ + book_author_cls(id=1, book=books[0], author=authors[0]), + book_author_cls(id=2, book=books[0], author=authors[1]), + ] with subtransactions(session): conn = session.connection().engine.connect().connection conn.set_isolation_level( @@ -57,6 +75,8 @@ def data( session.add_all(contact_items) session.add_all(users) session.add_all(books) + session.add_all(authors) + session.add_all(book_authors) sync.logical_slot_get_changes( f"{sync.database}_testdb", @@ -68,6 +88,8 @@ def data( contacts, contact_items, users, + authors, + book_authors, ) with subtransactions(session): @@ -86,6 +108,8 @@ def data( contact_item_cls.__table__.name, contact_cls.__table__.name, user_cls.__table__.name, + author_cls.__table__.name, + book_author_cls.__table__.name, ] ) @@ -105,9 +129,18 @@ def data( session.connection().engine.dispose() sync.search_client.close() - @pytest.fixture(scope="function") - def nodes(self): - return { + def test_sync_multiple_children_empty_leaf( + self, + sync, + data, + ): + """ + ----> User(buyer) ----> Contact ----> ContactItem + Book ----| + ----> User(seller) ----> Contact ----> ContactItem + Test regular sync produces the correct result + """ + nodes = { "table": "book", "columns": ["isbn", "title", "description"], "children": [ @@ -200,28 +233,13 @@ def nodes(self): ], } - def test_sync_multiple_children_empty_leaf( - self, - sync, - data, - nodes, - book_cls, - user_cls, - contact_cls, - contact_item_cls, - ): - """ - ----> User(buyer) ----> Contact ----> ContactItem - Book ----| - ----> User(seller) ----> Contact ----> ContactItem - Test regular sync produces the correct result - """ sync.tree.__nodes = {} sync.tree.__post_init__() sync.nodes = nodes sync.root = sync.tree.build(nodes) docs = [sort_list(doc) for doc in sync.sync()] docs = sorted(docs, key=lambda k: k["_id"]) + assert docs == [ { "_id": "abc", @@ -261,3 +279,78 @@ def test_sync_multiple_children_empty_leaf( ] assert_resync_empty(sync, nodes) + + def test_though_table_for_grandchildren( + self, + sync, + data, + ): + nodes = { + "table": "user", + "columns": ["id", "name"], + "children": [ + { + "table": "book", + "label": "books", + "columns": ["isbn", "title", "description"], + "relationship": { + "variant": "object", + "type": "one_to_many", + "foreign_key": { + "parent": ["id"], + "child": ["buyer_id"], + }, + }, + "children": [ + { + "table": "author", + "label": "authors", + "columns": ["id", "name"], + "relationship": { + "type": "one_to_many", + "variant": "object", + "through_tables": ["book_author"], + }, + } + ] + } + ] + } + + sync.tree.__nodes = {} + sync.tree.__post_init__() + sync.nodes = nodes + sync.root = sync.tree.build(nodes) + docs = [sort_list(doc) for doc in sync.sync()] + docs = sorted(docs, key=lambda k: k["_id"]) + + assert docs == [ + {'_id': '1', + '_index': 'testdb', + '_source': { + 'id': 1, + 'name': 'Fonzy Bear', + 'books': [{ + 'isbn': 'abc', + 'title': 'The Tiger Club', + 'authors': [ + {'id': 1, 'name': 'Roald Dahl'}, + {'id': 2, 'name': 'Haruki Murakami'} + ], + 'description': 'Tigers are fierce creatures' + }], + '_meta': { + 'book': {'isbn': ['abc']}, + 'author': {'id': [1, 2]}, + 'book_author': {'id': [1, 2]} + } + }}, + {'_id': '2', + '_index': 'testdb', + '_source': { + 'id': 2, + 'name': 'Jack Jones', + 'books': None, + '_meta': {} + }} + ]