Skip to content

Commit

Permalink
Fix the logical slot suffix regex
Browse files Browse the repository at this point in the history
  • Loading branch information
@toluaina
toluaina committed Aug 3, 2024
1 parent 8000739 commit 90052ee
Show file tree
Hide file tree
Showing 6 changed files with 32 additions and 27 deletions.
2 changes: 2 additions & 0 deletions pgsync/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -778,6 +778,8 @@ def parse_value(self, type_: str, value: str) -> t.Optional[str]:
NB: All integers are long in python3 and call to convert is just int
"""
if self.verbose:
logger.debug(f"type: {type_} value: {value}")
if value.lower() == "null":
return None
if type_.lower() in self.INT_TYPES:
Expand Down
2 changes: 1 addition & 1 deletion pgsync/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,5 +208,5 @@
r"table\s\"?(?P<schema>[\w-]+)\"?.\"?(?P<table>[\w-]+)\"?:\s(?P<tg_op>[A-Z]+):" # noqa E501
)
LOGICAL_SLOT_SUFFIX = re.compile(
r'\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>[\w\'"\-\\.\\+ ]+)' # noqa E501
r'\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>(?:"[^"]*"|\'[^\']*\'|null|\d+e[+-]?\d+|\w+))'
)
11 changes: 5 additions & 6 deletions requirements/base.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
#
async-timeout==4.0.3
# via redis
boto3==1.34.142
boto3==1.34.153
# via -r requirements/base.in
botocore==1.34.142
botocore==1.34.153
# via
# boto3
# s3transfer
Expand Down Expand Up @@ -61,13 +61,13 @@ python-dotenv==1.0.1
# via
# -r requirements/base.in
# environs
redis==5.0.7
redis==5.0.8
# via -r requirements/base.in
requests==2.32.3
# via
# opensearch-py
# requests-aws4auth
requests-aws4auth==1.2.3
requests-aws4auth==1.3.1
# via -r requirements/base.in
s3transfer==0.10.2
# via boto3
Expand All @@ -76,10 +76,9 @@ six==1.16.0
# opensearch-dsl
# opensearch-py
# python-dateutil
# requests-aws4auth
sqlalchemy==2.0.31
# via -r requirements/base.in
sqlparse==0.5.0
sqlparse==0.5.1
# via -r requirements/base.in
typing-extensions==4.12.2
# via
Expand Down
21 changes: 10 additions & 11 deletions requirements/dev.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
#
async-timeout==4.0.3
# via redis
black==24.4.2
black==24.8.0
# via -r requirements/dev.in
boto3==1.34.142
boto3==1.34.153
# via -r requirements/base.in
botocore==1.34.142
botocore==1.34.153
# via
# boto3
# s3transfer
Expand All @@ -27,7 +27,7 @@ click==8.1.7
# via
# -r requirements/base.in
# black
coverage[toml]==7.5.4
coverage[toml]==7.6.0
# via
# -r requirements/dev.in
# pytest-cov
Expand All @@ -45,9 +45,9 @@ environs==11.0.0
# via -r requirements/base.in
events==0.5
# via opensearch-py
exceptiongroup==1.2.1
exceptiongroup==1.2.2
# via pytest
faker==26.0.0
faker==26.1.0
# via -r requirements/dev.in
filelock==3.15.4
# via virtualenv
Expand Down Expand Up @@ -104,7 +104,7 @@ pycodestyle==2.12.0
# via flake8
pyflakes==3.2.0
# via flake8
pytest==8.2.2
pytest==8.3.2
# via
# -r requirements/dev.in
# pytest-cov
Expand All @@ -127,13 +127,13 @@ python-dotenv==1.0.1
# environs
pyyaml==6.0.1
# via pre-commit
redis==5.0.7
redis==5.0.8
# via -r requirements/base.in
requests==2.32.3
# via
# opensearch-py
# requests-aws4auth
requests-aws4auth==1.2.3
requests-aws4auth==1.3.1
# via -r requirements/base.in
s3transfer==0.10.2
# via boto3
Expand All @@ -142,10 +142,9 @@ six==1.16.0
# opensearch-dsl
# opensearch-py
# python-dateutil
# requests-aws4auth
sqlalchemy==2.0.31
# via -r requirements/base.in
sqlparse==0.5.0
sqlparse==0.5.1
# via -r requirements/base.in
tomli==2.0.1
# via
Expand Down
22 changes: 13 additions & 9 deletions tests/test_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,20 +433,20 @@ def test_parse_logical_slot(
assert "No match for row:" in str(excinfo.value)

row = """
table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'issue3' "CHANNEL_ID"[integer]:3 "CHANNEL_NAME"[character varying]:'channel3' "ITEM_ID"[integer]:3 "MESSAGE"[character varying]:'message3' "RETRY"[integer]:4 "STATUS"[character varying]:'status' "SUBJECT"[character varying]:'sub3' "TIMESTAMP"[bigint]:33
table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'issue to handle' "CHANNEL_ID"[integer]:3 "CHANNEL_NAME"[character varying]:'channel 45' "ITEM_ID"[integer]:3 "MESSAGE"[character varying]:'message3' "RETRY"[integer]:4 "STATUS"[character varying]:'status' "SUBJECT"[character varying]:'sub3' "TIMESTAMP"[bigint]:33
""" # noqa E501
payload = pg_base.parse_logical_slot(row)
assert payload.data == {
"CHANNEL_ID": 3,
"CHANNEL_NAME": "channel3",
"CHANNEL_NAME": "channel 45",
"CREATED_TIMESTAMP": 222,
"ADDRESS": "from3",
"ID": 5,
"ITEM_ID": 3,
"MESSAGE": "message3",
"RETRY": 4,
"SOME_FIELD_KEY": "key3",
"SOME_OTHER_FIELD_KEY": "issue3",
"SOME_OTHER_FIELD_KEY": "issue to handle",
"STATUS": "status",
"SUBJECT": "sub3",
"TIMESTAMP": 33,
Expand All @@ -472,16 +472,20 @@ def test_parse_logical_slot_with_double_precision(
""" # noqa E501
payload = pg_base.parse_logical_slot(row)
assert payload.data == {
"copyright": None,
"description": "Stephens Kings It",
"doc": '\'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": '
'5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", '
'"firstname": "Glenda", "generation": {"name": "X"}, "nick_names": '
'["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], '
'"coordinates": {"lat": 21.1, "lon": 32.9}}\'',
"id": 1,
"isbn": "001",
"title": "It",
"description": "Stephens",
"copyright": None,
"tags": "'",
"doc": "'",
"publish_date": "'1980-01-01 00:00:00'",
"publisher_id": 1,
"publish_date": "'1980-01-01",
"quad": 2e58,
"tags": '\'["a", "b", "c"]\'',
"title": "It",
}
assert payload.old == {}
assert payload.schema == "public"
Expand Down
1 change: 1 addition & 0 deletions tests/test_search_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,4 +109,5 @@ def test_get_search_client(self, mocker):
use_ssl=True,
verify_certs=True,
node_class=elastic_transport.RequestsHttpNode,
timeout=settings.ELASTICSEARCH_TIMEOUT,
)

0 comments on commit 90052ee

Please sign in to comment.