Feat: Adding tests and code coverage (#14)

* feat: Adding tests and code coverage

* Adding some tests for producer

* Adding more producer tests and pytest action

.coveragerc

@@ -0,0 +1,10 @@
+# .coveragerc
+[run]
+omit =
+    tests/*
+    */__init__.py
+    setup.py
+    */main.py
+
+[report]
+show_missing=true

.github/workflows/pytest.yml

@@ -0,0 +1,47 @@
+name: Run Pytest and Coverage
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [assigned, opened, synchronize, reopened]
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checkout the code from the repository
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      # Setup Python environment
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      # Install Python dependencies
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      # Run Pytest with coverage
+      - name: Run Tests
+        run: |
+          python -m pytest tests/ --cov=. --junit-xml=test-reports/report.xml --cov-report=term-missing --cov-fail-under=70 | tee pytest-coverage.txt
+          echo "STATUS=$(grep 'Required test' pytest-coverage.txt | awk '{ print $1 }')" >> $GITHUB_ENV
+          echo "FAILED=$(awk -F'=' '/failures=/ {gsub(/"/, "", $2); print $2}' test-reports/report.xml)" >> $GITHUB_ENV
+
+      # Post Pytest coverage as a comment on PR
+      - name: Pytest coverage comment
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          create-new-comment: true
+          pytest-coverage-path: ./pytest-coverage.txt
+          junitxml-path: ./test-reports/report.xml
+
+      # Evaluate test and coverage results
+      - name: Evaluate Coverage
+        if: env.STATUS == 'FAIL' || env.FAILED > 0
+        run: exit 1

.gitignore

@@ -15,3 +15,5 @@ dev.sql
 build/
 dist/
 *.egg-info/
+
+.coverage

docker-compose.yaml

@@ -31,8 +31,8 @@ services:
       PGDATABASE: dummy
       PGUSER: postgres
       PGPASSWORD: postgres
-      PGREPLICATIONSLOT: pgevents_slot
-      PGOUTPUTPLUGIN: pgoutput
+      PGREPLICATIONSLOT: pgevents
+      PGOUTPUTPLUGIN: wal2json
       PGTABLES: public.users
       PGPUBLICATION: events
       RABBITMQ_URL: amqp://admin:password@rabbitmq:5672/?heartbeat=0
@@ -41,6 +41,7 @@ services:
     command: ["bash", "-c", "producer"]
     volumes:
       - ./producer:/pgevents/producer
+      - ./test:/pgevents/test
     depends_on:
       - database
       - rabbitmq

pytest.ini

@@ -0,0 +1,4 @@
+# pytest.ini
+[pytest]
+addopts = --cov=. --cov-config=.coveragerc
+testpaths = tests/

requirements.txt

@@ -14,6 +14,7 @@ py==1.11.0
 pylint==2.5.3
 pyparsing==3.0.7
 pytest==7.0.1
+pytest-cov==4.1.0
 toml==0.10.2
 tomli==2.0.1
 wrapt==1.12.1

tests/conftest.py

@@ -1,35 +1,192 @@
 import os
+from unittest import mock
 import psycopg2
 import pytest
+from common.event import base_event
 
 from common.qconnector import RabbitMQConnector
 from common import log
+from producer.event_producer import EventProducer
 
 logger = log.get_logger(__name__)
 
 
-@pytest.fixture(scope='function')
-def db_conn():
-    db_connection = psycopg2.connect(
-        user=os.environ['PGUSER'],
-        password=os.environ['PGPASSWORD'],
-        host=os.environ['PGHOST'],
-        port=os.environ['PGPORT'],
-        dbname=os.environ['PGDATABASE']
-    )
+@pytest.fixture
+def producer_init_params():
+    return {
+        'qconnector_cls': RabbitMQConnector,
+        'event_cls': base_event.BaseEvent,
+        'pg_host': 'localhost',
+        'pg_port': 5432,
+        'pg_database': 'test',
+        'pg_user': 'test',
+        'pg_password': 'test',
+        'pg_replication_slot': 'test',
+        'pg_output_plugin': 'pgoutput',
+        'pg_tables': 'public.users',
+        'pg_publication_name': 'test',
+        'rabbitmq_url': 'amqp://admin:password@rabbitmq:5672/?heartbeat=0',
+        'rabbitmq_exchange': 'test'
+    }
 
-    yield db_connection
-    db_connection.close()
 
+@pytest.fixture
+def mock_producer(producer_init_params):
+    return EventProducer(**producer_init_params)
 
-@pytest.fixture(scope='session')
-def rmq_conn():
-    rmq_connector = RabbitMQConnector(
-        rabbitmq_url=os.environ['RABBITMQ_URL'],
-        rabbitmq_exchange=os.environ['RABBITMQ_EXCHANGE'],
-        queue_name='PRODUCER_TEST_QUEUE',
-        binding_keys='#'
-    )
 
-    rmq_connector.connect()
-    return rmq_connector
+@pytest.fixture
+def mock_pika_connect():
+    with mock.patch('pika.BlockingConnection') as mock_pika:
+        yield mock_pika
+
+
+@pytest.fixture
+def mock_pg_conn():
+    with mock.patch('psycopg2.connect') as mock_pg:
+        yield mock_pg
+
+
+@pytest.fixture
+def mock_basic_publish():
+    with mock.patch('pika.adapters.blocking_connection.BlockingChannel.basic_publish') as mock_publish:
+        yield mock_publish
+
+
+@pytest.fixture
+def mock_schema():
+    return {
+        'relation_id': 16385,
+        'table_name': 'public.users',
+        'columns': [
+            {'name': 'id'},
+            {'name': 'full_name'},
+            {'name': 'company'},
+            {'name': 'created_at'},
+            {'name': 'updated_at'}
+        ]
+    }
+
+@pytest.fixture
+def wal2json_payload():
+    return {"action":"D","lsn":"0/1671850","schema":"public","table":"users","identity":[{"name":"id","type":"integer","value":2},{"name":"full_name","type":"text","value":"Geezer Butler"},{"name":"company","type":"jsonb","value":"{\"name\": \"Fyle\"}"},{"name":"created_at","type":"timestamp with time zone","value":"2023-11-17 13:35:09.471909+00"},{"name":"updated_at","type":"timestamp with time zone","value":"2023-11-17 13:35:09.471909+00"}]}
+
+# Class to hold payload data
+class OutputData:
+    payload = None
+    data_start = 124122
+
+
+# Fixture for relation payload
+@pytest.fixture
+def relation_payload():
+    data = OutputData()
+    data.payload = b'R\x00\x00@\x01public\x00users\x00f\x00\x05\x01id\x00\x00\x00\x00\x17\xff\xff\xff\xff\x01full_name\x00\x00\x00\x00\x19\xff\xff\xff\xff\x01company\x00\x00\x00\x0e\xda\xff\xff\xff\xff\x01created_at\x00\x00\x00\x04\xa0\xff\xff\xff\xff\x01updated_at\x00\x00\x00\x04\xa0\xff\xff\xff\xff'
+    return data
+
+
+# Fixture for expected relation response
+@pytest.fixture
+def relation_response():
+    return {
+        'relation_id': 16385,
+        'table_name': 'public.users',
+        'columns': [
+            {'name': 'id'},
+            {'name': 'full_name'},
+            {'name': 'company'},
+            {'name': 'created_at'},
+            {'name': 'updated_at'}
+        ]
+    }
+
+
+# Fixture for insert payload
+@pytest.fixture
+def insert_payload():
+    data = OutputData()
+    data.payload = b'I\x00\x00@\x01N\x00\x05t\x00\x00\x00\x011t\x00\x00\x00\x04Miket\x00\x00\x00\x10{"name": "Fyle"}t\x00\x00\x00\x1d2023-11-17 13:44:14.700844+00t\x00\x00\x00\x1d2023-11-17 13:44:14.700844+00'
+    return data
+
+
+# Fixture for expected insert response
+@pytest.fixture
+def insert_response():
+    return {
+        'table_name': 'public.users',
+        'new': {
+            'id': '1',
+            'full_name': 'Mike',
+            'company': '{"name": "Fyle"}',
+            'created_at': '2023-11-17 13:44:14.700844+00',
+            'updated_at': '2023-11-17 13:44:14.700844+00'
+        },
+        'id': '1',
+        'old': {},
+        'diff': {
+            'id': '1',
+            'full_name': 'Mike',
+            'company': '{"name": "Fyle"}',
+            'created_at': '2023-11-17 13:44:14.700844+00',
+            'updated_at': '2023-11-17 13:44:14.700844+00'
+        },
+        'action': 'I'
+    }
+
+# Fixture for update payload
+@pytest.fixture
+def update_payload():
+    data = OutputData()
+    data.payload = b'U\x00\x00@\x01O\x00\x05t\x00\x00\x00\x011t\x00\x00\x00\x04Miket\x00\x00\x00\x10{"name": "Fyle"}t\x00\x00\x00\x1d2023-11-17 13:44:14.700844+00t\x00\x00\x00\x1d2023-11-17 13:44:14.700844+00N\x00\x05t\x00\x00\x00\x011t\x00\x00\x00\x05Mylest\x00\x00\x00\x10{"name": "Fyle"}t\x00\x00\x00\x1d2023-11-17 13:44:14.700844+00t\x00\x00\x00\x1d2023-11-17 13:44:14.700844+00'
+    return data
+
+# Fixture for expected update response
+@pytest.fixture
+def update_response():
+    return  {
+        'table_name': 'public.users',
+        'id': '1',
+        'old': {
+            'id': '1',
+            'full_name': 'Mike',
+            'company': '{"name": "Fyle"}',
+            'created_at': '2023-11-17 13:44:14.700844+00',
+            'updated_at': '2023-11-17 13:44:14.700844+00'
+        },
+        'new': {
+            'id': '1',
+            'full_name': 'Myles',
+            'company': '{"name": "Fyle"}',
+            'created_at': '2023-11-17 13:44:14.700844+00',
+            'updated_at': '2023-11-17 13:44:14.700844+00'
+        },
+        'diff': {
+            'full_name': 'Myles'
+        },
+        'action': 'U'
+    }
+
+# Fixture for delete payload
+@pytest.fixture
+def delete_payload():
+    data = OutputData()
+    data.payload = b'D\x00\x00@\x01O\x00\x05t\x00\x00\x00\x012t\x00\x00\x00\rGeezer Butlert\x00\x00\x00\x10{"name": "Fyle"}t\x00\x00\x00\x1d2023-11-17 13:35:09.471909+00t\x00\x00\x00\x1d2023-11-17 13:35:09.471909+00'
+    return data
+
+# Fixture for expected delete response
+@pytest.fixture
+def delete_response():
+    return {
+        'table_name': 'public.users',
+        'action': 'D',
+        'old': {
+            'id': '2',
+            'full_name': 'Geezer Butler', 
+            'company': '{"name": "Fyle"}',
+            'created_at': '2023-11-17 13:35:09.471909+00',
+            'updated_at': '2023-11-17 13:35:09.471909+00'
+        },
+        'id': '2',
+        'new': {},
+        'diff': {}
+    }