Add unit test for concurrent duplicate data

.github/workflows/pytest.yml

@@ -52,7 +52,7 @@ jobs:
     - name: Install Python dependencies
       run: pip-sync dependencies/pip/dev_requirements.txt
     - name: Run pytest
-      run: pytest -vv -rf
+      run: pytest -vv -rf --disable-warnings
       env:
         DJANGO_SECRET_KEY: ${{ secrets.DJANGO_SECRET_KEY }}
         TEST_DATABASE_URL: postgis://kobo:kobo@localhost:5432/kobocat_test

.gitlab-ci.yml

@@ -32,6 +32,7 @@ test:
     POSTGRES_PASSWORD: kobo
     POSTGRES_DB: kobocat_test
     SERVICE_ACCOUNT_BACKEND_URL: redis://redis_cache:6379/4
+    GIT_LAB: "True"
   services:
     - name: postgis/postgis:14-3.2
       alias: postgres
@@ -40,7 +41,7 @@ test:
   script:
     - apt-get update && apt-get install -y ghostscript gdal-bin libproj-dev gettext openjdk-11-jre
     - pip install -r dependencies/pip/dev_requirements.txt
-    - pytest -vv -rf
+    - pytest -vv -rf --disable-warnings
 
 deploy-beta:
   stage: deploy

conftest.py

@@ -83,7 +83,7 @@ def setup(request):
 @pytest.fixture(scope='session', autouse=True)
 def default_session_fixture(request):
     """
-    Globally patch redis_client with fake redis
+    Globally patch redis_client with fakeredis
     """
     with patch(
         'kobo_service_account.models.ServiceAccountUser.redis_client',

onadata/apps/api/tests/fixtures/users.json

@@ -0,0 +1,19 @@
+[
+    {
+        "fields": {
+            "date_joined": "2015-02-12T19:52:14.406Z",
+            "email": "[email protected]",
+            "first_name": "bob",
+            "groups": [],
+            "is_active": true,
+            "is_staff": false,
+            "is_superuser": false,
+            "last_login": "2015-02-12T19:52:14.406Z",
+            "last_name": "bob",
+            "password": "pbkdf2_sha256$260000$jSfi1lb5FclOUV9ZodfCdP$Up19DmjLFtBh0VREyow/oduVkwEoqQftljfwq6b9vIo=",
+            "username": "bob"
+        },
+        "model": "auth.user",
+        "pk": 2
+    }
+]

onadata/apps/api/tests/viewsets/test_xform_submission_api.py

@@ -1,11 +1,18 @@
 # coding: utf-8
+import multiprocessing
 import os
 import uuid
+from collections import defaultdict
+from functools import partial
 
+import pytest
+import requests
 import simplejson as json
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
 from django.core.files.uploadedfile import InMemoryUploadedFile
+from django.test.testcases import LiveServerTestCase
+from django.urls import reverse
 from django_digest.test import DigestAuth
 from guardian.shortcuts import assign_perm
 from kobo_service_account.utils import get_request_headers
@@ -15,6 +22,7 @@
     TestAbstractViewSet
 from onadata.apps.api.viewsets.xform_submission_api import XFormSubmissionApi
 from onadata.apps.logger.models import Attachment
+from onadata.apps.main import tests as main_tests
 from onadata.libs.constants import (
     CAN_ADD_SUBMISSIONS
 )
@@ -441,6 +449,7 @@ def test_edit_submission_with_service_account(self):
             self.assertEqual(
                 response['Location'], 'http://testserver/submission'
             )
+
     def test_submission_blocking_flag(self):
         # Set 'submissions_suspended' True in the profile metadata to test if
         # submission do fail with the flag set
@@ -488,3 +497,103 @@ def test_submission_blocking_flag(self):
                 )
                 response = self.view(request, username=username)
                 self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+
+class ConcurrentSubmissionTestCase(LiveServerTestCase):
+    """
+    Inherit from LiveServerTestCase to be able to test concurrent requests
+    to submission endpoint in different transactions (and different processes).
+    Otherwise, DB is populated only on the first request but still empty on
+    subsequent ones.
+    """
+
+    fixtures = ['onadata/apps/api/tests/fixtures/users']
+
+    def publish_xls_form(self):
+
+        path = os.path.join(
+            settings.ONADATA_DIR,
+            'apps',
+            'main',
+            'tests',
+            'fixtures',
+            'transportation',
+            'transportation.xls',
+        )
+
+        xform_list_url = reverse('xform-list')
+        self.client.login(username='bob', password='bob')
+        with open(path, 'rb') as xls_file:
+            post_data = {'xls_file': xls_file}
+            response = self.client.post(xform_list_url, data=post_data)
+
+        assert response.status_code == status.HTTP_201_CREATED
+
+    @pytest.mark.skipif(
+        settings.GIT_LAB, reason='GitLab does not seem to support multi-processes'
+    )
+    def test_post_concurrent_same_submissions(self):
+
+        DUPLICATE_SUBMISSIONS_COUNT = 2  # noqa
+
+        self.publish_xls_form()
+        username = 'bob'
+        survey = 'transport_2011-07-25_19-05-49'
+        results = defaultdict(int)
+
+        with multiprocessing.Pool() as pool:
+            for result in pool.map(
+                partial(
+                    submit_data,
+                    live_server_url=self.live_server_url,
+                    survey_=survey,
+                    username_=username,
+                ),
+                range(DUPLICATE_SUBMISSIONS_COUNT),
+            ):
+                results[result] += 1
+
+        assert results[status.HTTP_201_CREATED] == 1
+        assert results[status.HTTP_409_CONFLICT] == DUPLICATE_SUBMISSIONS_COUNT - 1
+
+
+def submit_data(identifier, survey_, username_, live_server_url):
+    """
+    Submit data to live server.
+
+    It has to be outside `ConcurrentSubmissionTestCase` class to be pickled by
+    `multiprocessing.Pool().map()`.
+    """
+    media_file = '1335783522563.jpg'
+    main_directory = os.path.dirname(main_tests.__file__)
+    path = os.path.join(
+        main_directory,
+        'fixtures',
+        'transportation',
+        'instances',
+        survey_,
+        media_file,
+    )
+    with open(path, 'rb') as f:
+        f = InMemoryUploadedFile(
+            f,
+            'media_file',
+            media_file,
+            'image/jpg',
+            os.path.getsize(path),
+            None,
+        )
+        submission_path = os.path.join(
+            main_directory,
+            'fixtures',
+            'transportation',
+            'instances',
+            survey_,
+            f'{survey_}.xml',
+        )
+        with open(submission_path) as sf:
+            files = {'xml_submission_file': sf, 'media_file': f}
+            response = requests.post(
+               f'{live_server_url}/{username_}/submission', files=files
+            )
+            return response.status_code

onadata/settings/base.py

@@ -681,7 +681,10 @@ def skip_suspicious_operations(record):
 # NOTE: this should be set to False for major deployments. This can take a long time
 SKIP_HEAVY_MIGRATIONS = env.bool('SKIP_HEAVY_MIGRATIONS', False)
 
-redis_lock_url = env.cache_url('REDIS_LOCK_URL', default=redis_session_url)
+redis_lock_url = env.cache_url(
+    'REDIS_LOCK_URL',
+    default=os.getenv('REDIS_SESSION_URL', 'redis://redis_cache:6380/2'),
+)
 REDIS_LOCK_CLIENT = redis.Redis.from_url(redis_lock_url['LOCATION'])
 
 ################################

onadata/settings/testing.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 import os
 
-from fakeredis import FakeConnection, FakeStrictRedis
+from fakeredis import FakeConnection, FakeStrictRedis, FakeServer
 from mongomock import MongoClient as MockMongoClient
 
 from .base import *
@@ -57,7 +57,10 @@
 SERVICE_ACCOUNT['WHITELISTED_HOSTS'] = ['testserver']
 SERVICE_ACCOUNT['NAMESPACE'] = 'kobo-service-account-test'
 
-REDIS_LOCK_CLIENT = FakeStrictRedis()
+server = FakeServer()
+REDIS_LOCK_CLIENT = FakeStrictRedis(server=server)
+
+GIT_LAB = os.getenv('GIT_LAB', False)
 
 ################################
 # Celery settings              #