862 Avoid duplicate submissions to be saved

.github/workflows/pytest.yml

@@ -20,6 +20,7 @@ jobs:
       CACHE_URL: redis://localhost:6379/3
       ENKETO_REDIS_MAIN_URL: redis://localhost:6379/0
       KOBOCAT_MEDIA_ROOT: /tmp/test_media
+      SKIP_TESTS_WITH_CONCURRENCY: "True"
     strategy:
       matrix:
         python-version: ['3.8', '3.10']

kobo/apps/openrosa/apps/api/tests/fixtures/users.json

@@ -0,0 +1,18 @@
+[
+    {
+        "model": "auth.user",
+        "pk": 2,
+        "fields": {
+            "username": "bob",
+            "password": "pbkdf2_sha256$260000$T1eA0O4Ub6c6FAaCsb0fqU$6vX4qMw1VV9tMXFf1d9pL/5z5/2T1MQYYn7vB3p+I2Y=",
+            "email": "[email protected]",
+            "first_name": "bob",
+            "last_name": "bob",
+            "is_active": true,
+            "is_staff": false,
+            "is_superuser": false,
+            "last_login": null,
+            "date_joined": "2015-02-12T19:52:14.406Z"
+        }
+    }
+]

kobo/apps/openrosa/apps/api/tests/viewsets/test_xform_submission_api.py

@@ -1,12 +1,24 @@
 # coding: utf-8
+import multiprocessing
 import os
 import uuid
+from collections import defaultdict
+from functools import partial
 
+import pytest
+import requests
 import simplejson as json
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
 from django.core.files.uploadedfile import InMemoryUploadedFile
+from django.test.testcases import LiveServerTestCase
+from django.urls import reverse
 from django_digest.test import DigestAuth
+from rest_framework.authtoken.models import Token
+
+from kobo.apps.kobo_auth.shortcuts import User
+from kobo.apps.openrosa.apps.main.models import UserProfile
+from kobo.apps.openrosa.libs.tests.mixins.request_mixin import RequestMixin
 from kobo.apps.openrosa.libs.utils.guardian import assign_perm
 from kobo_service_account.utils import get_request_headers
 from rest_framework import status
@@ -15,6 +27,7 @@
     TestAbstractViewSet
 from kobo.apps.openrosa.apps.api.viewsets.xform_submission_api import XFormSubmissionApi
 from kobo.apps.openrosa.apps.logger.models import Attachment
+from kobo.apps.openrosa.apps.main import tests as main_tests
 from kobo.apps.openrosa.libs.constants import (
     CAN_ADD_SUBMISSIONS
 )
@@ -510,3 +523,113 @@ def test_submission_blocking_flag(self):
                 )
                 response = self.view(request, username=username)
                 self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+
+class ConcurrentSubmissionTestCase(RequestMixin, LiveServerTestCase):
+    """
+    Inherit from LiveServerTestCase to be able to test concurrent requests
+    to submission endpoint in different transactions (and different processes).
+    Otherwise, DB is populated only on the first request but still empty on
+    subsequent ones.
+    """
+    fixtures = ['kobo/apps/openrosa/apps/api/tests/fixtures/users']
+
+    def setUp(self):
+        self.user = User.objects.get(username='bob')
+        self.token, _ = Token.objects.get_or_create(user=self.user)
+        new_profile, created = UserProfile.objects.get_or_create(
+            user=self.user
+        )
+
+    def publish_xls_form(self):
+        path = os.path.join(
+            settings.OPENROSA_APP_DIR,
+            'apps',
+            'main',
+            'tests',
+            'fixtures',
+            'transportation',
+            'transportation.xls',
+        )
+
+        xform_list_url = reverse('xform-list')
+        service_account_meta = self.get_meta_from_headers(
+            get_request_headers(self.user.username)
+        )
+        service_account_meta['HTTP_HOST'] = settings.TEST_HTTP_HOST
+
+        with open(path, 'rb') as xls_file:
+            post_data = {'xls_file': xls_file}
+            response = self.client.post(xform_list_url, data=post_data, **service_account_meta)
+
+        assert response.status_code == status.HTTP_201_CREATED
+
+    @pytest.mark.skipif(
+        settings.SKIP_TESTS_WITH_CONCURRENCY, reason='GitLab does not seem to support multi-processes'
+    )
+    def test_post_concurrent_same_submissions(self):
+        DUPLICATE_SUBMISSIONS_COUNT = 2  # noqa
+
+        self.publish_xls_form()
+        username = 'bob'
+        survey = 'transport_2011-07-25_19-05-49'
+        results = defaultdict(int)
+
+        with multiprocessing.Pool() as pool:
+            for result in pool.map(
+                partial(
+                    submit_data,
+                    live_server_url=self.live_server_url,
+                    survey_=survey,
+                    username_=username,
+                    token_=self.token.key
+                ),
+                range(DUPLICATE_SUBMISSIONS_COUNT),
+            ):
+                results[result] += 1
+
+        assert results[status.HTTP_201_CREATED] == 1
+        assert results[status.HTTP_409_CONFLICT] == DUPLICATE_SUBMISSIONS_COUNT - 1
+
+
+def submit_data(identifier, survey_, username_, live_server_url, token_):
+    """
+    Submit data to live server.
+
+    It has to be outside `ConcurrentSubmissionTestCase` class to be pickled by
+    `multiprocessing.Pool().map()`.
+    """
+    media_file = '1335783522563.jpg'
+    main_directory = os.path.dirname(main_tests.__file__)
+    path = os.path.join(
+        main_directory,
+        'fixtures',
+        'transportation',
+        'instances',
+        survey_,
+        media_file,
+    )
+    with open(path, 'rb') as f:
+        f = InMemoryUploadedFile(
+            f,
+            'media_file',
+            media_file,
+            'image/jpg',
+            os.path.getsize(path),
+            None,
+        )
+        submission_path = os.path.join(
+            main_directory,
+            'fixtures',
+            'transportation',
+            'instances',
+            survey_,
+            f'{survey_}.xml',
+        )
+        with open(submission_path) as sf:
+            files = {'xml_submission_file': sf, 'media_file': f}
+            headers = {'Authorization': f'Token {token_}'}
+            response = requests.post(
+                f'{live_server_url}/{username_}/submission', files=files, headers=headers
+            )
+            return response.status_code

kobo/apps/openrosa/apps/logger/exceptions.py

@@ -1,5 +1,11 @@
 # coding: utf-8
-from django.utils.translation import gettext as t
+class ConflictingXMLHashInstanceError(Exception):
+    pass
+
+
+class DuplicateInstanceError(Exception):
+    def __init__(self, message='Duplicate Instance'):
+        super().__init__(message)
 
 
 class DuplicateUUIDError(Exception):
@@ -10,5 +16,28 @@ class FormInactiveError(Exception):
     pass
 
 
+class InstanceEmptyError(Exception):
+    def __init__(self, message='Empty instance'):
+        super().__init__(message)
+
+
+class InstanceInvalidUserError(Exception):
+    def __init__(self, message='Could not determine the user'):
+        super().__init__(message)
+
+
+class InstanceMultipleNodeError(Exception):
+    pass
+
+
+class InstanceParseError(Exception):
+    def __init__(self, message='The instance could not be parsed'):
+        super().__init__(message)
+
+
 class TemporarilyUnavailableError(Exception):
     pass
+
+
+class XLSFormError(Exception):
+    pass