From ad4d917c7c27993f5c7ce21017c36278d687f849 Mon Sep 17 00:00:00 2001 From: Nilesh Pant Date: Tue, 12 Dec 2023 23:17:10 +0530 Subject: [PATCH 1/4] add-support-for-django-q --- admin_settings/cache_router.py | 19 ++++++++++++++++ admin_settings/settings.py | 40 +++++++++++++++++++++++++++++++++- requirements.txt | 3 +++ start_qcluster.sh | 7 ++++++ 4 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 admin_settings/cache_router.py create mode 100644 start_qcluster.sh diff --git a/admin_settings/cache_router.py b/admin_settings/cache_router.py new file mode 100644 index 00000000..83d64fa3 --- /dev/null +++ b/admin_settings/cache_router.py @@ -0,0 +1,19 @@ +app_to_database = { + 'django_cache': 'cache_db', +} + +class CacheRouter: + + def db_for_read(self, model, **hints): + return app_to_database.get(model._meta.app_label, None) + + def db_for_write(self, model, **hints): + return app_to_database.get(model._meta.app_label, None) + + def allow_syncdb(self, db, model): + _db = app_to_database.get(model._meta.app_label, None) + return db == _db if _db else None + + def allow_migrate(self, db, app_label, model_name=None, **hints): + _db = app_to_database.get(app_label, None) + return db == _db if _db else None diff --git a/admin_settings/settings.py b/admin_settings/settings.py index 44114dbd..49e55feb 100644 --- a/admin_settings/settings.py +++ b/admin_settings/settings.py @@ -11,7 +11,6 @@ """ import os -from pathlib import Path import sys import dj_database_url @@ -44,6 +43,7 @@ 'rest_framework', 'fyle_rest_auth', 'django_filters', + 'django_q', # User Created Apps 'apps.users', @@ -102,6 +102,37 @@ 'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'], } +Q_CLUSTER = { + 'name': 'integrations_settings_api', + 'save_limit': 0, + 'retry': 14400, + 'timeout': 3600, + 'catch_up': False, + 'workers': 4, + # How many tasks are kept in memory by a single cluster. + # Helps balance the workload and the memory overhead of each individual cluster + 'queue_limit': 10, + 'cached': False, + 'orm': 'default', + 'ack_failures': True, + 'poll': 1, + 'max_attempts': 1, + 'attempt_count': 1, + # The number of tasks a worker will process before recycling. + # Useful to release memory resources on a regular basis. + 'recycle': 50, + # The maximum resident set size in kilobytes before a worker will recycle and release resources. + # Useful for limiting memory usage. + 'max_rss': 100000 # 100mb +} + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', + 'LOCATION': 'auth_cache', + } +} + SERVICE_NAME = os.environ.get('SERVICE_NAME') @@ -173,6 +204,13 @@ 'default': dj_database_url.config(engine='django_db_geventpool.backends.postgresql_psycopg2') } +DATABASES['cache_db'] = { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'cache.db' +} + +DATABASE_ROUTERS = ['admin_settings.cache_router.CacheRouter'] + # Password validation # https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators diff --git a/requirements.txt b/requirements.txt index b09ce140..b57e55b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,9 @@ dj-database-url==0.5.0 # Platform SDK fyle==v0.29.0 +# DjangoQ for running async tasks +django-q==1.3.4 + # Reusable Fyle Packages fyle-rest-auth==1.1.0 diff --git a/start_qcluster.sh b/start_qcluster.sh new file mode 100644 index 00000000..3f8bb672 --- /dev/null +++ b/start_qcluster.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Creating the cache table +python manage.py createcachetable --database cache_db + +# Running qcluster server +python manage.py qcluster \ No newline at end of file From 9db3dc1837e12146e091cacf252428385e362cbe Mon Sep 17 00:00:00 2001 From: Nilesh Pant Date: Tue, 12 Dec 2023 23:33:07 +0530 Subject: [PATCH 2/4] add minor test --- admin_settings/tests/settings.py | 273 +++++++++++++++++++++++++++++++ pytest.ini | 2 +- 2 files changed, 274 insertions(+), 1 deletion(-) create mode 100644 admin_settings/tests/settings.py diff --git a/admin_settings/tests/settings.py b/admin_settings/tests/settings.py new file mode 100644 index 00000000..c5fac176 --- /dev/null +++ b/admin_settings/tests/settings.py @@ -0,0 +1,273 @@ +""" +Django settings for admin_settings project. + +Generated by 'django-admin startproject' using Django 4.1.2. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.1/ref/settings/ +""" + +import os +import sys + +import dj_database_url + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get('SECRET_KEY') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True if os.environ.get('DEBUG') == 'True' else False + +ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS').split(',') + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'corsheaders', + + # Installed Apps + 'rest_framework', + 'fyle_rest_auth', + 'django_filters', + 'django_q', + + # User Created Apps + 'apps.users', + 'apps.bamboohr', + 'apps.orgs', + 'apps.travelperk', + 'apps.gusto', + 'apps.integrations' +] + +MIDDLEWARE = [ + 'request_logging.middleware.LoggingMiddleware', + 'corsheaders.middleware.CorsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'admin_settings.logging_middleware.ErrorHandlerMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'admin_settings.urls' +APPEND_SLASH = False + +AUTH_USER_MODEL = 'users.User' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +FYLE_REST_AUTH_SERIALIZERS = { + 'USER_DETAILS_SERIALIZER': 'apps.users.serializers.UserSerializer' +} + +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': ( + 'rest_framework.permissions.IsAuthenticated', + ), + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'fyle_rest_auth.authentication.FyleJWTAuthentication', + ), + 'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'], +} + +Q_CLUSTER = { + 'name': 'integrations_settings_api', + 'save_limit': 0, + 'retry': 14400, + 'timeout': 3600, + 'catch_up': False, + 'workers': 4, + # How many tasks are kept in memory by a single cluster. + # Helps balance the workload and the memory overhead of each individual cluster + 'queue_limit': 10, + 'cached': False, + 'orm': 'default', + 'ack_failures': True, + 'poll': 1, + 'max_attempts': 1, + 'attempt_count': 1, + # The number of tasks a worker will process before recycling. + # Useful to release memory resources on a regular basis. + 'recycle': 50, + # The maximum resident set size in kilobytes before a worker will recycle and release resources. + # Useful for limiting memory usage. + 'max_rss': 100000 # 100mb +} + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', + 'LOCATION': 'auth_cache', + } +} + + +SERVICE_NAME = os.environ.get('SERVICE_NAME') + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'verbose': { + 'format': '{levelname} %s {asctime} {module} {message} ' % SERVICE_NAME, + 'style': '{', + }, + 'requests': { + 'format': 'request {levelname} %s {asctime} {message}' % SERVICE_NAME, + 'style': '{' + } + }, + 'handlers': { + 'debug_logs': { + 'class': 'logging.StreamHandler', + 'stream': sys.stdout, + 'formatter': 'verbose' + }, + 'request_logs': { + 'class': 'logging.StreamHandler', + 'stream': sys.stdout, + 'formatter': 'requests' + }, + }, + 'loggers': { + 'django': { + 'handlers': ['request_logs'], + 'propagate': True, + }, + 'django.request': { + 'handlers': ['request_logs'], + 'propagate': False + }, + 'admin_settings': { + 'handlers': ['debug_logs'], + 'level': 'ERROR', + 'propagate': False + }, + 'apps': { + 'handlers': ['debug_logs'], + 'level': 'ERROR', + 'propagate': False + }, + 'gunicorn': { + 'handlers': ['request_logs'], + 'level': 'INFO', + 'propagate': False + } + } +} + + +WSGI_APPLICATION = 'admin_settings.wsgi.application' + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases +# Defaulting django engine for qcluster +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': os.environ.get('DB_NAME'), + 'USER': os.environ.get('DB_USER'), + 'PASSWORD': os.environ.get('DB_PASSWORD'), + 'HOST': os.environ.get('DB_HOST'), + 'PORT': os.environ.get('DB_PORT'), + } +} + +# Password validation +# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.1/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.1/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +API_URL = os.environ.get('API_URL') +FYLE_TOKEN_URI = os.environ.get('FYLE_TOKEN_URI') +FYLE_CLIENT_ID = os.environ.get('FYLE_CLIENT_ID') +FYLE_CLIENT_SECRET = os.environ.get('FYLE_CLIENT_SECRET') +FYLE_BASE_URL = os.environ.get('FYLE_BASE_URL') +FYLE_APP_URL = os.environ.get('APP_URL') +SENDGRID_API_KEY = os.environ.get('SENDGRID_API_KEY') +SENDGRID_EMAIL = os.environ.get('SENDGRID_EMAIL') +BASE_URI = os.environ.get('BASE_URI') +WK_JWT_PRIVATE_KEY = os.environ.get('WK_JWT_PRIVATE_KEY') +WK_API_KEY = os.environ.get('WK_API_KEY') + +GUSTO_CLIENT_ID = os.environ.get('GUSTO_CLIENT_ID') +GUSTO_CLIENT_SECRET = os.environ.get('GUSTO_CLIENT_SECRET') +GUSTO_ENVIRONMENT = os.environ.get('GUSTO_ENVIRONMENT') +TRAVELPERK_CLIENT_ID = os.environ.get('TRAVELPERK_CLIENT_ID') +TRAVELPERK_CLIENT_SECRET = os.environ.get('TRAVELPERK_CLIENT_SECRET') +TRAVELPERK_AUTH_URL = os.environ.get('TRAVELPERK_AUTH_URL') +TRAVELPERK_TOKEN_URL = os.environ.get('TRAVELPERK_TOKEN_URL') +TRAVELPERK_BASE_URL = os.environ.get('TRAVELPERK_BASE_URL') +TRAVELPERK_REDIRECT_URI = os.environ.get('TRAVELPERK_REDIRECT_URI') +WORKATO_ORIGIN_URL = os.environ.get('WORKATO_ORIGIN_URL') +WORKATO_FRAME_ANCESTORS_URL = os.environ.get('WORKATO_FRAME_ANCESTORS_URL') +FYLE_NOTIFICATIONS_EMAIL = os.environ.get('FYLE_NOTIFICATIONS_EMAIL') + +# Environ Variable Required for Tests +FYLE_REFRESH_TOKEN = os.environ.get('FYLE_REFRESH_TOKEN') diff --git a/pytest.ini b/pytest.ini index d6af3aef..f3beb587 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -DJANGO_SETTINGS_MODULE = admin_settings.settings +DJANGO_SETTINGS_MODULE = admin_settings.tests.settings python_files = tests.py test_*.py *_tests.py addopts = -p no:warnings --strict-markers --no-migrations --create-db log_cli = 1 From 472c243c6dc664066243dbc02c2e488714e03bcd Mon Sep 17 00:00:00 2001 From: Nilesh Pant Date: Tue, 12 Dec 2023 23:52:03 +0530 Subject: [PATCH 3/4] minor change --- admin_settings/settings.py | 5 - admin_settings/tests/settings.py | 273 ------------------------------- pytest.ini | 2 +- 3 files changed, 1 insertion(+), 279 deletions(-) delete mode 100644 admin_settings/tests/settings.py diff --git a/admin_settings/settings.py b/admin_settings/settings.py index 49e55feb..66020ef9 100644 --- a/admin_settings/settings.py +++ b/admin_settings/settings.py @@ -204,11 +204,6 @@ 'default': dj_database_url.config(engine='django_db_geventpool.backends.postgresql_psycopg2') } -DATABASES['cache_db'] = { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'cache.db' -} - DATABASE_ROUTERS = ['admin_settings.cache_router.CacheRouter'] # Password validation diff --git a/admin_settings/tests/settings.py b/admin_settings/tests/settings.py deleted file mode 100644 index c5fac176..00000000 --- a/admin_settings/tests/settings.py +++ /dev/null @@ -1,273 +0,0 @@ -""" -Django settings for admin_settings project. - -Generated by 'django-admin startproject' using Django 4.1.2. - -For more information on this file, see -https://docs.djangoproject.com/en/4.1/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/4.1/ref/settings/ -""" - -import os -import sys - -import dj_database_url - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.environ.get('SECRET_KEY') - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True if os.environ.get('DEBUG') == 'True' else False - -ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS').split(',') - -# Application definition - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'corsheaders', - - # Installed Apps - 'rest_framework', - 'fyle_rest_auth', - 'django_filters', - 'django_q', - - # User Created Apps - 'apps.users', - 'apps.bamboohr', - 'apps.orgs', - 'apps.travelperk', - 'apps.gusto', - 'apps.integrations' -] - -MIDDLEWARE = [ - 'request_logging.middleware.LoggingMiddleware', - 'corsheaders.middleware.CorsMiddleware', - 'django.middleware.security.SecurityMiddleware', - 'admin_settings.logging_middleware.ErrorHandlerMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -ROOT_URLCONF = 'admin_settings.urls' -APPEND_SLASH = False - -AUTH_USER_MODEL = 'users.User' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -FYLE_REST_AUTH_SERIALIZERS = { - 'USER_DETAILS_SERIALIZER': 'apps.users.serializers.UserSerializer' -} - -REST_FRAMEWORK = { - 'DEFAULT_PERMISSION_CLASSES': ( - 'rest_framework.permissions.IsAuthenticated', - ), - 'DEFAULT_AUTHENTICATION_CLASSES': ( - 'fyle_rest_auth.authentication.FyleJWTAuthentication', - ), - 'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'], -} - -Q_CLUSTER = { - 'name': 'integrations_settings_api', - 'save_limit': 0, - 'retry': 14400, - 'timeout': 3600, - 'catch_up': False, - 'workers': 4, - # How many tasks are kept in memory by a single cluster. - # Helps balance the workload and the memory overhead of each individual cluster - 'queue_limit': 10, - 'cached': False, - 'orm': 'default', - 'ack_failures': True, - 'poll': 1, - 'max_attempts': 1, - 'attempt_count': 1, - # The number of tasks a worker will process before recycling. - # Useful to release memory resources on a regular basis. - 'recycle': 50, - # The maximum resident set size in kilobytes before a worker will recycle and release resources. - # Useful for limiting memory usage. - 'max_rss': 100000 # 100mb -} - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', - 'LOCATION': 'auth_cache', - } -} - - -SERVICE_NAME = os.environ.get('SERVICE_NAME') - -LOGGING = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'verbose': { - 'format': '{levelname} %s {asctime} {module} {message} ' % SERVICE_NAME, - 'style': '{', - }, - 'requests': { - 'format': 'request {levelname} %s {asctime} {message}' % SERVICE_NAME, - 'style': '{' - } - }, - 'handlers': { - 'debug_logs': { - 'class': 'logging.StreamHandler', - 'stream': sys.stdout, - 'formatter': 'verbose' - }, - 'request_logs': { - 'class': 'logging.StreamHandler', - 'stream': sys.stdout, - 'formatter': 'requests' - }, - }, - 'loggers': { - 'django': { - 'handlers': ['request_logs'], - 'propagate': True, - }, - 'django.request': { - 'handlers': ['request_logs'], - 'propagate': False - }, - 'admin_settings': { - 'handlers': ['debug_logs'], - 'level': 'ERROR', - 'propagate': False - }, - 'apps': { - 'handlers': ['debug_logs'], - 'level': 'ERROR', - 'propagate': False - }, - 'gunicorn': { - 'handlers': ['request_logs'], - 'level': 'INFO', - 'propagate': False - } - } -} - - -WSGI_APPLICATION = 'admin_settings.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases -# Defaulting django engine for qcluster -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': os.environ.get('DB_NAME'), - 'USER': os.environ.get('DB_USER'), - 'PASSWORD': os.environ.get('DB_PASSWORD'), - 'HOST': os.environ.get('DB_HOST'), - 'PORT': os.environ.get('DB_PORT'), - } -} - -# Password validation -# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - -# Internationalization -# https://docs.djangoproject.com/en/4.1/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/4.1/howto/static-files/ - -STATIC_URL = 'static/' - -# Default primary key field type -# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field - -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' - -API_URL = os.environ.get('API_URL') -FYLE_TOKEN_URI = os.environ.get('FYLE_TOKEN_URI') -FYLE_CLIENT_ID = os.environ.get('FYLE_CLIENT_ID') -FYLE_CLIENT_SECRET = os.environ.get('FYLE_CLIENT_SECRET') -FYLE_BASE_URL = os.environ.get('FYLE_BASE_URL') -FYLE_APP_URL = os.environ.get('APP_URL') -SENDGRID_API_KEY = os.environ.get('SENDGRID_API_KEY') -SENDGRID_EMAIL = os.environ.get('SENDGRID_EMAIL') -BASE_URI = os.environ.get('BASE_URI') -WK_JWT_PRIVATE_KEY = os.environ.get('WK_JWT_PRIVATE_KEY') -WK_API_KEY = os.environ.get('WK_API_KEY') - -GUSTO_CLIENT_ID = os.environ.get('GUSTO_CLIENT_ID') -GUSTO_CLIENT_SECRET = os.environ.get('GUSTO_CLIENT_SECRET') -GUSTO_ENVIRONMENT = os.environ.get('GUSTO_ENVIRONMENT') -TRAVELPERK_CLIENT_ID = os.environ.get('TRAVELPERK_CLIENT_ID') -TRAVELPERK_CLIENT_SECRET = os.environ.get('TRAVELPERK_CLIENT_SECRET') -TRAVELPERK_AUTH_URL = os.environ.get('TRAVELPERK_AUTH_URL') -TRAVELPERK_TOKEN_URL = os.environ.get('TRAVELPERK_TOKEN_URL') -TRAVELPERK_BASE_URL = os.environ.get('TRAVELPERK_BASE_URL') -TRAVELPERK_REDIRECT_URI = os.environ.get('TRAVELPERK_REDIRECT_URI') -WORKATO_ORIGIN_URL = os.environ.get('WORKATO_ORIGIN_URL') -WORKATO_FRAME_ANCESTORS_URL = os.environ.get('WORKATO_FRAME_ANCESTORS_URL') -FYLE_NOTIFICATIONS_EMAIL = os.environ.get('FYLE_NOTIFICATIONS_EMAIL') - -# Environ Variable Required for Tests -FYLE_REFRESH_TOKEN = os.environ.get('FYLE_REFRESH_TOKEN') diff --git a/pytest.ini b/pytest.ini index f3beb587..d6af3aef 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -DJANGO_SETTINGS_MODULE = admin_settings.tests.settings +DJANGO_SETTINGS_MODULE = admin_settings.settings python_files = tests.py test_*.py *_tests.py addopts = -p no:warnings --strict-markers --no-migrations --create-db log_cli = 1 From 252739ee919bd99286d16517a6604d384d43de62 Mon Sep 17 00:00:00 2001 From: Nilesh Pant Date: Tue, 12 Dec 2023 23:59:39 +0530 Subject: [PATCH 4/4] fix tests --- admin_settings/settings.py | 5 + admin_settings/tests/settings.py | 241 +++++++++++++++++++++++++++++++ pytest.ini | 2 +- 3 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 admin_settings/tests/settings.py diff --git a/admin_settings/settings.py b/admin_settings/settings.py index 66020ef9..49e55feb 100644 --- a/admin_settings/settings.py +++ b/admin_settings/settings.py @@ -204,6 +204,11 @@ 'default': dj_database_url.config(engine='django_db_geventpool.backends.postgresql_psycopg2') } +DATABASES['cache_db'] = { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'cache.db' +} + DATABASE_ROUTERS = ['admin_settings.cache_router.CacheRouter'] # Password validation diff --git a/admin_settings/tests/settings.py b/admin_settings/tests/settings.py new file mode 100644 index 00000000..b5bfd944 --- /dev/null +++ b/admin_settings/tests/settings.py @@ -0,0 +1,241 @@ +""" +Django settings for admin_settings project. + +Generated by 'django-admin startproject' using Django 4.1.2. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.1/ref/settings/ +""" + +import os +from pathlib import Path +import sys + +import dj_database_url + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get('SECRET_KEY') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True if os.environ.get('DEBUG') == 'True' else False + +ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS').split(',') + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'corsheaders', + + # Installed Apps + 'rest_framework', + 'fyle_rest_auth', + 'django_filters', + + # User Created Apps + 'apps.users', + 'apps.bamboohr', + 'apps.orgs', + 'apps.travelperk', + 'apps.gusto', + 'apps.integrations' +] + +MIDDLEWARE = [ + 'request_logging.middleware.LoggingMiddleware', + 'corsheaders.middleware.CorsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'admin_settings.logging_middleware.ErrorHandlerMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'admin_settings.urls' +APPEND_SLASH = False + +AUTH_USER_MODEL = 'users.User' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +FYLE_REST_AUTH_SERIALIZERS = { + 'USER_DETAILS_SERIALIZER': 'apps.users.serializers.UserSerializer' +} + +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': ( + 'rest_framework.permissions.IsAuthenticated', + ), + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'fyle_rest_auth.authentication.FyleJWTAuthentication', + ), + 'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'], +} + + +SERVICE_NAME = os.environ.get('SERVICE_NAME') + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'verbose': { + 'format': '{levelname} %s {asctime} {module} {message} ' % SERVICE_NAME, + 'style': '{', + }, + 'requests': { + 'format': 'request {levelname} %s {asctime} {message}' % SERVICE_NAME, + 'style': '{' + } + }, + 'handlers': { + 'debug_logs': { + 'class': 'logging.StreamHandler', + 'stream': sys.stdout, + 'formatter': 'verbose' + }, + 'request_logs': { + 'class': 'logging.StreamHandler', + 'stream': sys.stdout, + 'formatter': 'requests' + }, + }, + 'loggers': { + 'django': { + 'handlers': ['request_logs'], + 'propagate': True, + }, + 'django.request': { + 'handlers': ['request_logs'], + 'propagate': False + }, + 'admin_settings': { + 'handlers': ['debug_logs'], + 'level': 'ERROR', + 'propagate': False + }, + 'apps': { + 'handlers': ['debug_logs'], + 'level': 'ERROR', + 'propagate': False + }, + 'gunicorn': { + 'handlers': ['request_logs'], + 'level': 'INFO', + 'propagate': False + } + } +} + + +WSGI_APPLICATION = 'admin_settings.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases +# Defaulting django engine for qcluster +if len(sys.argv) > 0 and sys.argv[1] == 'qcluster': + DATABASES = { + 'default': dj_database_url.config() + } +else: + DATABASES = { + 'default': dj_database_url.config(engine='django_db_geventpool.backends.postgresql_psycopg2') + } + +# Password validation +# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.1/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.1/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +API_URL = os.environ.get('API_URL') +FYLE_TOKEN_URI = os.environ.get('FYLE_TOKEN_URI') +FYLE_CLIENT_ID = os.environ.get('FYLE_CLIENT_ID') +FYLE_CLIENT_SECRET = os.environ.get('FYLE_CLIENT_SECRET') +FYLE_BASE_URL = os.environ.get('FYLE_BASE_URL') +FYLE_APP_URL = os.environ.get('APP_URL') +SENDGRID_API_KEY = os.environ.get('SENDGRID_API_KEY') +SENDGRID_EMAIL = os.environ.get('SENDGRID_EMAIL') +BASE_URI = os.environ.get('BASE_URI') +WK_JWT_PRIVATE_KEY = os.environ.get('WK_JWT_PRIVATE_KEY') +WK_API_KEY = os.environ.get('WK_API_KEY') + +GUSTO_CLIENT_ID = os.environ.get('GUSTO_CLIENT_ID') +GUSTO_CLIENT_SECRET = os.environ.get('GUSTO_CLIENT_SECRET') +GUSTO_ENVIRONMENT = os.environ.get('GUSTO_ENVIRONMENT') +TRAVELPERK_CLIENT_ID = os.environ.get('TRAVELPERK_CLIENT_ID') +TRAVELPERK_CLIENT_SECRET = os.environ.get('TRAVELPERK_CLIENT_SECRET') +TRAVELPERK_AUTH_URL = os.environ.get('TRAVELPERK_AUTH_URL') +TRAVELPERK_TOKEN_URL = os.environ.get('TRAVELPERK_TOKEN_URL') +TRAVELPERK_BASE_URL = os.environ.get('TRAVELPERK_BASE_URL') +TRAVELPERK_REDIRECT_URI = os.environ.get('TRAVELPERK_REDIRECT_URI') +WORKATO_ORIGIN_URL = os.environ.get('WORKATO_ORIGIN_URL') +WORKATO_FRAME_ANCESTORS_URL = os.environ.get('WORKATO_FRAME_ANCESTORS_URL') +FYLE_NOTIFICATIONS_EMAIL = os.environ.get('FYLE_NOTIFICATIONS_EMAIL') + +FYLE_REFRESH_TOKEN = os.environ.get('FYLE_REFRESH_TOKEN') diff --git a/pytest.ini b/pytest.ini index d6af3aef..f3beb587 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -DJANGO_SETTINGS_MODULE = admin_settings.settings +DJANGO_SETTINGS_MODULE = admin_settings.tests.settings python_files = tests.py test_*.py *_tests.py addopts = -p no:warnings --strict-markers --no-migrations --create-db log_cli = 1