diff --git a/.travis.yml b/.travis.yml index 20b7afb..dfb9888 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,11 @@ python: - 3.6 - 3.7 - 3.8 + - 3.9 env: matrix: - - "NAUTOBOT_VER=1.0.0" + - "NAUTOBOT_VER=1.0.2" services: - "docker" @@ -37,7 +38,7 @@ jobs: script: - "invoke black" - "invoke bandit" - - "invoke pydocstyle" + # - "invoke pydocstyle" - "invoke flake8" - "invoke yamllint" # - "invoke pylint" diff --git a/development/Dockerfile b/development/Dockerfile index ba1f142..41c90f0 100644 --- a/development/Dockerfile +++ b/development/Dockerfile @@ -1,32 +1,18 @@ -ARG python_ver=3.7 -FROM python:${python_ver} +ARG PYTHON_VER +ARG NAUTOBOT_VER +FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER} -ARG nautobot_ver=v1.0.0b1 -ENV PYTHONUNBUFFERED 1 - -RUN mkdir /prom_cache -ENV prometheus_multiproc_dir /prom_cache - -RUN mkdir -p /opt/nautobot - -RUN pip install --upgrade pip\ - && pip install poetry +WORKDIR /source -# ------------------------------------------------------------------------------------- -# Install Nautobot -# ------------------------------------------------------------------------------------- -COPY packages/nautobot-1.0.0b1-py3-none-any.whl /tmp -RUN pip install /tmp/nautobot-1.0.0b1-py3-none-any.whl +# Copy in only pyproject.toml/poetry.lock to help with caching this layer if no updates to dependencies +COPY poetry.lock pyproject.toml /source/ +# --no-root declares not to install the project package since we're wanting to take advantage of caching dependency installation +# and the project is copied in and installed after this step +RUN poetry install --no-interaction --no-ansi --no-root -# ------------------------------------------------------------------------------------- -# Install Nautobot Plugin -# ------------------------------------------------------------------------------------- -RUN mkdir -p /source -WORKDIR /source +# Copy in the rest of the source code and install local Nautobot plugin COPY . /source -RUN poetry config virtualenvs.create false \ - && poetry install --no-interaction --no-ansi +RUN poetry install --no-interaction --no-ansi -ENV NAUTOBOT_CONFIG /opt/nautobot/configuration.py -WORKDIR /opt/nautobot/ +COPY development/nautobot_config.py /opt/nautobot/nautobot_config.py diff --git a/development/dev.env b/development/dev.env index 0a66644..01e20a8 100644 --- a/development/dev.env +++ b/development/dev.env @@ -1,20 +1,24 @@ -ALLOWED_HOSTS=* -CHANGELOG_RETENTION=0 -DB_HOST=postgres -DB_NAME=nautobot -DB_PASSWORD=decinablesprewad -DB_USER=nautobot -HIDE_RESTRICTED_UI=True -MAX_PAGE_SIZE=0 -NAPALM_TIMEOUT=5 -NAUTOBOT_CONFIG=/etc/nautobot/nautobot_config.py +NAUTOBOT_ALLOWED_HOSTS=* +NAUTOBOT_CHANGELOG_RETENTION=0 +NAUTOBOT_CONFIG=/opt/nautobot/nautobot_config.py +NAUTOBOT_DB_HOST=postgres +NAUTOBOT_DB_NAME=nautobot +NAUTOBOT_DB_PASSWORD=decinablesprewad +NAUTOBOT_DB_USER=nautobot +NAUTOBOT_MAX_PAGE_SIZE=0 +NAUTOBOT_NAPALM_TIMEOUT=5 +NAUTOBOT_REDIS_HOST=redis +NAUTOBOT_REDIS_PASSWORD=decinablesprewad +NAUTOBOT_REDIS_PORT=6379 +# Uncomment REDIS_SSL if using SSL +# NAUTOBOT_REDIS_SSL=True +NAUTOBOT_SECRET_KEY=012345678901234567890123456789012345678901234567890123456789 + +# Needed for Postgres should match the values for Nautobot above PGPASSWORD=decinablesprewad POSTGRES_DB=nautobot POSTGRES_PASSWORD=decinablesprewad POSTGRES_USER=nautobot -REDIS_HOST=redis + +# Needed for Redis should match the values for Nautobot above REDIS_PASSWORD=decinablesprewad -REDIS_PORT=6379 -# Uncomment REDIS_SSL if using SSL -# REDIS_SSL=True -SECRET_KEY=012345678901234567890123456789012345678901234567890123456789 diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 31f9b5d..b8606ef 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -1,53 +1,59 @@ + --- -version: "3" -services: - nautobot: + x-nautobot-build: &nautobot-build build: + args: + NAUTOBOT_VER: "${NAUTOBOT_VER}" + PYTHON_VER: "${PYTHON_VER}" context: "../" dockerfile: "development/Dockerfile" - image: "nautobot-data-validation-engine/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" - command: > - sh -c "nautobot-server migrate && - nautobot-server runserver 0.0.0.0:8000" - ports: - - "8000:8000" - depends_on: - - "postgres" - - "redis" + x-nautobot-base: &nautobot-base + image: "nautobot_data_validation_engine/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" env_file: - - "./dev.env" - volumes: - - "./nautobot_config.py:/etc/nautobot/nautobot_config.py" - - "../:/source" + - "dev.env" tty: true - worker: - build: - context: "../" - dockerfile: "development/Dockerfile" - image: "nautobot-data-validation-engine/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" - command: > - sh -c "nautobot-server rqworker" - depends_on: - - "nautobot" - env_file: - - "./dev.env" - volumes: - - "./nautobot_config.py:/etc/nautobot/nautobot_config.py" - - "../nautobot_data_validation_engine:/source/nautobot_data_validation_engine" - tty: true - postgres: - image: "postgres:12" - env_file: - - "./dev.env" - volumes: - - "pgdata_nautobot_data_validation_engine:/var/lib/postgresql/data" - redis: - image: "redis:5-alpine" - command: - - "sh" - - "-c" # this is to evaluate the $REDIS_PASSWORD from the env - - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD" ## $$ because of docker-compose - env_file: - - "./dev.env" -volumes: - pgdata_nautobot_data_validation_engine: {} + + version: "3.4" + services: + nautobot: + command: "nautobot-server runserver 0.0.0.0:8080 --insecure" + volumes: + - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" + - "../:/source" + ports: + - "8080:8080" + depends_on: + - "postgres" + - "redis" + <<: *nautobot-build + <<: *nautobot-base + worker: + entrypoint: "nautobot-server rqworker" + volumes: + - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" + - "../:/source" + depends_on: + - "nautobot" + healthcheck: + disable: true + <<: *nautobot-base + postgres: + image: "postgres:13-alpine" + env_file: + - "dev.env" + volumes: + - "pgdata_nautobot_data_validation_engine:/var/lib/postgresql/data" + ports: + - "5432:5432" + redis: + image: "redis:6-alpine" + command: + - "sh" + - "-c" # this is to evaluate the $REDIS_PASSWORD from the env + - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD" + env_file: + - "dev.env" + ports: + - "6379:6379" + volumes: + pgdata_nautobot_data_validation_engine: # yamllint disable-line rule:empty-values diff --git a/development/nautobot_config.py b/development/nautobot_config.py index c6413a5..6e21909 100644 --- a/development/nautobot_config.py +++ b/development/nautobot_config.py @@ -1,16 +1,19 @@ -"""Nautobot configuration file.""" +"""Nautobot development configuration file.""" import os import sys -ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "").split(" ") +from nautobot.core.settings import * +from nautobot.core.settings_funcs import is_truthy + +ALLOWED_HOSTS = os.environ.get("NAUTOBOT_ALLOWED_HOSTS", "").split(" ") DATABASES = { "default": { - "NAME": os.environ.get("DB_NAME", "nautobot"), - "USER": os.environ.get("DB_USER", ""), - "PASSWORD": os.environ.get("DB_PASSWORD", ""), - "HOST": os.environ.get("DB_HOST", "localhost"), - "PORT": os.environ.get("DB_PORT", ""), + "NAME": os.environ.get("NAUTOBOT_DB_NAME", "nautobot"), + "USER": os.environ.get("NAUTOBOT_DB_USER", ""), + "PASSWORD": os.environ.get("NAUTOBOT_DB_PASSWORD", ""), + "HOST": os.environ.get("NAUTOBOT_DB_HOST", "localhost"), + "PORT": os.environ.get("NAUTOBOT_DB_PORT", ""), "CONN_MAX_AGE": 300, "ENGINE": "django.db.backends.postgresql", } @@ -20,54 +23,94 @@ LOG_LEVEL = "DEBUG" if DEBUG else "INFO" -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "normal": { - "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s :\n %(message)s", - "datefmt": "%H:%M:%S", +TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" + +# Verbose logging during normal development operation, but quiet logging during unit test execution +if not TESTING: + LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "normal": { + "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s :\n %(message)s", + "datefmt": "%H:%M:%S", + }, + "verbose": { + "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() :\n %(message)s", + "datefmt": "%H:%M:%S", + }, }, - "verbose": { - "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() :\n %(message)s", - "datefmt": "%H:%M:%S", + "handlers": { + "normal_console": { + "level": "INFO", + "class": "rq.utils.ColorizingStreamHandler", + "formatter": "normal", + }, + "verbose_console": { + "level": "DEBUG", + "class": "rq.utils.ColorizingStreamHandler", + "formatter": "verbose", + }, }, - }, - "handlers": { - "normal_console": {"level": "INFO", "class": "rq.utils.ColorizingStreamHandler", "formatter": "normal"}, - "verbose_console": {"level": "DEBUG", "class": "rq.utils.ColorizingStreamHandler", "formatter": "verbose"}, - }, - "loggers": { - "django": {"handlers": ["normal_console"], "level": "INFO"}, - "nautobot": {"handlers": ["verbose_console" if DEBUG else "normal_console"], "level": LOG_LEVEL}, - "rq.worker": {"handlers": ["verbose_console" if DEBUG else "normal_console"], "level": LOG_LEVEL}, - }, -} + "loggers": { + "django": {"handlers": ["normal_console"], "level": "INFO"}, + "nautobot": { + "handlers": ["verbose_console" if DEBUG else "normal_console"], + "level": LOG_LEVEL, + }, + "rq.worker": { + "handlers": ["verbose_console" if DEBUG else "normal_console"], + "level": LOG_LEVEL, + }, + }, + } + + +# Redis variables +REDIS_HOST = os.getenv("NAUTOBOT_REDIS_HOST", "localhost") +REDIS_PORT = os.getenv("NAUTOBOT_REDIS_PORT", 6379) +REDIS_PASSWORD = os.getenv("NAUTOBOT_REDIS_PASSWORD", "") -REDIS = { - "caching": { - "HOST": os.environ.get("REDIS_HOST", "redis"), - "PORT": int(os.environ.get("REDIS_PORT", 6379)), - "PASSWORD": os.environ.get("REDIS_PASSWORD", ""), - "DATABASE": 1, - "SSL": bool(os.environ.get("REDIS_SSL", False)), - }, - "tasks": { - "HOST": os.environ.get("REDIS_HOST", "redis"), - "PORT": int(os.environ.get("REDIS_PORT", 6379)), - "PASSWORD": os.environ.get("REDIS_PASSWORD", ""), - "DATABASE": 0, - "SSL": bool(os.environ.get("REDIS_SSL", False)), - }, +# Check for Redis SSL +REDIS_SCHEME = "redis" +REDIS_SSL = is_truthy(os.environ.get("NAUTOBOT_REDIS_SSL", False)) +if REDIS_SSL: + REDIS_SCHEME = "rediss" + +# The django-redis cache is used to establish concurrent locks using Redis. The +# django-rq settings will use the same instance/database by default. +# +# This "default" server is now used by RQ_QUEUES. +# >> See: nautobot.core.settings.RQ_QUEUES +CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": f"{REDIS_SCHEME}://{REDIS_HOST}:{REDIS_PORT}/0", + "TIMEOUT": 300, + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "PASSWORD": REDIS_PASSWORD, + }, + } } -SECRET_KEY = os.environ.get("SECRET_KEY", "") +# RQ_QUEUES is not set here because it just uses the default that gets imported +# up top via `from nautobot.core.settings import *`. + +# REDIS CACHEOPS +CACHEOPS_REDIS = f"{REDIS_SCHEME}://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/1" + +HIDE_RESTRICTED_UI = os.environ.get("HIDE_RESTRICTED_UI", False) + +SECRET_KEY = os.environ.get("NAUTOBOT_SECRET_KEY", "") # Django Debug Toolbar -TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: DEBUG and not TESTING} -HIDE_RESTRICTED_UI = os.environ.get("HIDE_RESTRICTED_UI", False) -PLUGINS = [ - "nautobot_data_validation_engine", -] +if "debug_toolbar" not in INSTALLED_APPS: + INSTALLED_APPS.append("debug_toolbar") +if "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE: + MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware") + +# Enable installed plugins. Add the name of each plugin to the list. +PLUGINS = ["nautobot_data_validation_engine"] diff --git a/nautobot_data_validation_engine/api/__init__.py b/nautobot_data_validation_engine/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nautobot_data_validation_engine/api/serializers.py b/nautobot_data_validation_engine/api/serializers.py new file mode 100644 index 0000000..53d7aa9 --- /dev/null +++ b/nautobot_data_validation_engine/api/serializers.py @@ -0,0 +1,68 @@ +""" +API serializers +""" +from django.contrib.contenttypes.models import ContentType +from rest_framework import serializers + +from nautobot.core.api import ( + ContentTypeField, + ValidatedModelSerializer, +) +from nautobot.extras.utils import FeatureQuery + +from nautobot_data_validation_engine.models import MinMaxValidationRule, RegularExpressionValidationRule + + +class RegularExpressionValidationRuleSerializer(ValidatedModelSerializer): + """Serializer for `RegularExpressionValidationRule` objects.""" + + url = serializers.HyperlinkedIdentityField( + view_name="plugins-api:nautobot_data_validation_engine-api:regularexpressionvalidationrule-detail" + ) + content_type = ContentTypeField( + queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()), + ) + + class Meta: + model = RegularExpressionValidationRule + fields = [ + "id", + "url", + "name", + "slug", + "content_type", + "field", + "regular_expression", + "enabled", + "error_message", + "created", + "last_updated", + ] + + +class MinMaxValidationRuleSerializer(ValidatedModelSerializer): + """Serializer for `MinMaxValidationRule` objects.""" + + url = serializers.HyperlinkedIdentityField( + view_name="plugins-api:nautobot_data_validation_engine-api:minmaxvalidationrule-detail" + ) + content_type = ContentTypeField( + queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()), + ) + + class Meta: + model = MinMaxValidationRule + fields = [ + "id", + "url", + "name", + "slug", + "content_type", + "field", + "min", + "max", + "enabled", + "error_message", + "created", + "last_updated", + ] diff --git a/nautobot_data_validation_engine/api/urls.py b/nautobot_data_validation_engine/api/urls.py new file mode 100644 index 0000000..efa09cf --- /dev/null +++ b/nautobot_data_validation_engine/api/urls.py @@ -0,0 +1,19 @@ +""" +API routes +""" +from nautobot.core.api import OrderedDefaultRouter + +from nautobot_data_validation_engine.api import views + + +router = OrderedDefaultRouter() +router.APIRootView = views.DataValidationEngineRootView + +# Regular expression rules +router.register("rules/regex", views.RegularExpressionValidationRuleViewSet) + +# Min/max rules +router.register("rules/min-max", views.MinMaxValidationRuleViewSet) + + +urlpatterns = router.urls diff --git a/nautobot_data_validation_engine/api/views.py b/nautobot_data_validation_engine/api/views.py new file mode 100644 index 0000000..e54f435 --- /dev/null +++ b/nautobot_data_validation_engine/api/views.py @@ -0,0 +1,38 @@ +""" +API views +""" +from rest_framework.routers import APIRootView + +from nautobot.core.api.views import ModelViewSet + +from nautobot_data_validation_engine.api import serializers +from nautobot_data_validation_engine import models, filters + + +class DataValidationEngineRootView(APIRootView): + """ + Data Validation Engine API root view + """ + + def get_view_name(self): + return "Data Validation Engine" + + +class RegularExpressionValidationRuleViewSet(ModelViewSet): + """ + View to manage regular expression validation rules + """ + + queryset = models.RegularExpressionValidationRule.objects.all() + serializer_class = serializers.RegularExpressionValidationRuleSerializer + filterset_class = filters.RegularExpressionValidationRuleFilterSet + + +class MinMaxValidationRuleViewSet(ModelViewSet): + """ + View to manage min max expression validation rules + """ + + queryset = models.MinMaxValidationRule.objects.all() + serializer_class = serializers.MinMaxValidationRuleSerializer + filterset_class = filters.MinMaxValidationRuleFilterSet diff --git a/nautobot_data_validation_engine/custom_validators.py b/nautobot_data_validation_engine/custom_validators.py index f6860da..9a4ec7f 100644 --- a/nautobot_data_validation_engine/custom_validators.py +++ b/nautobot_data_validation_engine/custom_validators.py @@ -9,11 +9,12 @@ validation rules that have been defined for the given model. """ import re +from django.db.models.query_utils import Q from nautobot.extras.plugins import PluginCustomValidator from nautobot.extras.registry import registry -from nautobot_data_validation_engine.models import RegularExpressionValidationRule +from nautobot_data_validation_engine.models import MinMaxValidationRule, RegularExpressionValidationRule class BaseValidator(PluginCustomValidator): @@ -31,11 +32,44 @@ def clean(self): # Regex rules for rule in RegularExpressionValidationRule.objects.get_for_model(self.model): - if not re.match(rule.regular_expression, getattr(obj, rule.field)): + field_value = getattr(obj, rule.field) + if field_value is None: + # Coerce to a string for regex validation + field_value = "" + if not re.match(rule.regular_expression, field_value): self.validation_error( {rule.field: rule.error_message or f"Value does not conform to regex: {rule.regular_expression}"} ) + # Min/Max rules + for rule in MinMaxValidationRule.objects.get_for_model(self.model): + field_value = getattr(obj, rule.field) + + if field_value is None: + self.validation_error( + { + rule.field: rule.error_message + or f"Value does not conform to mix/max validation: min {rule.min}, max {rule.max}" + } + ) + + elif not isinstance(field_value, (int, float)): + self.validation_error( + { + rule.field: f"Unable to validate against min/max rule {rule} because the field value is not numeric." + } + ) + + elif rule.min is not None and field_value is not None and field_value < rule.min: + self.validation_error( + {rule.field: rule.error_message or f"Value is less than minimum value: {rule.min}"} + ) + + elif rule.max is not None and field_value is not None and field_value > rule.max: + self.validation_error( + {rule.field: rule.error_message or f"Value is more than maximum value: {rule.max}"} + ) + class CustomValidatorIterator: """ diff --git a/nautobot_data_validation_engine/filters.py b/nautobot_data_validation_engine/filters.py index d0f540e..b6e4625 100644 --- a/nautobot_data_validation_engine/filters.py +++ b/nautobot_data_validation_engine/filters.py @@ -5,9 +5,10 @@ from django.db.models import Q from nautobot.extras.filters import CreatedUpdatedFilterSet -from nautobot.utilities.filters import BaseFilterSet, ContentTypeFilter +from nautobot.extras.utils import FeatureQuery +from nautobot.utilities.filters import BaseFilterSet, ContentTypeMultipleChoiceFilter -from nautobot_data_validation_engine.models import RegularExpressionValidationRule +from nautobot_data_validation_engine.models import MinMaxValidationRule, RegularExpressionValidationRule class RegularExpressionValidationRuleFilterSet(BaseFilterSet, CreatedUpdatedFilterSet): @@ -19,11 +20,13 @@ class RegularExpressionValidationRuleFilterSet(BaseFilterSet, CreatedUpdatedFilt method="search", label="Search", ) - content_type = ContentTypeFilter() + content_type = ContentTypeMultipleChoiceFilter( + choices=FeatureQuery("custom_validators").get_choices, conjoined=False # Make this an OR with multi-values + ) class Meta: model = RegularExpressionValidationRule - fields = ["id", "name", "regular_expression", "enabled", "content_type", "field", "error_message"] + fields = ["id", "name", "slug", "regular_expression", "enabled", "content_type", "field", "error_message"] def search(self, queryset, name, value): """ @@ -33,6 +36,7 @@ def search(self, queryset, name, value): return queryset qs_filter = ( Q(name__icontains=value) + | Q(slug__icontains=value) | Q(regular_expression__icontains=value) | Q(error_message__icontains=value) | Q(content_type__app_label=value) @@ -40,3 +44,37 @@ def search(self, queryset, name, value): | Q(field=value) ) return queryset.filter(qs_filter) + + +class MinMaxValidationRuleFilterSet(BaseFilterSet, CreatedUpdatedFilterSet): + """ + Base filterset for the MinMaxValidationRule model. + """ + + q = django_filters.CharFilter( + method="search", + label="Search", + ) + content_type = ContentTypeMultipleChoiceFilter( + choices=FeatureQuery("custom_validators").get_choices, conjoined=False # Make this an OR with multi-values + ) + + class Meta: + model = MinMaxValidationRule + fields = ["id", "name", "slug", "min", "max", "enabled", "content_type", "field", "error_message"] + + def search(self, queryset, name, value): + """ + Custom filter method which searches a string value across several fields attached to the `q` filter field. + """ + if not value.strip(): + return queryset + qs_filter = ( + Q(name__icontains=value) + | Q(slug__icontains=value) + | Q(error_message__icontains=value) + | Q(content_type__app_label=value) + | Q(content_type__model=value) + | Q(field=value) + ) + return queryset.filter(qs_filter) diff --git a/nautobot_data_validation_engine/forms.py b/nautobot_data_validation_engine/forms.py index 56ffbbe..0be056c 100644 --- a/nautobot_data_validation_engine/forms.py +++ b/nautobot_data_validation_engine/forms.py @@ -4,6 +4,8 @@ from django import forms from django.contrib.contenttypes.models import ContentType +from nautobot.extras.forms import AddRemoveTagsForm +from nautobot.extras.models.tags import Tag from nautobot.extras.utils import FeatureQuery from nautobot.utilities.forms import ( BootstrapMixin, @@ -12,9 +14,11 @@ CSVContentTypeField, CSVMultipleContentTypeField, CSVModelForm, + DynamicModelMultipleChoiceField, + SlugField, ) -from nautobot_data_validation_engine.models import RegularExpressionValidationRule +from nautobot_data_validation_engine.models import MinMaxValidationRule, RegularExpressionValidationRule # @@ -27,6 +31,7 @@ class RegularExpressionValidationRuleForm(BootstrapMixin, forms.ModelForm): Base model form for the RegularExpressionValidationRule model. """ + slug = SlugField() content_type = forms.ModelChoiceField( queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( "app_label", "model" @@ -35,7 +40,7 @@ class RegularExpressionValidationRuleForm(BootstrapMixin, forms.ModelForm): class Meta: model = RegularExpressionValidationRule - fields = ["name", "enabled", "content_type", "field", "regular_expression", "error_message"] + fields = ["name", "slug", "enabled", "content_type", "field", "regular_expression", "error_message"] class RegularExpressionValidationRuleCSVForm(CSVModelForm): @@ -43,6 +48,7 @@ class RegularExpressionValidationRuleCSVForm(CSVModelForm): Base csv form for the RegularExpressionValidationRule model. """ + slug = SlugField() content_type = CSVContentTypeField( queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()), help_text="The object type to which this regular expression rule applies.", @@ -88,3 +94,79 @@ class RegularExpressionValidationRuleFilterForm(BootstrapMixin, forms.Form): ), required=False, ) + + +# +# MinMaxValidationRules +# + + +class MinMaxValidationRuleForm(BootstrapMixin, forms.ModelForm): + """ + Base model form for the MinMaxValidationRule model. + """ + + slug = SlugField() + content_type = forms.ModelChoiceField( + queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( + "app_label", "model" + ), + ) + + class Meta: + model = MinMaxValidationRule + fields = ["name", "slug", "enabled", "content_type", "field", "min", "max", "error_message"] + + +class MinMaxValidationRuleCSVForm(CSVModelForm): + """ + Base csv form for the MinMaxValidationRule model. + """ + + slug = SlugField() + content_type = CSVContentTypeField( + queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()), + help_text="The object type to which this regular expression rule applies.", + ) + + class Meta: + model = MinMaxValidationRule + fields = MinMaxValidationRule.csv_headers + + +class MinMaxValidationRuleBulkEditForm(BootstrapMixin, BulkEditForm): + """ + Base bulk edit form for the MinMaxValidationRule model. + """ + + pk = forms.ModelMultipleChoiceField(queryset=MinMaxValidationRule.objects.all(), widget=forms.MultipleHiddenInput) + enabled = forms.NullBooleanField( + required=False, + widget=BulkEditNullBooleanSelect(), + ) + min = forms.IntegerField(required=False) + max = forms.IntegerField(required=False) + error_message = forms.CharField(required=False) + + class Meta: + nullable_fields = ["error_message"] + + +class MinMaxValidationRuleFilterForm(BootstrapMixin, forms.Form): + """ + Base filter form for the MinMaxValidationRule model. + """ + + model = MinMaxValidationRule + field_order = ["q", "name", "enabled", "content_type", "field", "min", "max", "error_message"] + q = forms.CharField(required=False, label="Search") + # "CSV" field is being used here because it is using the slug-form input for + # content-types, which improves UX. + content_type = CSVMultipleContentTypeField( + queryset=ContentType.objects.filter(FeatureQuery("custom_validators").get_query()).order_by( + "app_label", "model" + ), + required=False, + ) + min = forms.IntegerField(required=False) + max = forms.IntegerField(required=False) diff --git a/nautobot_data_validation_engine/migrations/0001_initial.py b/nautobot_data_validation_engine/migrations/0001_initial.py index c80ed78..3517089 100644 --- a/nautobot_data_validation_engine/migrations/0001_initial.py +++ b/nautobot_data_validation_engine/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1.3 on 2021-02-22 23:21 +# Generated by Django 3.1.11 on 2021-05-26 06:04 from django.db import migrations, models import django.db.models.deletion @@ -28,13 +28,46 @@ class Migration(migrations.Migration): ("created", models.DateField(auto_now_add=True, null=True)), ("last_updated", models.DateTimeField(auto_now=True, null=True)), ("name", models.CharField(max_length=100, unique=True)), + ("slug", models.SlugField(max_length=100, unique=True)), + ("enabled", models.BooleanField(default=True)), + ("error_message", models.CharField(blank=True, max_length=255, null=True)), ("field", models.CharField(max_length=50)), ( "regular_expression", models.TextField(validators=[nautobot_data_validation_engine.models.validate_regex]), ), + ( + "content_type", + models.ForeignKey( + limit_choices_to=nautobot.extras.utils.FeatureQuery("custom_validators"), + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.contenttype", + ), + ), + ], + options={ + "ordering": ("name",), + "unique_together": {("content_type", "field")}, + }, + ), + migrations.CreateModel( + name="MinMaxValidationRule", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True + ), + ), + ("created", models.DateField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ("name", models.CharField(max_length=100, unique=True)), + ("slug", models.SlugField(max_length=100, unique=True)), ("enabled", models.BooleanField(default=True)), ("error_message", models.CharField(blank=True, max_length=255, null=True)), + ("field", models.CharField(max_length=50)), + ("min", models.FloatField(blank=True, null=True)), + ("max", models.FloatField(blank=True, null=True)), ( "content_type", models.ForeignKey( diff --git a/nautobot_data_validation_engine/models.py b/nautobot_data_validation_engine/models.py index 3f8a6ae..d546a74 100644 --- a/nautobot_data_validation_engine/models.py +++ b/nautobot_data_validation_engine/models.py @@ -6,6 +6,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.validators import ValidationError from django.db import models +from django.db.models.query_utils import Q from django.shortcuts import reverse from nautobot.extras.models import ChangeLoggedModel @@ -41,19 +42,16 @@ def get_for_model(self, content_type): return self.filter(enabled=True, content_type__app_label=app_label, content_type__model=model) -class RegularExpressionValidationRule(BaseModel, ChangeLoggedModel): +class ValidationRule(BaseModel, ChangeLoggedModel): """ - A type of validation rule that applies a regular expression to a given model field. + Base model for all validation engine rule models """ name = models.CharField(max_length=100, unique=True) + slug = models.SlugField(max_length=100, unique=True) content_type = models.ForeignKey( to=ContentType, on_delete=models.CASCADE, limit_choices_to=FeatureQuery("custom_validators") ) - field = models.CharField( - max_length=50, - ) - regular_expression = models.TextField(validators=[validate_regex]) enabled = models.BooleanField(default=True) error_message = models.CharField( max_length=255, null=True, blank=True, help_text="Optional error message to display when validation fails." @@ -61,12 +59,8 @@ class RegularExpressionValidationRule(BaseModel, ChangeLoggedModel): objects = ValidationRuleManager.as_manager() - csv_headers = ["name", "enabled", "content_type", "field", "regular_expression", "error_message"] - clone_fields = ["enabled", "content_type", "regular_expression", "error_message"] - class Meta: - ordering = ("name",) - unique_together = [["content_type", "field"]] + abstract = True def __str__(self): """ @@ -74,11 +68,29 @@ def __str__(self): """ return self.name + +class RegularExpressionValidationRule(ValidationRule): + """ + A type of validation rule that applies a regular expression to a given model field. + """ + + field = models.CharField( + max_length=50, + ) + regular_expression = models.TextField(validators=[validate_regex]) + + csv_headers = ["name", "slug", "enabled", "content_type", "field", "regular_expression", "error_message"] + clone_fields = ["enabled", "content_type", "regular_expression", "error_message"] + + class Meta: + ordering = ("name",) + unique_together = [["content_type", "field"]] + def get_absolute_url(self): """ Absolute url for the instance. """ - return reverse("plugins:nautobot_data_validation_engine:regularexpressionvalidationrule", args=[self.pk]) + return reverse("plugins:nautobot_data_validation_engine:regularexpressionvalidationrule", args=[self.slug]) def to_csv(self): """ @@ -86,6 +98,7 @@ def to_csv(self): """ return ( self.name, + self.slug, self.enabled, f"{self.content_type.app_label}.{self.content_type.model}", self.field, @@ -113,9 +126,11 @@ def clean(self): models.ForeignKey, models.ImageField, models.JSONField, + models.Manager, models.ManyToManyField, models.NullBooleanField, models.OneToOneField, + models.fields.related.RelatedField, models.SmallAutoField, models.UUIDField, ) @@ -123,4 +138,88 @@ def clean(self): model_field = self.content_type.model_class()._meta.get_field(self.field) if self.field.startswith("_") or not model_field.editable or isinstance(model_field, blacklisted_field_types): - raise ValidationError({"field": "This field does not support regular expression validation."}) + raise ValidationError({"field": "This field's type does not support regular expression validation."}) + + +class MinMaxValidationRule(ValidationRule): + """ + A type of validation rule that applies min/max constraints to a given numeric model field. + """ + + field = models.CharField( + max_length=50, + ) + min = models.FloatField( + null=True, blank=True, help_text="When set, apply a minimum value contraint to the value of the model field." + ) + max = models.FloatField( + null=True, blank=True, help_text="When set, apply a maximum value contraint to the value of the model field." + ) + + csv_headers = ["name", "slug", "enabled", "content_type", "field", "min", "max", "error_message"] + clone_fields = ["enabled", "content_type", "min", "max", "error_message"] + + class Meta: + ordering = ("name",) + unique_together = [["content_type", "field"]] + + def get_absolute_url(self): + """ + Absolute url for the instance. + """ + return reverse("plugins:nautobot_data_validation_engine:minmaxvalidationrule", args=[self.slug]) + + def to_csv(self): + """ + Return tuple representing the instance, which this used for CSV export. + """ + return ( + self.name, + self.slug, + self.enabled, + f"{self.content_type.app_label}.{self.content_type.model}", + self.field, + self.min, + self.max, + self.error_message, + ) + + def clean(self): + """ + Ensure field is valid for the model and has not been blacklisted. + """ + if self.field not in [f.name for f in self.content_type.model_class()._meta.get_fields()]: + raise ValidationError( + { + "field": f"Not a valid field for content type {self.content_type.app_label}.{self.content_type.model}." + } + ) + + whitelisted_field_types = ( + models.DecimalField, + models.FloatField, + models.IntegerField, + ) + + blacklisted_field_types = ( + models.AutoField, + models.BigAutoField, + ) + + model_field = self.content_type.model_class()._meta.get_field(self.field) + + if not isinstance(model_field, whitelisted_field_types) or ( + self.field.startswith("_") or not model_field.editable or isinstance(model_field, blacklisted_field_types) + ): + raise ValidationError({"field": "This field's type does not support min/max validation."}) + + if self.min is None and self.max is None: + raise ValidationError("At least a minimum or maximum value must be specified.") + + if self.min is not None and self.max is not None and self.min > self.max: + raise ValidationError( + { + "min": "Minimum value cannot be more than the maximum value.", + "max": "Maximum value cannot be less than the minimum value.", + } + ) diff --git a/nautobot_data_validation_engine/navigation.py b/nautobot_data_validation_engine/navigation.py index e4a71fe..116facd 100644 --- a/nautobot_data_validation_engine/navigation.py +++ b/nautobot_data_validation_engine/navigation.py @@ -6,6 +6,27 @@ menu_items = ( + PluginMenuItem( + link="plugins:nautobot_data_validation_engine:minmaxvalidationrule_list", + link_text="Min/Max Rules", + permissions=["nautobot_data_validation_engine.view_minmaxvalidationrule"], + buttons=( + PluginMenuButton( + link="plugins:nautobot_data_validation_engine:minmaxvalidationrule_add", + title="Add", + icon_class="mdi mdi-plus-thick", + color=ButtonColorChoices.GREEN, + permissions=["nautobot_data_validation_engine.add_minmaxvalidationrule"], + ), + PluginMenuButton( + link="plugins:nautobot_data_validation_engine:minmaxvalidationrule_import", + title="Import", + icon_class="mdi mdi-database-import-outline", + color=ButtonColorChoices.BLUE, + permissions=["nautobot_data_validation_engine.add_minmaxvalidationrule"], + ), + ), + ), PluginMenuItem( link="plugins:nautobot_data_validation_engine:regularexpressionvalidationrule_list", link_text="Regex Rules", diff --git a/nautobot_data_validation_engine/tables.py b/nautobot_data_validation_engine/tables.py index 76b22b8..455a970 100644 --- a/nautobot_data_validation_engine/tables.py +++ b/nautobot_data_validation_engine/tables.py @@ -5,7 +5,7 @@ from nautobot.utilities.tables import BaseTable, ToggleColumn -from nautobot_data_validation_engine.models import RegularExpressionValidationRule +from nautobot_data_validation_engine.models import MinMaxValidationRule, RegularExpressionValidationRule # @@ -25,3 +25,22 @@ class Meta(BaseTable.Meta): model = RegularExpressionValidationRule fields = ("pk", "name", "enabled", "content_type", "field", "regular_expression", "error_message") default_columns = ("pk", "name", "enabled", "content_type", "field", "regular_expression", "error_message") + + +# +# MinMaxValidationRules +# + + +class MinMaxValidationRuleTable(BaseTable): + """ + Base table for the MinMaxValidationRule model. + """ + + pk = ToggleColumn() + name = tables.LinkColumn(order_by=("name",)) + + class Meta(BaseTable.Meta): + model = MinMaxValidationRule + fields = ("pk", "name", "enabled", "content_type", "field", "min", "max", "error_message") + default_columns = ("pk", "name", "enabled", "content_type", "field", "min", "max", "error_message") diff --git a/nautobot_data_validation_engine/templates/nautobot_data_validation_engine/minmaxvalidationrule.html b/nautobot_data_validation_engine/templates/nautobot_data_validation_engine/minmaxvalidationrule.html new file mode 100644 index 0000000..efce5a8 --- /dev/null +++ b/nautobot_data_validation_engine/templates/nautobot_data_validation_engine/minmaxvalidationrule.html @@ -0,0 +1,100 @@ +{% extends 'base.html' %} +{% load buttons %} +{% load static %} +{% load helpers %} + +{% block title %}{{ object }}{% endblock %} + +{% block header %} +
+ +Name | +{{ object.name }} | +
Enabled | ++ {% if object.enabled %} + + {% else %} + + {% endif %} + | +
Content type | +{{ object.content_type.app_label }}.{{ object.content_type.model }} | +
Field | +{{ object.field }} | +
Min | +{{ object.min|placeholder }} | +
Max | +{{ object.max|placeholder }} | +
Error message | +{{ object.error_message|placeholder }} | +