New entities

README.md

@@ -62,7 +62,9 @@ You must first have these set up and ready to go:
       - If you get an error on your console about signatures, fix minio as above.
       - If you get an error on your console about tipofday not found, add the static page as above.
 15. Develop DocumentCloud and its frontend!
-
+16. You can run the tests with `inv test`.
+- If you want to run a subset of the tests, you can specify the directory containing the test you want with the `path` switch like so: `inv test --path documentcloud/documents`.
+    - You can specify a single file in `--path` if you only want to run the tests in that file.
 
 
 [docker-install]: https://docs.docker.com/install/

config/settings/base.py

@@ -92,6 +92,7 @@
     "documentcloud.statistics.apps.StatisticsConfig",
     "documentcloud.sidekick.apps.SidekickConfig",
     "documentcloud.users.apps.UsersConfig",
+    "documentcloud.entities.apps.EntitiesConfig",
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS

config/urls.py

@@ -25,14 +25,15 @@
     DocumentErrorViewSet,
     DocumentViewSet,
     EntityDateViewSet,
-    EntityViewSet,
+    EntityViewSet as LegacyEntity2ViewSet,
     LegacyEntityViewSet,
     ModificationViewSet,
     NoteViewSet,
     RedactionViewSet,
     SectionViewSet,
 )
 from documentcloud.drf_bulk.routers import BulkDefaultRouter, BulkRouterMixin
+from documentcloud.entities.views import EntityOccurrenceViewSet, EntityViewSet
 from documentcloud.organizations.views import OrganizationViewSet
 from documentcloud.projects.views import (
     CollaborationViewSet,
@@ -56,11 +57,14 @@ class BulkNestedDefaultRouter(BulkRouterMixin, NestedDefaultRouter):
 router.register("addons", AddOnViewSet)
 router.register("addon_runs", AddOnRunViewSet)
 router.register("addon_events", AddOnEventViewSet)
+router.register("entities", EntityViewSet, basename="entities")
+
 
 documents_router = BulkNestedDefaultRouter(router, "documents", lookup="document")
 documents_router.register("notes", NoteViewSet)
 documents_router.register("sections", SectionViewSet)
-documents_router.register("entities", EntityViewSet)
+documents_router.register("entities", EntityOccurrenceViewSet)
+documents_router.register("legacy_entities_2", LegacyEntity2ViewSet)
 documents_router.register("legacy_entities", LegacyEntityViewSet)
 documents_router.register("dates", EntityDateViewSet)
 documents_router.register("errors", DocumentErrorViewSet)

documentcloud/common/wikidata.py

@@ -0,0 +1,30 @@
+# Third Party
+from wikidata.client import Client
+from wikidata.entity import EntityState
+
+
+class EasyWikidataEntity:
+    def __init__(self, wikidata_id):
+        client = Client()
+        self.entity = client.get(wikidata_id, load=True)
+        if self.entity.state != EntityState.loaded:
+            raise ValueError("Wikidata ID does not exist")
+
+    def get_urls(self):
+        return self.entity.data.get("sitelinks")
+
+    def get_names(self):
+        return self.entity.label.texts
+
+    def get_description(self):
+        return self.entity.description.texts
+
+    def get_values(self):
+        localized_name = self.get_names()
+        name = localized_name.get("en", next(iter(localized_name.values())))
+        return {
+            "wikipedia_url": self.get_urls(),
+            "localized_names": localized_name,
+            "name": name,
+            "description": self.get_description(),
+        }

documentcloud/conftest.py

@@ -11,6 +11,10 @@
     NoteFactory,
     SectionFactory,
 )
+from documentcloud.entities.tests.factories import (
+    EntityFactory,
+    EntityOccurrenceFactory,
+)
 from documentcloud.organizations.tests.factories import (
     OrganizationFactory,
     ProfessionalOrganizationFactory,
@@ -70,3 +74,13 @@ def organization():
 @pytest.fixture
 def pro_organization():
     return ProfessionalOrganizationFactory()
+
+
+@pytest.fixture
+def entity():
+    return EntityFactory()
+
+
+@pytest.fixture
+def entity_occurrence():
+    return EntityOccurrenceFactory()

documentcloud/documents/migrations/0051_auto_20230214_1451.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.9 on 2023-02-14 14:51
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0050_document_noindex'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='entityoccurrence',
+            name='document',
+            field=models.ForeignKey(help_text='The document this entity belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='legacy_entities_2', to='documents.document', verbose_name='document'),
+        ),
+        migrations.AlterField(
+            model_name='page',
+            name='document',
+            field=models.ForeignKey(help_text='The document this page belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='+', to='documents.document', verbose_name='document'),
+        ),
+    ]

documentcloud/documents/models.py

@@ -942,7 +942,7 @@ class EntityOccurrence(models.Model):
         verbose_name=_("document"),
         to="documents.Document",
         on_delete=models.CASCADE,
-        related_name="entities",
+        related_name="legacy_entities_2",
         help_text=_("The document this entity belongs to"),
     )
 

documentcloud/documents/views.py

@@ -825,7 +825,7 @@ def get_queryset(self):
             Document.objects.get_viewable(self.request.user),
             pk=self.kwargs["document_pk"],
         )
-        return self.document.entities.all()
+        return self.document.legacy_entities_2.all()
 
     def create(self, request, *args, **kwargs):
         """Initiate asyncrhonous creation of entities"""
@@ -846,7 +846,7 @@ def create(self, request, *args, **kwargs):
                     {"error": "Already processing"}, status=status.HTTP_400_BAD_REQUEST
                 )
 
-            if document.entities.exists():
+            if document.legacy_entities_2.exists():
                 return Response(
                     {"error": "Entities already created"},
                     status=status.HTTP_400_BAD_REQUEST,
@@ -863,7 +863,7 @@ def create(self, request, *args, **kwargs):
     def bulk_destroy(self, request, *args, **kwargs):
         """Delete all entities for the document"""
         if request.user.has_perm("documents.change_document", self.document):
-            self.document.entities.all().delete()
+            self.document.legacy_entities_2.all().delete()
             return Response(status=status.HTTP_204_NO_CONTENT)
         else:
             raise exceptions.PermissionDenied(

documentcloud/entities/admin.py

@@ -0,0 +1,49 @@
+# Django
+from django.contrib import admin
+
+# DocumentCloud
+from documentcloud.core.pagination import LargeTablePaginator
+from documentcloud.entities.models import Entity
+
+
+@admin.register(Entity)
+class EntityAdmin(admin.ModelAdmin):
+    """Entity Admin"""
+
+    list_display = (
+        "name",
+        "wikidata_id",
+        "user",
+        "access",
+    )
+    list_filter = ("access",)
+    search_fields = (
+        "name",
+        "wikidata_id",
+        "description",
+    )
+    show_full_result_count = False
+    paginator = LargeTablePaginator
+    ordering = ("pk",)
+    fields = (
+        "name",
+        "localized_names",
+        "wikidata_id",
+        "wikipedia_url",
+        "user",
+        "description",
+        "created_at",
+        "updated_at",
+        "access",
+    )
+    readonly_fields = (
+        "name",
+        "localized_names",
+        "wikidata_id",
+        "wikipedia_url",
+        "user",
+        "description",
+        "created_at",
+        "updated_at",
+        "access",
+    )

documentcloud/entities/apps.py

@@ -0,0 +1,7 @@
+# Django
+from django.apps import AppConfig
+
+
+class EntitiesConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "documentcloud.entities"

documentcloud/entities/choices.py

@@ -0,0 +1,13 @@
+# Django
+from django.utils.translation import gettext_lazy as _
+
+# Third Party
+from djchoices import ChoiceItem, DjangoChoices
+
+
+class EntityAccess(DjangoChoices):
+    # `api` specifies if this attribute should be accessible via the API
+    # Free and public to all.
+    public = ChoiceItem(0, _("Public"), api=True)
+    # Visible to both the owner and her organization.
+    private = ChoiceItem(2, _("Private"), api=True)

documentcloud/entities/migrations/0001_initial.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.2.9 on 2023-01-30 15:53
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Entity',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.JSONField()),
+                ('wikidata_id', models.CharField(max_length=16)),
+                ('wikipedia_url', models.JSONField()),
+                ('description', models.JSONField()),
+                ('created_at', models.DateTimeField(auto_now=True)),
+                ('updated_at', models.DateTimeField(auto_now_add=True)),
+                ('access', models.CharField(choices=[('Public', 0), ('Private', 1)], default='Public', max_length=20)),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='entities', to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

documentcloud/entities/migrations/0002_auto_20230201_1554.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.9 on 2023-02-01 15:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('entities', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='entity',
+            name='localized_names',
+            field=models.JSONField(default={}),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='entity',
+            name='name',
+            field=models.CharField(max_length=500),
+        ),
+    ]

documentcloud/entities/migrations/0003_auto_20230201_1814.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.9 on 2023-02-01 18:14
+
+from django.db import migrations
+import django.utils.timezone
+import documentcloud.core.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('entities', '0002_auto_20230201_1554'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='entity',
+            name='created_at',
+            field=documentcloud.core.fields.AutoCreatedField(db_index=True, default=django.utils.timezone.now, editable=False, help_text='Timestamp of when the entity was created', verbose_name='created at'),
+        ),
+        migrations.AlterField(
+            model_name='entity',
+            name='updated_at',
+            field=documentcloud.core.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, help_text='Timestamp of when the entitywas last updated', verbose_name='updated at'),
+        ),
+    ]

documentcloud/entities/migrations/0004_alter_entity_access.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.9 on 2023-02-01 19:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('entities', '0003_auto_20230201_1814'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='entity',
+            name='access',
+            field=models.IntegerField(choices=[(0, 'Public'), (2, 'Private')], help_text='Designates who may access this entity.', verbose_name='access'),
+        ),
+    ]

documentcloud/entities/migrations/0005_auto_20230206_1943.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.2.9 on 2023-02-06 19:43
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('entities', '0004_alter_entity_access'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='entity',
+            name='owner',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='entities', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='entity',
+            name='wikidata_id',
+            field=models.CharField(max_length=16, unique=True),
+        ),
+    ]

documentcloud/entities/migrations/0006_entity_metadata.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.9 on 2023-02-09 17:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('entities', '0005_auto_20230206_1943'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='entity',
+            name='metadata',
+            field=models.JSONField(null=True),
+        ),
+    ]