Extract content urls

contentcuration/automation/migrations/0001_initial.py

@@ -1,26 +1,32 @@
-# Generated by Django 3.2.24 on 2024-07-11 11:33
+# Generated by Django 3.2.24 on 2024-07-24 13:18
 import uuid
 
+import django.db.models.deletion
 from django.db import migrations
 from django.db import models
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = [
+        ('kolibri_public', '0003_alter_file_preset'),
     ]
 
     operations = [
         migrations.CreateModel(
             name='RecommendationsCache',
             fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('request', models.JSONField(default=dict, null=True)),
-                ('response', models.JSONField(default=dict, null=True)),
-                ('priority', models.IntegerField(default=0)),
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True,
+                                        serialize=False)),
+                ('request_hash', models.CharField(max_length=32, null=True)),
+                ('rank', models.FloatField(default=0.0, null=True)),
+                ('override_threshold', models.BooleanField(default=False)),
                 ('timestamp', models.DateTimeField(auto_now_add=True)),
+                ('response', models.ForeignKey(blank=True, null=True,
+                                               on_delete=django.db.models.deletion.SET_NULL,
+                                               related_name='recommendations',
+                                               to='kolibri_public.contentnode')),
             ],
         ),
     ]

contentcuration/automation/models.py

@@ -1,12 +1,19 @@
 import uuid
 
 from django.db import models
-from django.db.models import JSONField
+from kolibri_public.models import ContentNode
 
 
 class RecommendationsCache(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    request = JSONField(default=dict, null=True)
-    response = JSONField(default=dict, null=True)
+    request_hash = models.CharField(max_length=32, null=True)
+    response = models.ForeignKey(
+        ContentNode,
+        null=True,
+        blank=True,
+        related_name='recommendations',
+        on_delete=models.SET_NULL,
+    )
     rank = models.FloatField(default=0.0, null=True)
+    override_threshold = models.BooleanField(default=False)
     timestamp = models.DateTimeField(auto_now_add=True)

contentcuration/contentcuration/tests/utils/test_recommendations.py

@@ -34,6 +34,7 @@ def setUp(self):
                 }
             ]
         }
+        self.channel_id = 'test_channel_id'
         self.resources = [
             MagicMock(spec=ContentNode),
         ]
@@ -42,6 +43,51 @@ def test_adapter_initialization(self):
         self.assertIsNotNone(self.adapter)
         self.assertIsInstance(self.adapter, RecommendationsAdapter)
 
+    def test_generate_embeddings_connect_failure(self):
+        mock_response = MagicMock(spec=EmbeddingsResponse)
+
+        self.adapter.backend.connect.return_value = False
+        self.adapter.backend.make_request.return_value = mock_response
+        with self.assertRaises(errors.ConnectionError):
+            self.adapter.generate_embeddings(EmbedTopicsRequest(
+                method='POST',
+                url='http://test.com',
+                path='/test/path',
+            ))
+        self.adapter.backend.connect.assert_called_once()
+        self.adapter.backend.make_request.assert_not_called()
+
+    def test_generate_embeddings(self):
+        mock_response = MagicMock(spec=EmbeddingsResponse)
+        mock_response.error = None
+
+        self.adapter.backend.connect.return_value = True
+        self.adapter.backend.make_request.return_value = mock_response
+        response = self.adapter.generate_embeddings(EmbedTopicsRequest(
+            method='POST',
+            url='http://test.com',
+            path='/test/path',
+        ))
+        self.adapter.backend.connect.assert_called_once()
+        self.adapter.backend.make_request.assert_called_once()
+        self.assertIsInstance(response, EmbeddingsResponse)
+
+    def test_generate_embeddings_failure(self):
+        mock_response = MagicMock(spec=EmbeddingsResponse)
+        mock_response.error = {}
+
+        self.adapter.backend.connect.return_value = True
+        self.adapter.backend.make_request.return_value = mock_response
+        response = self.adapter.generate_embeddings(EmbedTopicsRequest(
+            method='POST',
+            url='http://test.com',
+            path='/test/path',
+        ))
+        self.adapter.backend.connect.assert_called_once()
+        self.adapter.backend.make_request.assert_called_once()
+        self.assertIsInstance(response, EmbeddingsResponse)
+        self.assertIsNotNone(response.error)
+
     @patch('contentcuration.utils.recommendations.EmbedTopicsRequest')
     def test_get_recommendations_backend_connect_success(self, get_recommendations_request_mock):
         mock_response = MagicMock(spec=RecommendationsResponse)
@@ -73,7 +119,7 @@ def test_embed_content_backend_connect_success(self, embed_content_request_mock)
 
         self.adapter.backend.connect.return_value = True
         self.adapter.backend.make_request.return_value = mock_response
-        response = self.adapter.embed_content(self.resources)
+        response = self.adapter.embed_content(self.channel_id, self.resources)
         self.adapter.backend.connect.assert_called_once()
         self.adapter.backend.make_request.assert_called_once()
         self.assertIsInstance(response, bool)
@@ -83,6 +129,6 @@ def test_embed_content_backend_connect_success(self, embed_content_request_mock)
     def test_embed_content_backend_connect_failure(self, embed_content_request_mock):
         self.adapter.backend.connect.return_value = False
         with self.assertRaises(errors.ConnectionError):
-            self.adapter.embed_content(self.resources)
+            self.adapter.embed_content(self.channel_id, self.resources)
         self.adapter.backend.connect.assert_called_once()
         self.adapter.backend.make_request.assert_not_called()

contentcuration/contentcuration/utils/automation_manager.py

@@ -1,8 +1,11 @@
 from typing import Any
 from typing import Dict
 from typing import List
+from typing import Union
 
-from contentcuration.models import ContentNode
+from kolibri_public.models import ContentNode as PublicContentNode
+
+from contentcuration.models import ContentNode as ContentNode
 from contentcuration.utils.recommendations import RecommendationsAdapter
 from contentcuration.utils.recommendations import RecommendationsBackendFactory
 
@@ -13,15 +16,16 @@ def __init__(self):
         self.instance = self.factory.create_backend()
         self.adapter = RecommendationsAdapter(self.instance)
 
-    def generate_embeddings(self, nodes: List[ContentNode]):
+    def generate_embeddings(self, channel_id: str, nodes: List[Union[ContentNode, PublicContentNode]]):
         """
         Generates embeddings for the given list of nodes. This process is async.
 
+        :param channel_id: The channel id to which the nodes belong.
         :param nodes: The list of nodes for which to generate embeddings.
 
         :return: A boolean indicating that the process has started.
         """
-        return self.adapter.embed_content(nodes)
+        return self.adapter.embed_content(channel_id, nodes)
 
     def load_recommendations(self, topic: Dict[str, Any], override_threshold=False):
         """