Support for DFIQ v1.1 (#3163)

* 1.1 compatibilty
* UUID field for DFIQ models
* Deep link to question
* support creation of scenarios & questions based on uuid

---------

Co-authored-by: Janosch <[email protected]>

test_data/dfiq/facets/F1001.yaml

@@ -1,7 +1,8 @@
-display_name: Test Facet
+name: Test Facet
 description: Test Facet
 type: facet
 id: F1001
+uuid: 79c6d69d-4de0-4fc0-af5f-b02248430104
 tags:
   - test
 parent_ids:

test_data/dfiq/questions/Q1001.yaml

@@ -1,7 +1,26 @@
-display_name: Test Question
+name: Test Question
 description: Test Question
 type: question
 id: Q1001
+uuid: bb094c9a-afb7-4daf-a853-7859d2f99d0c
+approaches:
+- name: Test approach
+  description: Test approach description
+  tags:
+  - Test Tag
+  references:
+  - '[Test Tag](http://go/yeti-dev)'
+  notes:
+    covered:
+    - Test covered
+    not_covered:
+    - Test not covered
+  steps:
+  - name: Test step
+    description: Test step description
+    stage: collection
+    type: ForensicArtifact
+    value: TestArtifact
 tags:
 parent_ids:
   - F1001

test_data/dfiq/scenarios/S1001.yaml

@@ -1,6 +1,7 @@
-display_name: Test Scenario
+name: Test Scenario
 description: Test Scenario
 type: scenario
 id: S1001
+uuid: 6e5d9e63-b477-4392-8742-c85ac9b653e2
 tags:
   - test

timesketch/api/v1/resources/__init__.py

@@ -285,6 +285,7 @@ class ResourceMixin(object):
         "display_name": fields.String,
         "description": fields.String,
         "dfiq_identifier": fields.String,
+        "uuid": fields.String,
         "spec_json": fields.String,
         "user": fields.Nested(user_fields),
         "approaches": fields.List(fields.Nested(approach_fields)),
@@ -299,6 +300,7 @@ class ResourceMixin(object):
         "display_name": fields.String,
         "description": fields.String,
         "dfiq_identifier": fields.String,
+        "uuid": fields.String,
         "spec_json": fields.String,
         "user": fields.Nested(user_fields),
         "questions": fields.List(fields.Nested(question_fields)),
@@ -317,6 +319,7 @@ class ResourceMixin(object):
         "display_name": fields.String,
         "description": fields.String,
         "dfiq_identifier": fields.String,
+        "uuid": fields.String,
         "spec_json": fields.String,
         "user": fields.Nested(user_fields),
         "facets": fields.List(fields.Nested(facet_fields)),

timesketch/api/v1/resources/scenarios.py

@@ -14,6 +14,7 @@
 """API for asking Timesketch scenarios for version 1 of the Timesketch API."""
 
 import logging
+import json
 
 from flask import jsonify
 from flask import request
@@ -135,19 +136,38 @@ def post(self, sketch_id):
 
         dfiq_id = form.get("dfiq_id")
         display_name = form.get("display_name")
+        uuid = form.get("uuid")
+
+        scenario = None
+
+        if uuid:
+            scenario = next(
+                (s for s in dfiq.scenarios if s.uuid == uuid),
+                None,
+            )
+        elif dfiq_id:
+            scenario = next(
+                (s for s in dfiq.scenarios if s.id == dfiq_id),
+                None,
+            )
+        elif display_name:
+            scenario = next(
+                (s for s in dfiq.scenarios if s.name == display_name),
+                None,
+            )
 
-        scenario = next(
-            (scenario for scenario in dfiq.scenarios if scenario.id == dfiq_id),
-            None,
-        )
         if not scenario:
-            abort(HTTP_STATUS_CODE_NOT_FOUND, f"No such scenario template: {dfiq_id}")
+            abort(
+                HTTP_STATUS_CODE_NOT_FOUND,
+                f"No scenario found matching the provided data: {form}",
+            )
 
         if not display_name:
             display_name = scenario.name
 
         scenario_sql = Scenario(
             dfiq_identifier=scenario.id,
+            uuid=scenario.uuid,
             name=scenario.name,
             display_name=display_name,
             description=scenario.description,
@@ -163,6 +183,7 @@ def post(self, sketch_id):
             )
             facet_sql = Facet(
                 dfiq_identifier=facet.id,
+                uuid=facet.uuid,
                 name=facet.name,
                 display_name=facet.name,
                 description=facet.description,
@@ -183,30 +204,27 @@ def post(self, sketch_id):
                 )
                 question_sql = InvestigativeQuestion(
                     dfiq_identifier=question.id,
+                    uuid=question.uuid,
                     name=question.name,
                     display_name=question.name,
                     description=question.description,
                     spec_json=question.to_json(),
                     sketch=sketch,
-                    scenario=scenario_sql,
+                    # scenario=scenario_sql,
                     user=current_user,
                 )
-                facet_sql.questions.append(question_sql)
-
-                for approach_id in question.approaches:
-                    approach = next(
-                        (
-                            approach
-                            for approach in dfiq.approaches
-                            if approach.id == approach_id
-                        ),
-                        None,
-                    )
+                # facet_sql.questions.append(question_sql)
+
+                # TODO: This is a tmp hack to make all questions oprhaned!
+                # We need to fix this by loading connected questions as well in
+                # the frontend!
+                db_session.add(question_sql)
+
+                for approach in question.approaches:
                     approach_sql = InvestigativeQuestionApproach(
-                        dfiq_identifier=approach.id,
                         name=approach.name,
                         display_name=approach.name,
-                        description=approach.description.get("details", ""),
+                        description=approach.description,
                         spec_json=approach.to_json(),
                         user=current_user,
                     )
@@ -466,8 +484,8 @@ def get(self):
         if not dfiq:
             return jsonify({"objects": []})
 
-        scenarios = [scenario.__dict__ for scenario in dfiq.questions]
-        return jsonify({"objects": scenarios})
+        questions = [json.loads(question.to_json()) for question in dfiq.questions]
+        return jsonify({"objects": questions})
 
 
 class QuestionListResource(resources.ResourceMixin, Resource):
@@ -493,44 +511,44 @@ def post(self, sketch_id):
         scenario_id = form.get("scenario_id")
         facet_id = form.get("facet_id")
         template_id = form.get("template_id")
+        uuid = form.get("uuid")
 
         scenario = Scenario.get_by_id(scenario_id) if scenario_id else None
         facet = Facet.get_by_id(facet_id) if facet_id else None
 
-        if template_id:
+        if template_id or uuid:
             dfiq = load_dfiq_from_config()
             if not dfiq:
                 abort(
                     HTTP_STATUS_CODE_NOT_FOUND, "DFIQ is not configured on this server"
                 )
-            dfiq_question = [
-                question for question in dfiq.questions if question.id == template_id
-            ][0]
+            if uuid:
+                dfiq_question = [
+                    question for question in dfiq.questions if question.uuid == uuid
+                ][0]
+            else:
+                dfiq_question = [
+                    question
+                    for question in dfiq.questions
+                    if question.id == template_id
+                ][0]
 
         if dfiq_question:
             new_question = InvestigativeQuestion(
                 dfiq_identifier=dfiq_question.id,
+                uuid=dfiq_question.uuid,
                 name=dfiq_question.name,
                 display_name=dfiq_question.name,
                 description=dfiq_question.description,
                 spec_json=dfiq_question.to_json(),
                 sketch=sketch,
                 user=current_user,
             )
-            for approach_id in dfiq_question.approaches:
-                approach = next(
-                    (
-                        approach
-                        for approach in dfiq.approaches
-                        if approach.id == approach_id
-                    ),
-                    None,
-                )
+            for approach in dfiq_question.approaches:
                 approach_sql = InvestigativeQuestionApproach(
-                    dfiq_identifier=approach.id,
                     name=approach.name,
                     display_name=approach.name,
-                    description=approach.description.get("details", ""),
+                    description=approach.description,
                     spec_json=approach.to_json(),
                     user=current_user,
                 )

timesketch/frontend-ng/src/components/Scenarios/QuestionApproach.vue

@@ -18,7 +18,7 @@ limitations under the License.
     <div
       class="pb-4 markdown-body"
       style="font-size: 1em; background-color: transparent"
-      v-html="toHtml(approach.description.details)"
+      v-html="toHtml(approach.description)"
     ></div>
 
     <div v-if="opensearchQueries.length" class="pb-4">
@@ -41,11 +41,11 @@ limitations under the License.
     <v-expand-transition>
       <div v-if="showDetails" class="mt-3">
         <!-- References -->
-        <div v-if="approach.description.references && approach.description.references.length">
+        <div v-if="approach.references && approach.references.length">
           <v-icon class="mr-2">mdi-link-variant</v-icon>
           <strong>References</strong>
           <ul class="mb-4 mt-2 markdown-body" style="line-height: 70%; background-color: transparent">
-            <li v-for="reference in approach.description.references" :key="reference">
+            <li v-for="reference in approach.references" :key="reference">
               <div v-html="toHtml(reference)" style="font-size: 0.9em"></div>
             </li>
           </ul>
@@ -55,15 +55,15 @@ limitations under the License.
           <v-icon color="success" class="mr-2">mdi-check</v-icon>
           <strong>Covered</strong>
           <ul class="mt-2">
-            <li v-for="note in approach._view.notes.covered" :key="note">{{ note }}</li>
+            <li v-for="note in approach.notes.covered" :key="note">{{ note }}</li>
           </ul>
         </v-sheet>
 
         <v-sheet style="max-width: 80%; background-color: transparent">
           <v-icon color="error" class="mr-2">mdi-close</v-icon>
           <strong>Not covered</strong>
           <ul class="mt-2">
-            <li v-for="note in approach._view.notes.not_covered" :key="note">{{ note }}</li>
+            <li v-for="note in approach.notes.not_covered" :key="note">{{ note }}</li>
           </ul>
         </v-sheet>
       </div>
@@ -90,17 +90,13 @@ export default {
     },
     opensearchQueries() {
       let opensearchQueries = []
-      this.approach._view.processors.forEach((processor) => {
-        processor.analysis.forEach((analysis) => {
-          if (analysis.name === 'OpenSearch') {
-            analysis.steps.forEach((step) => {
-              if (step.type === 'opensearch-query') {
-                opensearchQueries.push(step)
-              }
-            })
+      if (this.approach.steps) {
+        this.approach.steps.forEach((step) => {
+          if (step.type === 'opensearch-query') {
+            opensearchQueries.push(step)
           }
         })
-      })
+      }
       return opensearchQueries
     },
   },