diff --git a/timesketch/api/v1/resources/timeline.py b/timesketch/api/v1/resources/timeline.py index 78dfc38f57..3c81023d44 100644 --- a/timesketch/api/v1/resources/timeline.py +++ b/timesketch/api/v1/resources/timeline.py @@ -20,6 +20,7 @@ import six import opensearchpy +from flask import jsonify from flask import request from flask import abort from flask import current_app @@ -40,6 +41,7 @@ from timesketch.models.sketch import SearchIndex from timesketch.models.sketch import Sketch from timesketch.models.sketch import Timeline +from timesketch.lib.aggregators import manager as aggregator_manager logger = logging.getLogger("timesketch.timeline_api") @@ -531,3 +533,100 @@ def post(self): utils.update_sketch_last_activity(sketch) return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED) + + +# TODO(Issue 3200): Research more efficient ways to gather unique fields. +class TimelineFieldsResource(resources.ResourceMixin, Resource): + """Resource to retrieve unique fields present in a timeline. + + This resource aggregates data types within a timeline and then queries + OpenSearch to retrieve all unique fields present across those data types, + excluding default Timesketch fields. + """ + + @login_required + def get(self, sketch_id, timeline_id): + """Handles GET request to retrieve unique fields in a timeline. + + Args: + sketch_id (int): The ID of the sketch. + timeline_id (int): The ID of the timeline. + + Returns: + flask.wrappers.Response: A JSON response containing a list of + unique fields in the timeline, sorted alphabetically. Returns + an empty list if no fields are found or if there's an error. + Possible error codes: 400, 403, 404. + """ + + sketch = Sketch.get_with_acl(sketch_id) + if not sketch: + abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.") + if not sketch.has_permission(current_user, "read"): + abort( + HTTP_STATUS_CODE_FORBIDDEN, + "User does not have read access controls on sketch.", + ) + + timeline = Timeline.get_by_id(timeline_id) + if not timeline: + abort(HTTP_STATUS_CODE_NOT_FOUND, "No timeline found with this ID.") + + # Check that this timeline belongs to the sketch + if timeline.sketch.id != sketch.id: + abort( + HTTP_STATUS_CODE_NOT_FOUND, + "The timeline does not belong to the sketch.", + ) + + index_name = timeline.searchindex.index_name + timeline_fields = set() + + # 1. Get distinct data types for the timeline using aggregation + aggregator_name = "field_bucket" + aggregator_parameters = { + "field": "data_type", + "limit": "10000", # Get all data types + } + + agg_class = aggregator_manager.AggregatorManager.get_aggregator(aggregator_name) + if not agg_class: + abort(HTTP_STATUS_CODE_NOT_FOUND, f"Aggregator {aggregator_name} not found") + + aggregator = agg_class( + sketch_id=sketch_id, indices=[index_name], timeline_ids=[timeline_id] + ) + result_obj = aggregator.run(**aggregator_parameters) + + if not result_obj: + abort(HTTP_STATUS_CODE_BAD_REQUEST, "Error running data type aggregation.") + + data_types = sorted([bucket["data_type"] for bucket in result_obj.values]) + + # 2. For each data type, query for a single event to get fields + for data_type in data_types: + query_filter = {"indices": [timeline_id], "size": 1} + + try: + result = self.datastore.search( + sketch_id=sketch_id, + query_string=f'data_type:"{data_type}"', + query_filter=query_filter, + query_dsl=None, + indices=[index_name], + timeline_ids=[timeline_id], + ) + except ValueError as e: + abort(HTTP_STATUS_CODE_BAD_REQUEST, str(e)) + + if isinstance(result, dict) and result.get("hits", {}).get("hits", []): + event = result["hits"]["hits"][0]["_source"] + for field in event: + if field not in [ + "datetime", + "timestamp", + "__ts_timeline_id", + ]: + timeline_fields.add(field) + + return jsonify({"objects": sorted(list(timeline_fields))}) diff --git a/timesketch/api/v1/routes.py b/timesketch/api/v1/routes.py index 9846d90a7c..0993110caa 100644 --- a/timesketch/api/v1/routes.py +++ b/timesketch/api/v1/routes.py @@ -56,6 +56,7 @@ from .resources.explore import QueryResource from .resources.timeline import TimelineResource from .resources.timeline import TimelineListResource +from .resources.timeline import TimelineFieldsResource from .resources.searchindex import SearchIndexListResource from .resources.searchindex import SearchIndexResource from .resources.session import SessionResource @@ -167,6 +168,10 @@ TimelineResource, "/sketches/<int:sketch_id>/timelines/<int:timeline_id>/", ), + ( + TimelineFieldsResource, + "/sketches/<int:sketch_id>/timelines/<int:timeline_id>/fields/", + ), (SearchIndexListResource, "/searchindices/"), (SearchIndexResource, "/searchindices/<int:searchindex_id>/"), ( diff --git a/timesketch/frontend-ng/dist/vis_placeholder.png b/timesketch/frontend-ng/dist/vis_placeholder.png new file mode 100644 index 0000000000..bbcdd1e865 Binary files /dev/null and b/timesketch/frontend-ng/dist/vis_placeholder.png differ diff --git a/timesketch/frontend-ng/public/vis_placeholder.png b/timesketch/frontend-ng/public/vis_placeholder.png new file mode 100644 index 0000000000..bbcdd1e865 Binary files /dev/null and b/timesketch/frontend-ng/public/vis_placeholder.png differ diff --git a/timesketch/frontend-ng/src/components/Analyzer/TimelineSearch.vue b/timesketch/frontend-ng/src/components/Analyzer/TimelineSearch.vue index dd35c05367..f0786bf321 100644 --- a/timesketch/frontend-ng/src/components/Analyzer/TimelineSearch.vue +++ b/timesketch/frontend-ng/src/components/Analyzer/TimelineSearch.vue @@ -92,8 +92,17 @@ export default { }, analyzerTimelineId: { handler: function (id) { - if (id) this.selectedTimelines.push(id) - if (!id) this.selectedTimelines = [] + if (Array.isArray(id)) { + this.selectedTimelines = id + } else { + if (id) { + this.selectedTimelines.push(id) + } + else { + this.selectedTimelines = [] + } + } + }, }, }, diff --git a/timesketch/frontend-ng/src/components/Explore/EventList.vue b/timesketch/frontend-ng/src/components/Explore/EventList.vue index d957f27c3a..9d6a9f5678 100644 --- a/timesketch/frontend-ng/src/components/Explore/EventList.vue +++ b/timesketch/frontend-ng/src/components/Explore/EventList.vue @@ -30,12 +30,10 @@ limitations under the License. <p> <v-dialog v-model="saveSearchMenu" v-if="!disableSaveSearch" width="500"> <template v-slot:activator="{ on, attrs }"> - <div v-bind="attrs" v-on="on"> - <v-btn small depressed> - <v-icon left small title="Save current search">mdi-content-save-outline</v-icon> - Save search - </v-btn> - </div> + <v-btn small depressed v-bind="attrs" v-on="on" title="Save Search"> + <v-icon left small >mdi-content-save-outline</v-icon> + Save search + </v-btn> </template> <v-card class="pa-4"> diff --git a/timesketch/frontend-ng/src/components/Visualization/AggregationConfig.vue b/timesketch/frontend-ng/src/components/Visualization/AggregationConfig.vue index 3fb73e778a..24b26904f5 100644 --- a/timesketch/frontend-ng/src/components/Visualization/AggregationConfig.vue +++ b/timesketch/frontend-ng/src/components/Visualization/AggregationConfig.vue @@ -20,6 +20,8 @@ limitations under the License. <TsEventFieldSelect :field="selectedField" @selectedField="selectedField = $event" + :timelineFields="timelineFields" + :loadingFields="loadingFields" :rules="[rules.required]" > </TsEventFieldSelect> @@ -114,6 +116,14 @@ export default { splitByTimeline: { type: Boolean, }, + timelineFields: { + type: Array, + default: () => [], + }, + loadingFields: { + type: Boolean, + default: false + }, }, data() { return { diff --git a/timesketch/frontend-ng/src/components/Visualization/AggregationEventSelect.vue b/timesketch/frontend-ng/src/components/Visualization/AggregationEventSelect.vue new file mode 100644 index 0000000000..1cb4d14ea7 --- /dev/null +++ b/timesketch/frontend-ng/src/components/Visualization/AggregationEventSelect.vue @@ -0,0 +1,205 @@ +<!-- +Copyright 2023 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<template> + <v-container class='ma-0'> + <v-row class="mt-3"> + <v-col > + <!-- <v-container class="ma-0"> --> + <ts-timeline-search + componentName="visualization" + :analyzer-timeline-id="timelineIDs" + @selectedTimelines="selectedTimelineIDs = $event" + > + </ts-timeline-search> + <!-- </v-container> --> + </v-col> + </v-row> + <v-row class="mt-n10"> + <v-col> + <v-text-field + outlined + autofocus + label="Event query" + v-model="selectedQueryString" + > + </v-text-field> + </v-col> + </v-row> + <v-row class="mt-n10"> + <v-col> + <v-select + v-bind="$attrs" + v-model="selectedRecentSearch" + :items="allRecentSearches" + clearable + solo + dense + label="Recent search" + :disabled="!!selectedSavedSearch" + > + </v-select> + </v-col> + <v-col> + <v-select + v-bind="$attrs" + v-model="selectedSavedSearch" + :items="allSavedSearches" + clearable + solo + dense + label="Saved search" + :disabled="!!selectedRecentSearch" + > + </v-select> + </v-col> + </v-row> + </v-container> +</template> + +<script> +import TsTimelineSearch from '../Analyzer/TimelineSearch.vue' + +export default { + components: { + TsTimelineSearch, + }, + props: { + queryString: { + type: String + }, + queryChips: { + type: Array + }, + recentSearch: { + type: Object, + }, + timelineIDs: { + type: Array, + default: function() { + return [] + }, + }, + }, + data() { + return { + selectedFilter: '', + selectedRecentSearch: null, + selectedSavedSearch: null, + selectedTimelineIDs: this.timelineIDs, + selectedQueryString: this.queryString, + selectedQueryChips: this.queryChips + } + }, + computed: { + sketch() { + return this.$store.state.sketch + }, + allReadyTimelines() { + // Sort alphabetically based on timeline name. + const timelines = this.sketch.timelines.filter( + tl => tl.status[0].status === 'ready' + ); + timelines.sort((a, b) => a.name.localeCompare(b.name)) + return timelines; + }, + allRecentSearches() { + let searchHistory = this.$store.state.searchHistory + if (Array.isArray(searchHistory)) { + let recentSearches = this.$store.state.searchHistory.map( + (view) => { + return { + text: `${view['query_string']} (${view['query_result_count']})`, + value: view + } + } + ) + return recentSearches + } + return [] + }, + allSavedSearches() { + let savedSearches = this.$store.state.meta.views.map( + (view) => { + return { + text: view['name'], value: view + } + } + ) + return savedSearches + } + }, + watch: { + queryString() { + console.log(this.queryString) + if (this.queryString) { + return + } + this.selectedRecentSearch = null + this.selectedSavedSearch = null + }, + selectedSavedSearch() { + if (!this.selectedSavedSearch) { + this.selectedQueryString = '' + this.selectedTimelineIDs = [] + this.selectedQueryChips = null + } else { + this.selectedQueryString = this.selectedSavedSearch.query + let queryFilter = JSON.parse(this.selectedSavedSearch.filter) + + let indices = queryFilter.indices + if (indices === '_all') { + this.selectedTimelineIDs = this.allReadyTimelines.map((x) => x.id) + } else { + this.selectedTimelineIDs = indices + } + + this.selectedQueryChips = queryFilter.chips.filter( + (chip) => chip.type === 'label' || chip.type === 'term' || chip.type === 'datetime_range' + ); + } + }, + selectedRecentSearch() { + if (!this.selectedRecentSearch) { + this.selectedQueryString = '' + this.selectedTimelineIDs = [] + this.selectedQueryChips = null + } else { + this.selectedQueryString = this.selectedRecentSearch.query_string + let queryFilter = JSON.parse(this.selectedRecentSearch.query_filter) + let indices = queryFilter.indices + if (indices === '_all') { + this.selectedTimelineIDs = this.allReadyTimelines.map((x) => x.id) + } else { + this.selectedTimelineIDs = indices + } + + this.selectedQueryChips = queryFilter.chips.filter( + (chip) => chip.type === 'label' || chip.type === 'term' || chip.type === 'datetime_range' + ); + } + }, + selectedTimelineIDs() { + this.$emit('updateTimelineIDs', this.selectedTimelineIDs) + }, + selectedQueryString() { + this.$emit('updateQueryString', this.selectedQueryString) + }, + selectedQueryChips() { + this.$emit('updateQueryChips', this.selectedQueryChips) + } + } +} +</script> diff --git a/timesketch/frontend-ng/src/components/Visualization/EventFieldSelect.vue b/timesketch/frontend-ng/src/components/Visualization/EventFieldSelect.vue index a6976e2bb1..bf4694c915 100644 --- a/timesketch/frontend-ng/src/components/Visualization/EventFieldSelect.vue +++ b/timesketch/frontend-ng/src/components/Visualization/EventFieldSelect.vue @@ -17,19 +17,20 @@ limitations under the License. <v-autocomplete outlined v-model="selectedField" - :items="allNonTimestampFields" + :items="mappedTimelineFields" label="Field name to aggregate" + :loading="loadingFields" @input="$emit('selectedField', $event)" > - <template + <template #item="{ item, on, attrs }" > - <v-list-item - v-on="on" + <v-list-item + v-on="on" v-bind="attrs" > <v-list-item-avatar> - <v-icon> + <v-icon> {{ item.value.type === 'text' ? 'mdi-code-string' : 'mdi-pound-box' }} </v-icon> </v-list-item-avatar> @@ -38,11 +39,11 @@ limitations under the License. </v-list-item-content> </v-list-item> </template> - <template + <template #selection="{ item }" > - <v-icon> - {{ item.value.type === 'text' ? 'mdi-code-string' : 'mdi-pound-box' }} + <v-icon> + {{ item.value.type === 'text' ? 'mdi-code-string' : 'mdi-pound-box' }} </v-icon> {{ item.text }} </template> @@ -56,34 +57,40 @@ export default { field: { type: Object, }, + timelineFields: { + type: Array, + default: () => [], + }, + loadingFields: { + type: Boolean, + default: false + }, }, data() { return { selectedField: this.field, - } }, computed: { - allNonTimestampFields() { - let mappings = this.$store.state.meta.mappings - .filter( - (mapping) => { - return ( - mapping['field'] !== 'datetime' - && mapping['field'] !== 'timestamp' - ) - }) - .map( - (mapping) => { - return {text: mapping['field'], value: mapping}} - ) - return mappings + mappedTimelineFields() { + const mappings = this.$store.state.meta.mappings; + + return this.timelineFields.map(field => { + const mapping = mappings.find(m => m.field === field); + const type = mapping ? mapping.type : 'unknown'; + return { text: field, value: { field, type } }; + }); }, }, watch: { field() { this.selectedField = this.field - } + }, + timelineFields(newFields) { + if (!newFields || newFields.length === 0) { + this.selectedField = null; + } + }, } } </script> diff --git a/timesketch/frontend-ng/src/components/Visualization/VisualizationEditor.vue b/timesketch/frontend-ng/src/components/Visualization/VisualizationEditor.vue index 111e6bebcd..49b3ee1d4b 100644 --- a/timesketch/frontend-ng/src/components/Visualization/VisualizationEditor.vue +++ b/timesketch/frontend-ng/src/components/Visualization/VisualizationEditor.vue @@ -36,19 +36,20 @@ limitations under the License. </v-btn> </v-toolbar> <v-divider class="mx-3"></v-divider> - <v-row class="mt-3"> - <v-col > - <v-container class="ma-0"> - <ts-timeline-search - componentName="visualization" - @selectedTimelines="selectedTimelineIDs = $event"></ts-timeline-search> - </v-container> - </v-col> - </v-row> <v-row class="mt-3"> <v-col> + <TsAggregationEventSelect + @updateTimelineIDs="getTimelineFields" + :timelineIDs="selectedTimelineIDs" + @updateQueryString="selectedQueryString = $event" + :queryString="selectedQueryString" + @updateQueryChips="selectedQueryChips = $event" + :queryChips="selectedQueryChips" + > + </TsAggregationEventSelect> <TsAggregationConfig - @enabled="selectedTimelineIDs.length > 0" + :timelineFields="availableTimelineFields" + :loadingFields="loadingFields" :field="selectedField" @updateField="selectedField = $event" :aggregator="selectedAggregator" @@ -86,7 +87,14 @@ limitations under the License. @updateShowDataLabels="selectedShowDataLabels = $event" ></TsChartConfig> </v-col> - <v-col cols="8"> + <v-col cols="8" :class="chartSeries == null? 'd-flex justify-center align-center' : ''"> + <v-img + v-if="chartSeries == null" + src="/dist/vis_placeholder.png" + max-width="600" + max-height="500" + contain + ></v-img> <TsChartCard v-if="chartSeries && selectedChartType" :fieldName="selectedField.field" @@ -120,7 +128,7 @@ limitations under the License. text color="primary" @click="loadAggregationData" - :disabled="selectedTimelineIDs.length == 0 || !( + :disabled="!validAggregation || !( selectedField && selectedAggregator && selectedChartType @@ -133,7 +141,7 @@ limitations under the License. <v-btn text @click="clear" - :disabled="!( + :disabled="!validAggregation || !( selectedField && selectedAggregator && selectedChartType @@ -157,14 +165,14 @@ import ApiClient from '../../utils/RestApiClient' import TsAggregationConfig from './AggregationConfig.vue' import TsChartConfig from './ChartConfig.vue' import TsChartCard from './ChartCard.vue' -import TsTimelineSearch from '../Analyzer/TimelineSearch.vue' +import TsAggregationEventSelect from './AggregationEventSelect.vue' export default { components: { TsAggregationConfig, TsChartConfig, TsChartCard, - TsTimelineSearch, + TsAggregationEventSelect, }, props: { aggregator: { @@ -208,6 +216,7 @@ export default { }, queryString: { type: String, + default: "*" }, range: { type: Object, @@ -283,9 +292,14 @@ export default { selectedYTitle: this.yTitle, renameVisualDialog: false, selectedChartTitleDraft: this.selectedChartTitleDraft, + availableTimelineFields: [], + loadingFields: false, } }, computed: { + validAggregation() { + return this.selectedTimelineIDs.length > 0 && this.selectedQueryString + }, sketch() { return this.$store.state.sketch }, @@ -304,15 +318,53 @@ export default { } return undefined }, - currentQueryString() { - const currentSearchNode = this.$store.state.currentSearchNode - if (!currentSearchNode) { - return "" - } - return currentSearchNode.query_string - }, }, methods: { + getTimelineFields(timelineIDs) { + this.selectedTimelineIDs = timelineIDs + if (!timelineIDs || timelineIDs.length === 0 ) { + this.availableTimelineFields = [] + return; + } + + this.loadingFields = true; + + if (timelineIDs.length === 1) { + ApiClient.getTimelineFields(this.sketch.id, timelineIDs[0]) + .then(response => { + this.availableTimelineFields = response.data.objects + this.loadingFields = false + }) + .catch(error => { + console.error("Error fetching fields:", error); + this.availableTimelineFields = []; + this.loadingFields = false + }); + } + else { + const promises = timelineIDs.map(timelineID => { + return ApiClient.getTimelineFields(this.sketch.id, timelineID) + .then(response => response.data.objects) + .catch(error => { + console.error("Error fetching timeline fields:", error); + return [] + }) + }) + + Promise.all(promises) + .then(results => { + // Flatten the arrays and create a set to guarantee uniqueness + const intersection = results.reduce((a, b) => a.filter(c => b.includes(c))); + this.availableTimelineFields = intersection + this.loadingFields = false; + }) + .catch(error => { + console.error("Error in Promise.all", error) + this.availableTimelineFields = [] + this.loadingFields = false; + }) + } + }, rename() { this.renameVisualDialog = false this.selectedChartTitle = this.selectedChartTitleDraft diff --git a/timesketch/frontend-ng/src/utils/RestApiClient.js b/timesketch/frontend-ng/src/utils/RestApiClient.js index 3fe3809479..b514354d8b 100644 --- a/timesketch/frontend-ng/src/utils/RestApiClient.js +++ b/timesketch/frontend-ng/src/utils/RestApiClient.js @@ -111,6 +111,9 @@ export default { getSketchTimelineAnalysis(sketchId, timelineId) { return RestApiClient.get('/sketches/' + sketchId + '/timelines/' + timelineId + '/analysis/') }, + getTimelineFields(sketchId, timelineId){ + return RestApiClient.get('/sketches/' + sketchId + '/timelines/' + timelineId + '/fields/') + }, saveSketchTimeline(sketchId, timelineId, name, description, color) { let formData = { name: name, diff --git a/timesketch/lib/aggregators/apex.py b/timesketch/lib/aggregators/apex.py index f3a0b818c5..c52b71d150 100644 --- a/timesketch/lib/aggregators/apex.py +++ b/timesketch/lib/aggregators/apex.py @@ -137,7 +137,8 @@ def add_match_phrase_filter(self, field, value, clause="must"): if clause not in self._VALID_QUERY_CLAUSES: raise ValueError(f"Unknown boolean clause {clause}") - + if isinstance(value, str): + field = f"{field}.keyword" self.bool_queries[clause].append({"match_phrase": {field: {"query": value}}}) def add_term_filter(self, field, value, clause="filter", term_type="term"): @@ -409,7 +410,7 @@ def _build_aggregation_query_spec(self, aggregator_options): elif chip_type == "datetime_range": aggregation_query.add_datetime_range(chip_value, chip_operator) elif chip_type == "term": - aggregation_query.add_term_filter( + aggregation_query.add_match_phrase_filter( chip_field, chip_value, chip_operator ) else: @@ -418,7 +419,6 @@ def _build_aggregation_query_spec(self, aggregator_options): aggregation_query.aggregation_query = self._get_aggregation_dsl( aggregator_options ) - return aggregation_query.spec def run(