Stats on indices (#1129)

* Add search hit count per index * Stats per index * Aggregation * Use ES client * Fix tests * More resiliant error handling Co-authored-by: Kristinn <[email protected]>
google · Mar 10, 2020 · cca6751 · cca6751
1 parent bf05853
commit cca6751
Show file tree

Hide file tree

Showing 14 changed files with 81 additions and 11 deletions.
diff --git a/timesketch/api/v1/resources.py b/timesketch/api/v1/resources.py
@@ -410,6 +410,17 @@ def get(self, sketch_id):
             for t in sketch.active_timelines
         ]
 
+        # Get event count and size on disk for each index in the sketch.
+        stats_per_index = {}
+        es_stats = self.datastore.client.indices.stats(
+            index=sketch_indices, metric='docs, store')
+        for index_name, stats in es_stats.get('indices', {}).items():
+            stats_per_index[index_name] = {
+                'count': stats.get('total', {}).get('docs', {}).get('count', 0),
+                'bytes': stats.get(
+                    'total', {}).get('store', {}).get('size_in_bytes', 0)
+            }
+
         if not sketch_indices:
             mappings_settings = {}
         else:
@@ -467,7 +478,8 @@ def get(self, sketch_id):
             analyzers=[
                 x for x, y in analyzer_manager.AnalysisManager.get_analyzers()
             ],
-            mappings=list(mappings)
+            mappings=list(mappings),
+            stats=stats_per_index
         )
         return self.to_json(sketch, meta=meta)
 
@@ -815,6 +827,15 @@ def post(self, sketch_id):
                 HTTP_STATUS_CODE_BAD_REQUEST,
                 'The request needs a query string/DSL and or a star filter.')
 
+        # Aggregate hit count per index.
+        index_stats_agg = {
+            "indices": {
+                "terms": {
+                    "field": "_index"
+                }
+            }
+        }
+
         if scroll_id:
             # pylint: disable=unexpected-keyword-arg
             result = self.datastore.client.scroll(
@@ -826,10 +847,20 @@ def post(self, sketch_id):
                 query_filter,
                 query_dsl,
                 indices,
-                aggregations=None,
+                aggregations=index_stats_agg,
                 return_fields=return_fields,
                 enable_scroll=enable_scroll)
 
+        # Get number of matching documents per index.
+        count_per_index = {}
+        try:
+            for bucket in result['aggregations']['indices']['buckets']:
+                key = bucket.get('key')
+                if key:
+                    count_per_index[key] = bucket.get('doc_count')
+        except KeyError:
+            pass
+
         # Get labels for each event that matches the sketch.
         # Remove all other labels.
         for event in result['hits']['hits']:
@@ -868,6 +899,7 @@ def post(self, sketch_id):
             'es_total_count': result['hits']['total'],
             'timeline_colors': tl_colors,
             'timeline_names': tl_names,
+            'count_per_index': count_per_index,
             'scroll_id': result.get('_scroll_id', ''),
         }
 

diff --git a/timesketch/api/v1/resources_test.py b/timesketch/api/v1/resources_test.py
@@ -183,6 +183,7 @@ class ExploreResourceTest(BaseTest):
             },
             'es_total_count': 1,
             'es_time': 5,
+            'count_per_index': {},
             'scroll_id': ''
         },
         'objects': [{

diff --git a/...ontend/dist/css/chunk-common.bcebcc29.css → ...ontend/dist/css/chunk-common.d99b6eab.css b/...ontend/dist/css/chunk-common.bcebcc29.css → ...ontend/dist/css/chunk-common.d99b6eab.css
diff --git a/timesketch/frontend/dist/index.html b/timesketch/frontend/dist/index.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><title>Timesketch</title><link href=/dist/css/chunk-common.bcebcc29.css rel=preload as=style><link href=/dist/css/chunk-vendors.7ec8b058.css rel=preload as=style><link href=/dist/js/chunk-common.ea56227b.js rel=preload as=script><link href=/dist/js/chunk-vendors.8813ff27.js rel=preload as=script><link href=/dist/js/index.0c45ccdf.js rel=preload as=script><link href=/dist/css/chunk-vendors.7ec8b058.css rel=stylesheet><link href=/dist/css/chunk-common.bcebcc29.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.8813ff27.js></script><script src=/dist/js/chunk-common.ea56227b.js></script><script src=/dist/js/index.0c45ccdf.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><title>Timesketch</title><link href=/dist/css/chunk-common.d99b6eab.css rel=preload as=style><link href=/dist/css/chunk-vendors.7ec8b058.css rel=preload as=style><link href=/dist/js/chunk-common.f5d9bf3b.js rel=preload as=script><link href=/dist/js/chunk-vendors.8813ff27.js rel=preload as=script><link href=/dist/js/index.0c45ccdf.js rel=preload as=script><link href=/dist/css/chunk-vendors.7ec8b058.css rel=stylesheet><link href=/dist/css/chunk-common.d99b6eab.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.8813ff27.js></script><script src=/dist/js/chunk-common.f5d9bf3b.js></script><script src=/dist/js/index.0c45ccdf.js></script></body></html>
diff --git a/timesketch/frontend/dist/js/chunk-common.ea56227b.js b/timesketch/frontend/dist/js/chunk-common.ea56227b.js
diff --git a/timesketch/frontend/dist/js/chunk-common.ea56227b.js.map b/timesketch/frontend/dist/js/chunk-common.ea56227b.js.map
diff --git a/timesketch/frontend/dist/js/chunk-common.f5d9bf3b.js b/timesketch/frontend/dist/js/chunk-common.f5d9bf3b.js
diff --git a/timesketch/frontend/dist/js/chunk-common.f5d9bf3b.js.map b/timesketch/frontend/dist/js/chunk-common.f5d9bf3b.js.map
diff --git a/timesketch/frontend/dist/login.html b/timesketch/frontend/dist/login.html
@@ -15,4 +15,4 @@
         .card-content {
             justify-content: center;
             align-items: center;
-        }</style><link href=/dist/css/chunk-common.bcebcc29.css rel=preload as=style><link href=/dist/css/chunk-vendors.7ec8b058.css rel=preload as=style><link href=/dist/js/chunk-common.ea56227b.js rel=preload as=script><link href=/dist/js/chunk-vendors.8813ff27.js rel=preload as=script><link href=/dist/js/login.c1fa2383.js rel=preload as=script><link href=/dist/css/chunk-vendors.7ec8b058.css rel=stylesheet><link href=/dist/css/chunk-common.bcebcc29.css rel=stylesheet></head><body class=login-page><div class=columns><div class=column><div class="card is-wide has-text-centered"><div class=card-content><div class=content><div><img src=/dist/timesketch-color.png style=width:40px;><div style="color: #333; font-size: 1.5em;">time<b>sketch</b></div><div style="color: #666; font-size: 0.8em;">Digital Forensic Timeline Analysis</div></div><br><form method=post style="width:50%;margin-left: auto; margin-right: auto;"><div class=field><div class=control><input type=text class=input name=username placeholder=Username style=text-align:center;></div></div><div class=field><div class=control><input type=password class=input name=password placeholder=Password style=text-align:center;></div></div><div class=field><div class="control has-text-centered"><button type=submit class="button is-rounded is-wide is-info" style=width:100%;>Sign in</button></div></div>{{ form.csrf_token }}</form></div></div></div></div></div><script src=/dist/js/chunk-vendors.8813ff27.js></script><script src=/dist/js/chunk-common.ea56227b.js></script><script src=/dist/js/login.c1fa2383.js></script></body></html>
+        }</style><link href=/dist/css/chunk-common.d99b6eab.css rel=preload as=style><link href=/dist/css/chunk-vendors.7ec8b058.css rel=preload as=style><link href=/dist/js/chunk-common.f5d9bf3b.js rel=preload as=script><link href=/dist/js/chunk-vendors.8813ff27.js rel=preload as=script><link href=/dist/js/login.c1fa2383.js rel=preload as=script><link href=/dist/css/chunk-vendors.7ec8b058.css rel=stylesheet><link href=/dist/css/chunk-common.d99b6eab.css rel=stylesheet></head><body class=login-page><div class=columns><div class=column><div class="card is-wide has-text-centered"><div class=card-content><div class=content><div><img src=/dist/timesketch-color.png style=width:40px;><div style="color: #333; font-size: 1.5em;">time<b>sketch</b></div><div style="color: #666; font-size: 0.8em;">Digital Forensic Timeline Analysis</div></div><br><form method=post style="width:50%;margin-left: auto; margin-right: auto;"><div class=field><div class=control><input type=text class=input name=username placeholder=Username style=text-align:center;></div></div><div class=field><div class=control><input type=password class=input name=password placeholder=Password style=text-align:center;></div></div><div class=field><div class="control has-text-centered"><button type=submit class="button is-rounded is-wide is-info" style=width:100%;>Sign in</button></div></div>{{ form.csrf_token }}</form></div></div></div></div></div><script src=/dist/js/chunk-vendors.8813ff27.js></script><script src=/dist/js/chunk-common.f5d9bf3b.js></script><script src=/dist/js/login.c1fa2383.js></script></body></html>
diff --git a/timesketch/frontend/src/components/Sketch/TimelineListItem.vue b/timesketch/frontend/src/components/Sketch/TimelineListItem.vue
@@ -29,6 +29,8 @@ limitations under the License.
             <div class="content">
               <ul>
                 <li>Elasticsearch index: {{ timeline.searchindex.index_name }}</li>
+                <li>Number of events: {{ meta.stats[timeline.searchindex.index_name]['count'] | compactNumber }} ({{ meta.stats[timeline.searchindex.index_name]['count']}})</li>
+                <li>Size on disk: {{ meta.stats[timeline.searchindex.index_name]['bytes'] | compactBytes }} ({{ meta.stats[timeline.searchindex.index_name]['bytes']}})</li>
                 <li>Original name: {{ timeline.searchindex.name }}</li>
                 <li>Added by: {{ timeline.searchindex.user.username }}</li>
                 <li>Added: {{ timeline.searchindex.created_at | moment("YYYY-MM-DD HH:mm") }}</li>
@@ -131,6 +133,7 @@ limitations under the License.
 
     <span v-if="timelineStatus === 'ready'" class="is-size-7">
       Added {{ timeline.updated_at | moment("YYYY-MM-DD HH:mm") }}
+      <span class="tag is-small" :title="meta.stats[timeline.searchindex.index_name]['count'] + ' events in index'">{{ meta.stats[timeline.searchindex.index_name]['count'] | compactNumber }}</span>
     </span>
     <span v-else-if="timelineStatus === 'fail'" class="is-size-7">
       ERROR: <span v-on:click="showInfoModal =! showInfoModal" style="cursor:pointer;text-decoration: underline">Click here for details</span>

diff --git a/timesketch/frontend/src/components/Sketch/TimelinePicker.vue b/timesketch/frontend/src/components/Sketch/TimelinePicker.vue
@@ -16,7 +16,7 @@ limitations under the License.
 <template>
   <div>
     <span v-for="timeline in sketch.active_timelines" :key="timeline.id" class="tag is-medium" style="cursor: pointer; margin-right: 7px;margin-bottom:7px;" v-bind:style="timelineColor(timeline)" v-on:click="toggleIndex(timeline.searchindex.index_name)">
-      {{ timeline.name }}
+      {{ timeline.name }} <span v-if="indexIsEnabled(timeline.searchindex.index_name)" class="tag is-small" style="margin-left:10px;margin-right:-7px;background-color: rgba(255,255,255,0.5);">{{ countPerIndex[timeline.searchindex.index_name] || '0' }}</span>
     </span>
     <div v-if="sketch.active_timelines.length > 3" style="margin-top:7px;">
       <span style="text-decoration: underline; cursor: pointer; margin-right: 10px;" v-on:click="enableAllIndices">Enable all</span>
@@ -27,7 +27,7 @@ limitations under the License.
 
 <script>
 export default {
-  props: ['currentQueryFilter'],
+  props: ['currentQueryFilter', 'countPerIndex'],
   computed: {
     sketch () {
       return this.$store.state.sketch
@@ -70,6 +70,9 @@ export default {
     disableAllIndices: function () {
       this.currentQueryFilter.indices = []
       this.$emit('updateQueryFilter', this.currentQueryFilter)
+    },
+    indexIsEnabled: function (index) {
+      return this.currentQueryFilter.indices.includes(index)
     }
   },
   created: function () {

diff --git a/timesketch/frontend/src/filters/CompactBytes.js b/timesketch/frontend/src/filters/CompactBytes.js
@@ -0,0 +1,28 @@
+/*
+Copyright 2020 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+export default {
+  name: 'compactBytes',
+  filter: function (input) {
+    // Based on https://gist.github.com/james2doyle/4aba55c22f084800c199
+    if (!input) {
+      input = 0
+    }
+    let units = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+    let exponent = Math.min(Math.floor(Math.log(input) / Math.log(1000)), units.length - 1)
+    let num = (input / Math.pow(1000, exponent)).toFixed(2) * 1
+    return num + units[exponent]
+  }
+}
diff --git a/timesketch/frontend/src/views/SketchExplore.vue b/timesketch/frontend/src/views/SketchExplore.vue
@@ -123,7 +123,7 @@ limitations under the License.
               </span>
             </div>
 
-            <ts-explore-timeline-picker v-if="sketch.active_timelines" @updateQueryFilter="updateQueryFilter($event)" :current-query-filter="currentQueryFilter"></ts-explore-timeline-picker>
+            <ts-explore-timeline-picker v-if="sketch.active_timelines" @updateQueryFilter="updateQueryFilter($event)" :current-query-filter="currentQueryFilter" :count-per-index="eventList.meta.count_per_index"></ts-explore-timeline-picker>
           </div>
         </div>
       </div>

diff --git a/timesketch/lib/testlib.py b/timesketch/lib/testlib.py
@@ -87,6 +87,9 @@ def get_mapping(self, *args, **kwargs):
         """Mock get mapping call."""
         return {}
 
+    def stats(self, *args, **kwargs):
+        return {'indices': {}}
+
 
 class MockDataStore(object):
     """A mock implementation of a Datastore."""
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><title>Timesketch</title><link href=/dist/css/chunk-common.bcebcc29.css rel=preload as=style><link href=/dist/css/chunk-vendors.7ec8b058.css rel=preload as=style><link href=/dist/js/chunk-common.ea56227b.js rel=preload as=script><link href=/dist/js/chunk-vendors.8813ff27.js rel=preload as=script><link href=/dist/js/index.0c45ccdf.js rel=preload as=script><link href=/dist/css/chunk-vendors.7ec8b058.css rel=stylesheet><link href=/dist/css/chunk-common.bcebcc29.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.8813ff27.js></script><script src=/dist/js/chunk-common.ea56227b.js></script><script src=/dist/js/index.0c45ccdf.js></script></body></html>
		<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><title>Timesketch</title><link href=/dist/css/chunk-common.d99b6eab.css rel=preload as=style><link href=/dist/css/chunk-vendors.7ec8b058.css rel=preload as=style><link href=/dist/js/chunk-common.f5d9bf3b.js rel=preload as=script><link href=/dist/js/chunk-vendors.8813ff27.js rel=preload as=script><link href=/dist/js/index.0c45ccdf.js rel=preload as=script><link href=/dist/css/chunk-vendors.7ec8b058.css rel=stylesheet><link href=/dist/css/chunk-common.d99b6eab.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.8813ff27.js></script><script src=/dist/js/chunk-common.f5d9bf3b.js></script><script src=/dist/js/index.0c45ccdf.js></script></body></html>
-Original file line number
+Diff line change
@@ Expand Up / @@ -123,7 +123,7 @@ limitations under the License. @@
                   </span>
                 </div>
-                <ts-explore-timeline-picker v-if="sketch.active_timelines" @updateQueryFilter="updateQueryFilter($event)" :current-query-filter="currentQueryFilter"></ts-explore-timeline-picker>
+                <ts-explore-timeline-picker v-if="sketch.active_timelines" @updateQueryFilter="updateQueryFilter($event)" :current-query-filter="currentQueryFilter" :count-per-index="eventList.meta.count_per_index"></ts-explore-timeline-picker>
               </div>
             </div>
           </div>
@@ Expand Down @@