Skip to content

Commit

Permalink
Bugfix: Wrong count number in timeline list (#1539)
Browse files Browse the repository at this point in the history
  • Loading branch information
@berggren
berggren authored Dec 17, 2020
1 parent 00208bf commit 859bac6
Show file tree
Hide file tree
Showing 12 changed files with 41 additions and 42 deletions.
6 changes: 3 additions & 3 deletions timesketch/api/v1/resources/event.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ class EventResource(resources.ResourceMixin, Resource):
event_id: The datastore event id as string
"""
def __init__(self):
super(EventResource, self).__init__()
super().__init__()
self.parser = reqparse.RequestParser()
self.parser.add_argument(
'searchindex_id', type=six.text_type, required=True)
Expand Down Expand Up @@ -660,7 +660,7 @@ def get(self, sketch_id):
t.searchindex.index_name for t in sketch.active_timelines
if t.get_status.status != 'archived'
]
count = self.datastore.count(indices)
meta = dict(count=count)
count, bytes_on_disk = self.datastore.count(indices)
meta = dict(count=count, bytes=bytes_on_disk)
schema = dict(meta=meta, objects=[])
return jsonify(schema)
30 changes: 3 additions & 27 deletions timesketch/api/v1/resources/sketch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,39 +326,15 @@ def get(self, sketch_id):
}

if sketch_indices:
try:
es_stats = self.datastore.client.indices.stats(
index=sketch_indices, metric='docs, store')
except elasticsearch.NotFoundError:
es_stats = {}
logger.error(
'Unable to find index in datastore', exc_info=True)

# Stats for index. Num docs per shard and size on disk.
for index_name, stats in es_stats.get('indices', {}).items():
doc_count_all_shards = stats.get(
'total', {}).get('docs', {}).get('count', 0)
bytes_on_disk = stats.get(
'total', {}).get('store', {}).get('size_in_bytes', 0)
num_shards = stats.get('_shards', {}).get('total', 1)
doc_count = int(doc_count_all_shards / num_shards)

for index_name in sketch_indices:
doc_count, bytes_on_disk = self.datastore.count(
indices=index_name)
stats_per_index[index_name] = {
'count': doc_count,
'bytes': bytes_on_disk
}

# Stats per data type in the index.
parameters = {
'limit': '100',
'field': 'data_type'
}
result_obj, _ = utils.run_aggregator(
sketch.id, aggregator_name='field_bucket',
aggregator_parameters=parameters,
index=[index_name])
stats_per_index[index_name]['data_types'] = result_obj.values

if not sketch_indices:
mappings_settings = {}
else:
Expand Down
2 changes: 1 addition & 1 deletion ...ontend/dist/css/chunk-common.f83adf06.css → ...ontend/dist/css/chunk-common.b4a070aa.css
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion timesketch/frontend/dist/index.html
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><title>Timesketch</title><link href=/dist/css/chunk-common.f83adf06.css rel=preload as=style><link href=/dist/css/chunk-vendors.4a602e4f.css rel=preload as=style><link href=/dist/js/chunk-common.ead7bca0.js rel=preload as=script><link href=/dist/js/chunk-vendors.f0ee5ca0.js rel=preload as=script><link href=/dist/js/index.0c45ccdf.js rel=preload as=script><link href=/dist/css/chunk-vendors.4a602e4f.css rel=stylesheet><link href=/dist/css/chunk-common.f83adf06.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.f0ee5ca0.js></script><script src=/dist/js/chunk-common.ead7bca0.js></script><script src=/dist/js/index.0c45ccdf.js></script></body></html>
<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><title>Timesketch</title><link href=/dist/css/chunk-common.b4a070aa.css rel=preload as=style><link href=/dist/css/chunk-vendors.4a602e4f.css rel=preload as=style><link href=/dist/js/chunk-common.6e554ecc.js rel=preload as=script><link href=/dist/js/chunk-vendors.f0ee5ca0.js rel=preload as=script><link href=/dist/js/index.0c45ccdf.js rel=preload as=script><link href=/dist/css/chunk-vendors.4a602e4f.css rel=stylesheet><link href=/dist/css/chunk-common.b4a070aa.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.f0ee5ca0.js></script><script src=/dist/js/chunk-common.6e554ecc.js></script><script src=/dist/js/index.0c45ccdf.js></script></body></html>
2 changes: 2 additions & 0 deletions timesketch/frontend/dist/js/chunk-common.6e554ecc.js
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions timesketch/frontend/dist/js/chunk-common.6e554ecc.js.map
View file

Large diffs are not rendered by default.

2 changes: 0 additions & 2 deletions timesketch/frontend/dist/js/chunk-common.ead7bca0.js
View file

This file was deleted.

1 change: 0 additions & 1 deletion timesketch/frontend/dist/js/chunk-common.ead7bca0.js.map
View file

This file was deleted.

2 changes: 1 addition & 1 deletion timesketch/frontend/dist/login.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@
.card-content {
justify-content: center;
align-items: center;
}</style><link href=/dist/css/chunk-common.f83adf06.css rel=preload as=style><link href=/dist/css/chunk-vendors.4a602e4f.css rel=preload as=style><link href=/dist/js/chunk-common.ead7bca0.js rel=preload as=script><link href=/dist/js/chunk-vendors.f0ee5ca0.js rel=preload as=script><link href=/dist/js/login.c1fa2383.js rel=preload as=script><link href=/dist/css/chunk-vendors.4a602e4f.css rel=stylesheet><link href=/dist/css/chunk-common.f83adf06.css rel=stylesheet></head><body class=login-page><div class=columns><div class=column><div class="card is-wide has-text-centered"><div class=card-content><div class=content><div><img src=/dist/timesketch-color.png style=width:40px;><div style="font-size: 1.5em;">time<b>sketch</b></div><div style="font-size: 0.8em;">Digital Forensic Timeline Analysis</div></div><br><form method=post style="width:50%;margin-left: auto; margin-right: auto;"><div class=field><div class=control><input type=text class=input name=username placeholder=Username style=text-align:center;></div></div><div class=field><div class=control><input type=password class=input name=password placeholder=Password style=text-align:center;></div></div><div class=field><div class="control has-text-centered"><button type=submit class="button is-rounded is-wide is-info" style=width:100%;>Sign in</button></div></div>{{ form.csrf_token }}</form></div></div></div></div></div><script src=/dist/js/chunk-vendors.f0ee5ca0.js></script><script src=/dist/js/chunk-common.ead7bca0.js></script><script src=/dist/js/login.c1fa2383.js></script></body></html>
}</style><link href=/dist/css/chunk-common.b4a070aa.css rel=preload as=style><link href=/dist/css/chunk-vendors.4a602e4f.css rel=preload as=style><link href=/dist/js/chunk-common.6e554ecc.js rel=preload as=script><link href=/dist/js/chunk-vendors.f0ee5ca0.js rel=preload as=script><link href=/dist/js/login.c1fa2383.js rel=preload as=script><link href=/dist/css/chunk-vendors.4a602e4f.css rel=stylesheet><link href=/dist/css/chunk-common.b4a070aa.css rel=stylesheet></head><body class=login-page><div class=columns><div class=column><div class="card is-wide has-text-centered"><div class=card-content><div class=content><div><img src=/dist/timesketch-color.png style=width:40px;><div style="font-size: 1.5em;">time<b>sketch</b></div><div style="font-size: 0.8em;">Digital Forensic Timeline Analysis</div></div><br><form method=post style="width:50%;margin-left: auto; margin-right: auto;"><div class=field><div class=control><input type=text class=input name=username placeholder=Username style=text-align:center;></div></div><div class=field><div class=control><input type=password class=input name=password placeholder=Password style=text-align:center;></div></div><div class=field><div class="control has-text-centered"><button type=submit class="button is-rounded is-wide is-info" style=width:100%;>Sign in</button></div></div>{{ form.csrf_token }}</form></div></div></div></div></div><script src=/dist/js/chunk-vendors.f0ee5ca0.js></script><script src=/dist/js/chunk-common.6e554ecc.js></script><script src=/dist/js/login.c1fa2383.js></script></body></html>
2 changes: 2 additions & 0 deletions timesketch/frontend/src/components/Sketch/TimelineListItem.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ limitations under the License.
</p>
<p v-if="timelineStatus === 'ready'" class="control">

<!-- Disabled 2020-12-17. Too expensive for large sketches. TODO: Refactor to do lazy loading instead.
<b-dropdown position="is-bottom-left" aria-role="menu" trap-focus append-to-body :scrollable="true" :max-height="300">
<button class="button is-outlined is-rounded is-small" slot="trigger">
<span class="icon is-small">
Expand All @@ -130,6 +131,7 @@ limitations under the License.
</div>
</b-dropdown-item>
</b-dropdown>
-->

<ts-analyzer-list-dropdown :timeline="timeline" @newAnalysisSession="setAnalysisSession($event)"></ts-analyzer-list-dropdown>
</p>
Expand Down
21 changes: 15 additions & 6 deletions timesketch/lib/datastores/elastic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -594,18 +594,27 @@ def count(self, indices):
indices: List of indices.
Returns:
Number of documents.
Tuple containing number of documents and size on disk.
"""
if not indices:
return 0
return 0, 0

try:
result = self.client.count(index=indices)
except (NotFoundError, RequestError):
es_stats = self.client.indices.stats(
index=indices, metric='docs, store')
except NotFoundError:
es_logger.error(
'Unable to count indexes (index not found)',
exc_info=True)
return 0
return result.get('count', 0)
es_stats = {}

doc_count_total = es_stats.get(
'_all', {}).get('primaries', {}).get('docs', {}).get('count', 0)
doc_bytes_total = es_stats.get(
'_all', {}).get(
'primaries', {}).get('store', {}).get('size_in_bytes', 0)

return doc_count_total, doc_bytes_total

def set_label(self, searchindex_id, event_id, event_type, sketch_id,
user_id, label, toggle=False, remove=False,
Expand Down
12 changes: 12 additions & 0 deletions timesketch/lib/testlib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,18 @@ def get_event(self, searchindex_id, event_id):
"""
return self.event_dict

@staticmethod
def count(indices):
"""Mock returning a single event from the datastore.
Args:
indices: List of indices.
Returns:
A tuple with count and bytes.
"""
return 1, 1

@staticmethod
def get_filter_labels(sketch_id, indices):
"""Mock returning a single event from the datastore.
Expand Down

0 comments on commit 859bac6

Please sign in to comment.