From 420bec749fa92a06cf98d6932bca84bbd6700d49 Mon Sep 17 00:00:00 2001 From: Johan Berggren Date: Wed, 23 Sep 2015 14:27:55 +0200 Subject: [PATCH 1/7] Fix bug #111 --- timesketch.conf | 4 ++++ timesketch/lib/tasks.py | 27 +++++++++++++++++++++++++++ timesketch/lib/tasks_test.py | 29 +++++++++++++++++++++++++++++ wsgi.py | 2 +- 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 timesketch/lib/tasks_test.py diff --git a/timesketch.conf b/timesketch.conf index a5d9072a37..a7932ba31f 100644 --- a/timesketch.conf +++ b/timesketch.conf @@ -61,3 +61,7 @@ UPLOAD_FOLDER = u'/tmp' # server is running. CELERY_BROKER_URL='redis://ip:port', CELERY_RESULT_BACKEND='redis://ip:port' + +# Path to Plaso WinEvtx message string database. +# If not set, defaults to system prefix + share/plaso +#WINEVT_DB = u'/path/to/dir/with/plaso/data/files' diff --git a/timesketch/lib/tasks.py b/timesketch/lib/tasks.py index d8f079aefc..63e80462d0 100644 --- a/timesketch/lib/tasks.py +++ b/timesketch/lib/tasks.py @@ -13,13 +13,36 @@ # limitations under the License. """Celery task for processing Plaso storage files.""" +import os +import sys + from plaso.frontend import psort from timesketch import create_celery_app +from flask import current_app celery = create_celery_app() +def get_data_location(data_location=None): + """Get the location (path) to the winevt-rc.db file. + + Args: + data_location: Path to the directory where the database file is located. + If this is None we will use sys.prefix + share/plaso as + default. + + Returns: + The path to where the database file is located or None if not existing. + """ + + if not data_location: + data_location = os.path.join(sys.prefix, u'share', u'plaso') + if not os.path.exists(data_location): + data_location = None + return data_location + + @celery.task(track_started=True) def run_plaso(source_file_path, timeline_name, index_name): """Create a Celery task for processing Plaso storage file. @@ -32,11 +55,15 @@ def run_plaso(source_file_path, timeline_name, index_name): Returns: Dictionary with count of processed events. """ + # Try to read the winevt-rc database path from the config file. + data_location = get_data_location( + data_location=current_app.config.get(u'WINEVT_DB', None)) analysis_plugins = None flush_interval_ms = 1000 # Use the Psort frontend for processing. frontend = psort.PsortFrontend() + frontend.SetDataLocation(data_location) storage_file = frontend.OpenStorage( source_file_path, read_only=True) diff --git a/timesketch/lib/tasks_test.py b/timesketch/lib/tasks_test.py new file mode 100644 index 0000000000..2cd4e3968a --- /dev/null +++ b/timesketch/lib/tasks_test.py @@ -0,0 +1,29 @@ +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for tasks.""" + +from timesketch.lib.testlib import BaseTest +from timesketch.lib.tasks import get_data_location + + +class TestTasks(BaseTest): + """Tests for the functionality on the tasks module.""" + def test_get_data_location(self): + """Test to get data_location path.""" + data_location_none = get_data_location( + data_location=u'/tmp/non_existing') + data_location_exists = get_data_location( + data_location=u'/tmp') + self.assertFalse(data_location_none) + self.assertEqual(u'/tmp', data_location_exists) diff --git a/wsgi.py b/wsgi.py index 9b7fe31685..e6ab999abb 100644 --- a/wsgi.py +++ b/wsgi.py @@ -15,7 +15,7 @@ """This module is for creating the app for a WSGI server. Example with Gunicorn: -$ gunicorn -b 127.0.0.1:4000 --log-file - wsgi:application +$ gunicorn -b 127.0.0.1:4000 --log-file --timeout 120 - wsgi:application Example configuration for Apache with mod_wsgi (a2enmod mod_wsgi): From 07cbd5e0dfd2fe48c5bf6bb9e750c6ca47542e08 Mon Sep 17 00:00:00 2001 From: Johan Berggren Date: Wed, 23 Sep 2015 14:34:49 +0200 Subject: [PATCH 2/7] Fix import order --- timesketch/lib/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timesketch/lib/tasks.py b/timesketch/lib/tasks.py index 63e80462d0..4ccdfda00f 100644 --- a/timesketch/lib/tasks.py +++ b/timesketch/lib/tasks.py @@ -16,10 +16,10 @@ import os import sys +from flask import current_app from plaso.frontend import psort from timesketch import create_celery_app -from flask import current_app celery = create_celery_app() From 25cdcb5d546a2d3be3ce0305827cdc2b0cd9dfda Mon Sep 17 00:00:00 2001 From: Johan Berggren Date: Wed, 23 Sep 2015 14:44:56 +0200 Subject: [PATCH 3/7] Travis need Plaso --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index f0e54972e5..07af1a1b58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,5 +5,9 @@ python: install: - "pip install ." - "pip install Flask-Testing nose mock pylint coverage" + - "sudo add-apt-repository universe" + - "sudo add-apt-repository ppa:gift/stable" + - "sudo apt-get update" + - "sudo apt-get install python-plaso" # command to run tests script: nosetests From 476b95d1cc96fa266f104d78e516d158b2c2d7b0 Mon Sep 17 00:00:00 2001 From: Johan Berggren Date: Wed, 23 Sep 2015 14:57:45 +0200 Subject: [PATCH 4/7] Travis need Plaso --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 07af1a1b58..1578d01925 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,11 @@ language: python python: - "2.7" +before_install: + - if test `uname -s` = 'Linux'; then sudo add-apt-repository ppa:gift/stable -y && sudo apt-get update -q && sudo apt-get install python-plaso; fi # command to install dependencies install: - "pip install ." - "pip install Flask-Testing nose mock pylint coverage" - - "sudo add-apt-repository universe" - - "sudo add-apt-repository ppa:gift/stable" - - "sudo apt-get update" - - "sudo apt-get install python-plaso" # command to run tests script: nosetests From 069fe76afd7588f649e02e224d391bd947b818ba Mon Sep 17 00:00:00 2001 From: Johan Berggren Date: Wed, 23 Sep 2015 15:01:58 +0200 Subject: [PATCH 5/7] Travis need Plaso --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1578d01925..6e13bef77d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: python python: - "2.7" before_install: - - if test `uname -s` = 'Linux'; then sudo add-apt-repository ppa:gift/stable -y && sudo apt-get update -q && sudo apt-get install python-plaso; fi + - if test `uname -s` = 'Linux'; then sudo add-apt-repository ppa:gift/stable -y && sudo add-apt-repository universe -y && sudo apt-get update -q && sudo apt-get install python-plaso; fi # command to install dependencies install: - "pip install ." From af79ac4a0ad2b538b17baf3e0b89a4494ff07522 Mon Sep 17 00:00:00 2001 From: Johan Berggren Date: Wed, 23 Sep 2015 15:06:52 +0200 Subject: [PATCH 6/7] Travis dont have Plaso --- .travis.yml | 2 -- timesketch/lib/tasks.py | 8 +++++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e13bef77d..f0e54972e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ language: python python: - "2.7" -before_install: - - if test `uname -s` = 'Linux'; then sudo add-apt-repository ppa:gift/stable -y && sudo add-apt-repository universe -y && sudo apt-get update -q && sudo apt-get install python-plaso; fi # command to install dependencies install: - "pip install ." diff --git a/timesketch/lib/tasks.py b/timesketch/lib/tasks.py index 4ccdfda00f..b0ee519302 100644 --- a/timesketch/lib/tasks.py +++ b/timesketch/lib/tasks.py @@ -17,7 +17,13 @@ import sys from flask import current_app -from plaso.frontend import psort +# We currently don't have plaso in our Travis setup. This is a workaround +# for that until we fix the Travis environment. +# TODO: Add Plaso to our Travis environment we are running our tests in. +try: + from plaso.frontend import psort +except ImportError: + pass from timesketch import create_celery_app From f5ca294f04ab1a5a7555587d8218e4d1dfaa7824 Mon Sep 17 00:00:00 2001 From: Johan Berggren Date: Wed, 23 Sep 2015 17:47:21 +0200 Subject: [PATCH 7/7] Plaso data location --- timesketch.conf | 4 ++-- timesketch/lib/tasks.py | 19 ++++++------------- timesketch/lib/tasks_test.py | 9 +++++---- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/timesketch.conf b/timesketch.conf index a7932ba31f..cdfe0718ee 100644 --- a/timesketch.conf +++ b/timesketch.conf @@ -62,6 +62,6 @@ UPLOAD_FOLDER = u'/tmp' CELERY_BROKER_URL='redis://ip:port', CELERY_RESULT_BACKEND='redis://ip:port' -# Path to Plaso WinEvtx message string database. +# Path to plaso data directory. # If not set, defaults to system prefix + share/plaso -#WINEVT_DB = u'/path/to/dir/with/plaso/data/files' +#PLASO_DATA_LOCATION = u'/path/to/dir/with/plaso/data/files' diff --git a/timesketch/lib/tasks.py b/timesketch/lib/tasks.py index b0ee519302..28034cfff4 100644 --- a/timesketch/lib/tasks.py +++ b/timesketch/lib/tasks.py @@ -30,18 +30,13 @@ celery = create_celery_app() -def get_data_location(data_location=None): - """Get the location (path) to the winevt-rc.db file. - - Args: - data_location: Path to the directory where the database file is located. - If this is None we will use sys.prefix + share/plaso as - default. +def get_data_location(): + """Path to the plaso data directory. Returns: - The path to where the database file is located or None if not existing. + The path to where the plaso data directory is or None if not existing. """ - + data_location = current_app.config.get(u'PLASO_DATA_LOCATION', None) if not data_location: data_location = os.path.join(sys.prefix, u'share', u'plaso') if not os.path.exists(data_location): @@ -61,15 +56,13 @@ def run_plaso(source_file_path, timeline_name, index_name): Returns: Dictionary with count of processed events. """ - # Try to read the winevt-rc database path from the config file. - data_location = get_data_location( - data_location=current_app.config.get(u'WINEVT_DB', None)) + plaso_data_location = get_data_location() analysis_plugins = None flush_interval_ms = 1000 # Use the Psort frontend for processing. frontend = psort.PsortFrontend() - frontend.SetDataLocation(data_location) + frontend.SetDataLocation(plaso_data_location) storage_file = frontend.OpenStorage( source_file_path, read_only=True) diff --git a/timesketch/lib/tasks_test.py b/timesketch/lib/tasks_test.py index 2cd4e3968a..b8ca662b74 100644 --- a/timesketch/lib/tasks_test.py +++ b/timesketch/lib/tasks_test.py @@ -13,6 +13,8 @@ # limitations under the License. """Tests for tasks.""" +from flask import current_app + from timesketch.lib.testlib import BaseTest from timesketch.lib.tasks import get_data_location @@ -21,9 +23,8 @@ class TestTasks(BaseTest): """Tests for the functionality on the tasks module.""" def test_get_data_location(self): """Test to get data_location path.""" - data_location_none = get_data_location( - data_location=u'/tmp/non_existing') - data_location_exists = get_data_location( - data_location=u'/tmp') + data_location_none = get_data_location() + current_app.config[u'PLASO_DATA_LOCATION'] = u'/tmp' + data_location_exists = get_data_location() self.assertFalse(data_location_none) self.assertEqual(u'/tmp', data_location_exists)