Merge branch 'develop'

.coveragerc

@@ -0,0 +1,2 @@
+[report]
+omit = */tests/*

.gitignore

@@ -57,4 +57,5 @@ docs/_build/
 target/
 
 
-.idea/
+.idea/
+venv/

.travis.yml

@@ -7,8 +7,11 @@ language: python
 python:
   - "3.5"
 # command to install dependencies
-install: "pip install -r requirements.txt"
+install: 
+  - "pip install -r requirements.txt"
+  - pip install coveralls
 # command to run tests
 script:
-    - nosetests -w ./tanner -vv
+    - nosetests -vv --with-coverage --cover-inclusive --cover-package=tanner
     - pycodestyle . --max-line-length=120
+after_success: coveralls

README.md

@@ -2,6 +2,8 @@ TANNER
 ======
 [![Documentation Status](https://readthedocs.org/projects/tanner/badge/?version=latest)](http://tanner.readthedocs.io/en/latest/?badge=latest)
 [![Build Status](https://travis-ci.org/mushorg/tanner.svg?branch=master)](https://travis-ci.org/mushorg/tanner)
+[![Coverage Status](https://coveralls.io/repos/github/mushorg/tanner/badge.svg?branch=master)](https://coveralls.io/github/mushorg/tanner?branch=master)
+[![Coverage Status](https://coveralls.io/repos/github/mushorg/tanner/badge.svg?branch=develop)](https://coveralls.io/github/mushorg/tanner?branch=develop)
 
 <b><i>He who flays the hide</b></i>
 

docker/phpox/dist/sandbox.py

@@ -29,6 +29,7 @@
 
 from pprint import pprint
 
+
 class PHPSandbox(object):
     @classmethod
     def php_tag_check(cls, script):

requirements.txt

@@ -1,13 +1,14 @@
-aiohttp==2.2
+aiohttp
 aiomysql
 aiohttp_jinja2==0.14.0
 docker<2.6
 elizabeth==0.3.27
-yarl<0.14
+yarl
 redis
-asyncio_redis
+aioredis
 uvloop
 pymongo
 pylibinjection
 jinja2
 pycodestyle
+geoip2

setup.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python
-from setuptools import find_packages
 from distutils.core import setup
+from setuptools import find_packages
 
 setup(name='Tanner',
-      version='0.5.0',
+      version='0.6.0',
       description='He who flays the hide',
       author='MushMush Foundation',
       author_email='[email protected]',
       url='https://github.com/mushorg/tanner',
       packages=find_packages(exclude=['*.pyc']),
       scripts=['bin/tanner', 'bin/tannerweb', 'bin/tannerapi'],
-      data_files=[('/opt/tanner/db/', ['tanner/data/db_config.json']),
-                  ('/opt/tanner/data/', ['tanner/data/dorks.pickle'])]
+      data_files=[('/opt/tanner/db/', ['tanner/data/db_config.json', 'tanner/data/GeoLite2-City.mmdb']),
+                  ('/opt/tanner/data/', ['tanner/data/dorks.pickle', 'tanner/data/crawler_user_agents.txt'])]
       )

tanner/__init__.py

@@ -1 +1 @@
-__version__ = '0.5.0'
+__version__ = '0.6.0'

tanner/api/api.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import operator
-import asyncio_redis
+import aioredis
 
 
 class Api:
@@ -12,9 +12,8 @@ def __init__(self, redis_client):
     async def return_snares(self):
         query_res = []
         try:
-            query_res = await self.redis_client.smembers('snare_ids')
-            query_res = await query_res.asset()
-        except asyncio_redis.NotConnectedError as connection_error:
+            query_res = await self.redis_client.smembers('snare_ids', encoding='utf-8')
+        except aioredis.ProtocolError as connection_error:
             self.logger.error('Can not connect to redis %s', connection_error)
         return list(query_res)
 
@@ -43,8 +42,10 @@ async def return_snare_stats(self, snare_uuid):
     async def return_snare_info(self, uuid, count=-1):
         query_res = []
         try:
-            query_res = await self.redis_client.lrange_aslist(uuid, 0, count)
-        except asyncio_redis.NotConnectedError as connection_error:
+            query_res = await self.redis_client.zrevrangebyscore(
+                uuid, offset=0, count=count, encoding='utf-8'
+            )
+        except aioredis.ProtocolError as connection_error:
             self.logger.error('Can not connect to redis %s', connection_error)
         else:
             if not query_res:

tanner/config.py

@@ -5,8 +5,12 @@
 
 LOGGER = logging.getLogger(__name__)
 
-config_template = {'DATA': {'db_config': '/opt/tanner/db/db_config.json', 'dorks': '/opt/tanner/data/dorks.pickle',
-                            'user_dorks': '/opt/tanner/data/user_dorks.pickle'},
+config_template = {'DATA': {'db_config': '/opt/tanner/db/db_config.json',
+                            'dorks': '/opt/tanner/data/dorks.pickle',
+                            'user_dorks': '/opt/tanner/data/user_dorks.pickle',
+                            'crawler_stats': '/opt/tanner/data/crawler_user_agents.txt',
+                            'geo_db': '/opt/tanner/db/GeoLite2-City.mmdb'
+                            },
                    'TANNER': {'host': '0.0.0.0', 'port': 8090},
                    'WEB': {'host': '0.0.0.0', 'port': 8091},
                    'API': {'host': '0.0.0.0', 'port': 8092},

tanner/data/crawler_user_agents.txt

@@ -0,0 +1,50 @@
+Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)
+Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
+Clickagy Intelligence Bot v2
+SirdataBot
+Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)
+A6-Indexer/1.0 (http://www.a6corp.com/a6-web-scraping-policy/)
+Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)
+Mozilla/5.0 (compatible; AhrefsBot/5.1; +http://ahrefs.com/robot/)
+Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)
+Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
+Screaming Frog SEO Spider/9.2
+Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)
+CCBot/2.0 (http://commoncrawl.org/faq/)
+Mozilla/5.0 (iPhone; CPU iPhone OS 60 like Mac OS X) AppleWebKit/536.26 (KHTML like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
+YisouSpider
+Googlebot/2.1 (+http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; Cliqzbot/1.0; +http://cliqz.com/company/cliqzbot)
+Googlebot/2.1 (+http://www.googlebot.com/bot.html)
+Mozilla/5.0 (compatible; 008/0.85; http://www.80legs.com/webcrawler.html) Gecko/2008032620
+Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
+Googlebot-Video/1.0
+SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; Googlebot/2.1;+http://www.google.com/bot.html)
+Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.tropicdesigns.net)
+Mozilla/5.0 (iPhone; U; CPU iPhone OS 41 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)
+DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
+Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
+Mozilla/5.0 (iPhone; CPU iPhone OS 60 like Mac OS X) AppleWebKit/536.26 (KHTML like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)
+Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; DomainAppender /1.0; +http://www.profound.net/domainappender)
+msnbot/2.0b (+http://search.msn.com/msnbot.htm)
+msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)
+Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider
+Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
+Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)
+Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
+Mozilla/5.0 (iPhone; U; CPU iPhone OS 41 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider
+Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)
+Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)
+Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)
+Mozilla/5.0 (iPhone; CPU iPhone OS 83 like Mac OS X) AppleWebKit/600.1.4 (KHTML like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php+)
+DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)
+ia_archiver (+http://www.alexa.com/site/help/webmasters; [email protected])

tanner/dorks_manager.py

@@ -6,7 +6,7 @@
 import re
 import uuid
 
-import asyncio_redis
+import aioredis
 
 from tanner import config
 from tanner.utils import patterns
@@ -31,25 +31,25 @@ async def push_init_dorks(file_name, redis_key, redis_client):
                 dorks = dorks.split()
             if isinstance(dorks, set):
                 dorks = [x for x in dorks if x is not None]
-            await redis_client.sadd(redis_key, dorks)
+            await redis_client.sadd(redis_key, *dorks)
 
     async def extract_path(self, path, redis_client):
         extracted = re.match(patterns.QUERY, path)
         if extracted:
             extracted = extracted.group(0)
             try:
-                await redis_client.sadd(self.user_dorks_key, [extracted])
-            except asyncio_redis.NotConnectedError as connection_error:
+                await redis_client.sadd(self.user_dorks_key, *[extracted])
+            except aioredis.ProtocolError as connection_error:
                 self.logger.error('Problem with redis connection: %s', connection_error)
 
     async def init_dorks(self, redis_client):
         try:
-            transaction = await redis_client.multi()
-            dorks_exist = await transaction.exists(self.dorks_key)
-            user_dorks_exist = await transaction.exists(self.user_dorks_key)
+            transaction = redis_client.multi_exec()
+            dorks_exist = transaction.exists(self.dorks_key)
+            user_dorks_exist = transaction.exists(self.user_dorks_key)
 
-            await transaction.exec()
-        except (asyncio_redis.TransactionError, asyncio_redis.NotConnectedError) as redis_error:
+            await transaction.execute()
+        except (aioredis.MultiExecError, aioredis.ProtocolError) as redis_error:
             self.logger.error('Problem with transaction: %s', redis_error)
         else:
             dorks_existed = await dorks_exist
@@ -69,12 +69,12 @@ async def choose_dorks(self, redis_client):
         chosen_dorks = []
         max_dorks = 50
         try:
-            transaction = await redis_client.multi()
-            dorks = await transaction.smembers_asset(self.dorks_key)
-            user_dorks = await transaction.smembers_asset(self.user_dorks_key)
+            transaction = redis_client.multi_exec()
+            dorks = transaction.smembers(self.dorks_key, encoding='utf-8')
+            user_dorks = transaction.smembers(self.user_dorks_key, encoding='utf-8')
 
-            await transaction.exec()
-        except (asyncio_redis.TransactionError, asyncio_redis.NotConnectedError) as redis_error:
+            await transaction.execute()
+        except (aioredis.MultiExecError, aioredis.ProtocolError) as redis_error:
             self.logger.error('Problem with transaction: %s', redis_error)
         else:
             dorks = await dorks

tanner/redis_client.py

@@ -1,7 +1,7 @@
 import asyncio
 import logging
 
-import asyncio_redis
+import aioredis
 
 from tanner.config import TannerConfig
 
@@ -18,8 +18,8 @@ async def get_redis_client(poolsize=None):
             if poolsize is None:
                 poolsize = TannerConfig.get('REDIS', 'poolsize')
             timeout = TannerConfig.get('REDIS', 'timeout')
-            redis_client = await asyncio.wait_for(asyncio_redis.Pool.create(
-                host=host, port=int(port), poolsize=int(poolsize)), timeout=int(timeout))
+            redis_client = await asyncio.wait_for(aioredis.create_redis_pool(
+                (host, int(port)), maxsize=int(poolsize)), timeout=int(timeout))
         except asyncio.TimeoutError as timeout_error:
             LOGGER.error('Problem with redis connection. Please, check your redis server. %s', timeout_error)
             exit()

tanner/server.py

@@ -6,7 +6,7 @@
 import yarl
 from aiohttp import web
 
-from tanner import api, dorks_manager, session_manager, redis_client
+from tanner import dorks_manager, session_manager, redis_client
 from tanner.config import TannerConfig
 from tanner.emulators import base
 from tanner.reporting.log_local import Reporting as local_report
@@ -51,7 +51,7 @@ async def handle_event(self, request):
         data = await request.read()
         try:
             data = json.loads(data.decode('utf-8'))
-            path = yarl.unquote(data['path'])
+            path = yarl.URL(data['path']).human_repr()
         except (TypeError, ValueError, KeyError) as error:
             self.logger.error('error parsing request: %s', data)
             response_msg = self._make_response(msg=type(error).__name__)

tanner/session.py

@@ -1,6 +1,7 @@
 import json
 import time
 import uuid
+from urllib.parse import urlparse
 
 from tanner.config import TannerConfig
 from tanner.utils import docker_helper
@@ -19,6 +20,10 @@ def __init__(self, data):
             self.snare_uuid = data['uuid']
             self.paths = [{'path': data['path'], 'timestamp': time.time(),
                            'response_status': data['status']}]
+            self.referer = None
+            if 'referer' in data['headers']:
+                ref = urlparse(data['headers']['referer'])
+                self.referer = ref.path
             self.cookies = data['cookies']
             self.associated_db = None
             self.associated_env = None
@@ -52,7 +57,8 @@ def to_json(self):
                     end_time=self.timestamp,
                     count=self.count,
                     paths=self.paths,
-                    cookies=self.cookies
+                    cookies=self.cookies,
+                    referer=self.referer
                     )
         return json.dumps(sess)