Updated changelog, tweaked a few formats and other values

.travis.yml

@@ -24,7 +24,6 @@ install:
 - pip install pytest
 - pip install -e .
 script:
-- wget https://raw.githubusercontent.com/kevthehermit/PasteHunter/master/settings.json.sample -O ~/.config/pastehunter.json
 - pastehunter-cli
 - python -m pytest
 after_success:

CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.0] - 2020-11-22
+## Changed
+- Added some error state checks and retry logic to pastebin scraping (#116)
+- Refactored paste inputs to use a base class
+
+## Added
+- Support for ix.io (#95)
+- Additional unit tests (pytest still has some issues with import paths on travis)
+
+
 ## [1.3.2] - 2020-02-15
 ### Changed
 Minor patch fixing error in email yara regexp

pastehunter-cli

@@ -19,7 +19,7 @@ import yara
 import pastehunter
 from pastehunter.common import parse_config
 
-VERSION = 1.0
+VERSION = '1.4.0'
 
 # Decided not to make this configurable as it currently really only applies to pastebin but may change in functionality later.
 # If someone would like this as a config key, please feel free to open an issue or a PR :)
@@ -38,7 +38,7 @@ logger = logging.getLogger('pastehunter')
 logger.setLevel(logging.INFO)
 
 # Version info
-logger.info("Starting PasteHunter Version: {0}".format(VERSION))
+logger.info("Starting PasteHunter Version: {}".format(VERSION))
 
 # Parse the config file
 logger.info("Reading Configs")
@@ -48,9 +48,11 @@ conf = parse_config()
 if not conf:
     sys.exit()
 
+
 class TimeoutError(Exception):
     pass
 
+
 class timeout:
     def __init__(self, seconds=1, error_message='Timeout'):
         self.seconds = seconds
@@ -72,17 +74,19 @@ if "log" in conf and conf["log"]["log_to_file"]:
     if conf["log"]["log_path"] != "":
         logfile = "{0}/{1}.log".format(conf["log"]["log_path"], conf["log"]["log_file"])
         # Assure directory exists
-        try: os.makedirs(conf["log"]["log_path"], exist_ok=True)  # Python>3.2
+        try:
+            os.makedirs(conf["log"]["log_path"], exist_ok=True)  # Python>3.2
         except TypeError:
             try:
                 os.makedirs(conf["log"]["log_path"])
-            except OSError as exc: # Python >2.5
+            except OSError as exc:  # Python >2.5
                 if exc.errno == errno.EEXIST and os.path.isdir(conf["log"]["log_path"]):
                     pass
-                else: logger.error("Can not create log file {0}: {1}".format(conf["log"]["log_path"], exc))
+                else:
+                    logger.error("Can not create log file {0}: {1}".format(conf["log"]["log_path"], exc))
     else:
         logfile = "{0}.log".format(conf["log"]["log_file"])
-    fileHandler = handlers.RotatingFileHandler(logfile, mode='a+', maxBytes=(1048576*5), backupCount=7)
+    fileHandler = handlers.RotatingFileHandler(logfile, mode='a+', maxBytes=(1048576 * 5), backupCount=7)
     if conf["log"]["format"] != "":
         fileFormatter = logging.Formatter("{0}".format(conf["log"]["format"]))
         fileHandler.setFormatter(fileFormatter)
@@ -118,7 +122,6 @@ for input_type, input_values in conf["inputs"].items():
         input_list.append(input_values["module"])
         logger.info("Enabled Input: {0}".format(input_type))
 
-
 # Configure Outputs
 logger.info("Configure Outputs")
 outputs = []
@@ -296,7 +299,6 @@ def paste_scanner(paste_data, rules_buff):
             # remove the confname key as its not really needed past this point
             del paste_data['confname']
 
-
             # Blacklist Check
             # If any of the blacklist rules appear then empty the result set
             blacklisted = False
@@ -310,7 +312,6 @@ def paste_scanner(paste_data, rules_buff):
                     return True
                 return False
 
-
             # Post Process
 
             # If post module is enabled and the paste has a matching rule.
@@ -322,14 +323,13 @@ def paste_scanner(paste_data, rules_buff):
                             logger.info("Running Post Module {0} on {1}".format(post_values["module"], paste_data["pasteid"]))
                             post_module = importlib.import_module(post_values["module"])
                             post_results = post_module.run(results,
-                                                            raw_paste_data,
-                                                            paste_data
-                                                            )
+                                                           raw_paste_data,
+                                                           paste_data
+                                                           )
 
             # Throw everything back to paste_data for ease.
             paste_data = post_results
 
-
             # If we have a result add some meta data and send to storage
             # If results is empty, ie no match, and store_all is True,
             # then append "no_match" to results. This will then force output.
@@ -356,6 +356,7 @@ def paste_scanner(paste_data, rules_buff):
     except Exception as e:
         logging.error(e)
 
+
 def main():
     logger.info("Compile Yara Rules")
     try:
@@ -364,7 +365,7 @@ def main():
             default_rules = os.path.join(pastehunter_path, "YaraRules")
         else:
             default_rules = False
-        
+
         if conf["yara"]["custom_rules"] != "none":
             custom_rules = conf["yara"]["custom_rules"]
         else:
@@ -376,7 +377,7 @@ def main():
             conf['yara']['exclude_rules'],
             conf['yara']['blacklist'],
             conf['yara']['test_rules']
-            )
+        )
 
         rules = yara.compile(filepaths=rule_files, externals={'filename': ''})
 
@@ -445,5 +446,6 @@ def main():
         pool.terminate()
         pool.join()
 
+
 if __name__ == '__main__':
-    main()
+    main()

pytest.ini

@@ -1,2 +1,3 @@
 [pytest]
-addopts = test/
+addopts = test/
+norecursedirs = .git build docs logs

settings.json.sample

@@ -19,7 +19,7 @@
       "module": "pastehunter.inputs.dumpz",
       "api_scrape": "https://dumpz.org/api/recent",
       "api_raw": "https://dumpz.org/api/dump",
-      "paste_limit": 200,
+      "paste_limit": 100,
       "store_all": false
     },
     "gists": {

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name='pastehunter',
-    version='1.3.2',
+    version='1.4.0',
     author='@kevthehermit @Plazmaz',
     author_email='[email protected]',
     description="Pastehunter",

test/test_paste_objects.py

@@ -31,6 +31,7 @@ def test_slexy_site():
         assert paste == 'pid_is_' + str(pid)
         assert paste_data == {"pid": 123}
 
+
 def test_pastebin_site_remap():
     fake_conf = {
         'inputs': {
@@ -47,8 +48,9 @@ def test_pastebin_site_remap():
     }
     pastebin_site = PastebinPasteSite(fake_conf)
     out = pastebin_site.remap_raw_item(data)
-    assert  out == {'key': 'a', 'test': 'b', 'date': '1582595793', 'filename': 'a', 'confname': 'pastebin',
-                    'pasteid': 'a', 'pastesite': 'pastebin.com', '@timestamp': '2020-02-25T01:56:33'}
+    assert out == {'key': 'a', 'test': 'b', 'date': '1582595793', 'filename': 'a', 'confname': 'pastebin',
+                   'pasteid': 'a', 'pastesite': 'pastebin.com', '@timestamp': '2020-02-25T01:56:33'}
+
 
 def test_pastebin_site():
     fake_conf = {
@@ -68,7 +70,7 @@ def test_pastebin_site():
         {
             'key': 'bc',
             'date': '1582595793'
-         }
+        }
     ])
     pastes, paste_ids = pastebin_site.get_recent_items([])
     assert paste_ids == ['ab', 'bc']