Merge pull request #10 from smicallef/master

Update

.pylintrc

@@ -54,11 +54,9 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=print-statement,
-        parameter-unpacking,
+disable=parameter-unpacking,
         unpacking-in-except,
         old-raise-syntax,
-        backtick,
         long-suffix,
         old-ne-operator,
         old-octal-literal,
@@ -129,7 +127,6 @@ disable=print-statement,
         dict-keys-not-iterating,
         dict-values-not-iterating,
         deprecated-operator-function,
-        deprecated-urllib-function,
         xreadlines-attribute,
         deprecated-sys-function,
         exception-escape,
@@ -183,14 +180,14 @@ never-returning-functions=optparse.Values,sys.exit
 [BASIC]
 
 # Naming style matching correct argument names
-argument-naming-style=snake_case
+argument-naming-style=camelCase
 
 # Regular expression matching correct argument names. Overrides argument-
 # naming-style
 #argument-rgx=
 
 # Naming style matching correct attribute names
-attr-naming-style=snake_case
+attr-naming-style=camelCase
 
 # Regular expression matching correct attribute names. Overrides attr-naming-
 # style
@@ -229,7 +226,7 @@ const-naming-style=UPPER_CASE
 docstring-min-length=-1
 
 # Naming style matching correct function names
-function-naming-style=snake_case
+function-naming-style=camelCase
 
 # Regular expression matching correct function names. Overrides function-
 # naming-style
@@ -240,9 +237,7 @@ good-names=i,
            j,
            k,
            e,
-           ex,
            sf,
-           Run,
            _
 
 # Include a hint for the correct naming format with invalid-name
@@ -256,14 +251,14 @@ inlinevar-naming-style=any
 #inlinevar-rgx=
 
 # Naming style matching correct method names
-method-naming-style=snake_case
+method-naming-style=camelCase
 
 # Regular expression matching correct method names. Overrides method-naming-
 # style
 #method-rgx=
 
 # Naming style matching correct module names
-module-naming-style=snake_case
+module-naming-style=camelCase
 
 # Regular expression matching correct module names. Overrides module-naming-
 # style
@@ -282,7 +277,7 @@ no-docstring-rgx=^_
 property-classes=abc.abstractproperty
 
 # Naming style matching correct variable names
-variable-naming-style=snake_case
+variable-naming-style=camelCase
 
 # Regular expression matching correct variable names. Overrides variable-
 # naming-style
@@ -347,7 +342,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=120
 
 # Maximum number of lines in a module
 max-module-lines=1000

.travis.yml

@@ -10,13 +10,14 @@ before_install:
   - python --version
   - pip install -U pip
   - pip install -U pytest
+  - pip install -U pytest-flake8
   - pip install codecov
 
 install:
   - pip install -r requirements_test.txt
 
 script:
-  - python -m pytest --durations=5 --cov-report term --cov=. test/
+  - python -m pytest --flake8 --durations=5 --cov-report term --cov=. .
 
 after_success:
   - bash <(curl -s https://codecov.io/bash)

modules/sfp__stor_db.py

@@ -11,7 +11,7 @@
 # Licence:     GPL
 # -------------------------------------------------------------------------------
 
-from sflib import SpiderFoot, SpiderFootPlugin
+from sflib import SpiderFootPlugin
 
 
 class sfp__stor_db(SpiderFootPlugin):

modules/sfp__stor_stdout.py

@@ -10,7 +10,7 @@
 # Licence:     GPL
 # -------------------------------------------------------------------------------
 
-from sflib import SpiderFoot, SpiderFootPlugin
+from sflib import SpiderFootPlugin
 import json
 
 
@@ -27,7 +27,7 @@ class sfp__stor_stdout(SpiderFootPlugin):
 
     # Default options
     opts = {
-        "_format": "tab", # tab, csv, json
+        "_format": "tab",  # tab, csv, json
         "_requested": [],
         "_showonlyrequested": False,
         "_stripnewline": False,

modules/sfp_abusech.py

@@ -13,44 +13,38 @@
 from netaddr import IPAddress, IPNetwork
 import re
 
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
 
 malchecks = {
     'abuse.ch Zeus Tracker (Domain)': {
         'id': 'abusezeusdomain',
-        'type': 'list',
         'checks': ['domain'],
         'url': 'https://zeustracker.abuse.ch/blocklist.php?download=baddomains'
     },
     'abuse.ch Zeus Tracker (IP)': {
         'id': 'abusezeusip',
-        'type': 'list',
         'checks': ['ip', 'netblock'],
         'url': 'https://zeustracker.abuse.ch/blocklist.php?download=badips'
     },
     'abuse.ch Feodo Tracker (IP)': {
         'id': 'abusefeodoip',
-        'type': 'list',
         'checks': ['ip', 'netblock'],
         'url': 'https://feodotracker.abuse.ch/downloads/ipblocklist.txt'
     },
     'abuse.ch SSL Blacklist (IP)': {
         'id': 'abusesslblip',
-        'type': 'list',
         'checks': ['ip', 'netblock'],
         'url': 'https://sslbl.abuse.ch/blacklist/sslipblacklist.csv',
         'regex': '{0},.*'
     },
     'abuse.ch URLhaus (Domain)': {
         'id': 'abuseurlhaus',
-        'type': 'list',
         'checks': ['domain'],
         'url': 'https://urlhaus.abuse.ch/downloads/csv/',
         'regex': '.*//{0}/.*'
     },
     'abuse.ch Ransomware Blocklist (Domain)': {
         'id': 'abuseransomdom',
-        'type': 'list',
         'checks': ['domain'],
         'url': 'https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt',
         'regex': '^{0}$'
@@ -64,8 +58,8 @@ class sfp_abusech(SpiderFootPlugin):
         'name': "abuse.ch",
         'summary': "Check if a host/domain, IP or netblock is malicious according to abuse.ch.",
         'flags': [],
-        'useCases': [ "Passive", "Investigate" ],
-        'categories': [ "Reputation Systems" ],
+        'useCases': ["Passive", "Investigate"],
+        'categories': ["Reputation Systems"],
         'dataSource': {
             'website': "https://www.abuse.ch",
             'model': "FREE_AUTH_UNLIMITED",
@@ -160,44 +154,6 @@ def producedEvents(self):
                 "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME",
                 "MALICIOUS_SUBNET", "MALICIOUS_COHOST", "MALICIOUS_NETBLOCK"]
 
-    # Check the regexps to see whether the content indicates maliciousness
-    def contentMalicious(self, content, goodregex, badregex):
-        # First, check for the bad indicators
-        if len(badregex) > 0:
-            for rx in badregex:
-                if re.match(rx, content, re.IGNORECASE | re.DOTALL):
-                    self.sf.debug("Found to be bad against bad regex: " + rx)
-                    return True
-
-        # Finally, check for good indicators
-        if len(goodregex) > 0:
-            for rx in goodregex:
-                if re.match(rx, content, re.IGNORECASE | re.DOTALL):
-                    self.sf.debug("Found to be good againt good regex: " + rx)
-                    return False
-
-        # If nothing was matched, reply None
-        self.sf.debug("Neither good nor bad, unknown.")
-        return None
-
-    # Look up 'query' type sources
-    def resourceQuery(self, id, target, targetType):
-        self.sf.debug("Querying " + id + " for maliciousness of " + target)
-        for check in list(malchecks.keys()):
-            cid = malchecks[check]['id']
-            if id == cid and malchecks[check]['type'] == "query":
-                url = str(malchecks[check]['url'])
-                res = self.sf.fetchUrl(url.format(target), timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])
-                if res['content'] is None:
-                    self.sf.error("Unable to fetch " + url.format(target), False)
-                    return None
-                if self.contentMalicious(res['content'],
-                                         malchecks[check]['goodregex'],
-                                         malchecks[check]['badregex']):
-                    return url.format(target)
-
-        return None
-
     # Look up 'list' type resources
     def resourceList(self, id, target, targetType):
         targetDom = ''
@@ -209,7 +165,7 @@ def resourceList(self, id, target, targetType):
 
         for check in list(malchecks.keys()):
             cid = malchecks[check]['id']
-            if id == cid and malchecks[check]['type'] == "list":
+            if id == cid:
                 data = dict()
                 url = malchecks[check]['url']
                 data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))
@@ -228,8 +184,7 @@ def resourceList(self, id, target, targetType):
                     # build a list of IP.
                     # Cycle through each IP and check if it's in the netblock.
                     if 'regex' in malchecks[check]:
-                        rx = malchecks[check]['regex'].replace("{0}",
-                                                               "(\d+\.\d+\.\d+\.\d+)")
+                        rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
                         pat = re.compile(rx, re.IGNORECASE)
                         self.sf.debug("New regex for " + check + ": " + rx)
                         for line in data['content'].split('\n'):
@@ -282,12 +237,8 @@ def lookupItem(self, resourceId, itemType, target):
         for check in list(malchecks.keys()):
             cid = malchecks[check]['id']
             if cid == resourceId and itemType in malchecks[check]['checks']:
-                self.sf.debug("Checking maliciousness of " + target + " (" +
-                              itemType + ") with: " + cid)
-                if malchecks[check]['type'] == "query":
-                    return self.resourceQuery(cid, target, itemType)
-                if malchecks[check]['type'] == "list":
-                    return self.resourceList(cid, target, itemType)
+                self.sf.debug("Checking maliciousness of " + target + " (" + itemType + ") with: " + cid)
+                return self.resourceList(cid, target, itemType)
 
         return None
 
@@ -297,13 +248,13 @@ def handleEvent(self, event):
         srcModuleName = event.module
         eventData = event.data
 
-        self.sf.debug("Received event, %s, from %s" % (eventName, srcModuleName))
+        self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
 
         if eventData in self.results:
-            self.sf.debug("Skipping " + eventData + ", already checked.")
+            self.sf.debug(f"Skipping {eventData}, already checked.")
             return None
-        else:
-            self.results[eventData] = True
+
+        self.results[eventData] = True
 
         if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):
             return None
@@ -319,11 +270,11 @@ def handleEvent(self, event):
             cid = malchecks[check]['id']
 
             if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:
-               typeId = 'ip'
-               if eventName == 'IP_ADDRESS':
-                   evtType = 'MALICIOUS_IPADDR'
-               else:
-                   evtType = 'MALICIOUS_AFFILIATE_IPADDR'
+                typeId = 'ip'
+                if eventName == 'IP_ADDRESS':
+                    evtType = 'MALICIOUS_IPADDR'
+                else:
+                    evtType = 'MALICIOUS_AFFILIATE_IPADDR'
 
             if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']:
                 typeId = 'asn'
@@ -333,11 +284,11 @@ def handleEvent(self, event):
                              'AFFILIATE_INTERNET_NAME']:
                 typeId = 'domain'
                 if eventName == "INTERNET_NAME":
-                   evtType = "MALICIOUS_INTERNET_NAME"
+                    evtType = "MALICIOUS_INTERNET_NAME"
                 if eventName == 'AFFILIATE_INTERNET_NAME':
-                   evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'
+                    evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'
                 if eventName == 'CO_HOSTED_SITE':
-                   evtType = 'MALICIOUS_COHOST'
+                    evtType = 'MALICIOUS_COHOST'
 
             if eventName == 'NETBLOCK_OWNER':
                 typeId = 'netblock'