sflib: Move several functions to SpiderFootHelpers (smicallef#1679)

modules/sfp_botscout.py

@@ -15,7 +15,7 @@
 import urllib.parse
 import urllib.request
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_botscout(SpiderFootPlugin):
@@ -92,7 +92,7 @@ def queryIp(self, ip):
         return self.parseApiResponse(res)
 
     def queryEmail(self, email):
-        if not self.sf.validEmail(email):
+        if not SpiderFootHelpers.validEmail(email):
             return None
 
         params = urllib.parse.urlencode({

modules/sfp_builtwith.py

@@ -12,7 +12,7 @@
 import json
 import time
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_builtwith(SpiderFootPlugin):
@@ -155,7 +155,7 @@ def handleEvent(self, event):
                                             self.__name__, event)
                         self.notifyListeners(e)
                         if nb.get('Email', None):
-                            if self.sf.validEmail(nb['Email']):
+                            if SpiderFootHelpers.validEmail(nb['Email']):
                                 if nb['Email'].split("@")[0] in self.opts['_genericusers'].split(","):
                                     evttype = "EMAILADDR_GENERIC"
                                 else:
@@ -166,7 +166,7 @@ def handleEvent(self, event):
 
                 if data['Meta'].get("Emails", []):
                     for email in data['Meta']['Emails']:
-                        if self.sf.validEmail(email):
+                        if SpiderFootHelpers.validEmail(email):
                             if email.split("@")[0] in self.opts['_genericusers'].split(","):
                                 evttype = "EMAILADDR_GENERIC"
                             else:

modules/sfp_countryname.py

@@ -16,7 +16,7 @@
 import phonenumbers
 from phonenumbers.phonenumberutil import region_code_for_country_code
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_countryname(SpiderFootPlugin):
@@ -55,7 +55,7 @@ def setup(self, sfc, userOpts=dict()):
         for opt in userOpts.keys():
             self.opts[opt] = userOpts[opt]
 
-    def detectCountryFromPhone(self, srcPhoneNumber):
+    def detectCountryFromPhone(self, srcPhoneNumber: str) -> str:
         """Lookup name of country from phone number region code.
 
         Args:
@@ -83,9 +83,9 @@ def detectCountryFromPhone(self, srcPhoneNumber):
         if not countryCode:
             return None
 
-        return self.sf.countryNameFromCountryCode(countryCode.upper())
+        return SpiderFootHelpers.countryNameFromCountryCode(countryCode.upper())
 
-    def detectCountryFromDomainName(self, srcDomain):
+    def detectCountryFromDomainName(self, srcDomain: str) -> str:
         """Lookup name of country from TLD of domain name.
 
         Args:
@@ -103,13 +103,13 @@ def detectCountryFromDomainName(self, srcDomain):
 
         # Search for country TLD in the domain parts - reversed
         for part in domainParts[::-1]:
-            country_name = self.sf.countryNameFromTld(part)
+            country_name = SpiderFootHelpers.countryNameFromTld(part)
             if country_name:
                 return country_name
 
         return None
 
-    def detectCountryFromIBAN(self, srcIBAN):
+    def detectCountryFromIBAN(self, srcIBAN: str) -> str:
         """Detect name of country from IBAN.
 
         Args:
@@ -121,9 +121,9 @@ def detectCountryFromIBAN(self, srcIBAN):
         if not isinstance(srcIBAN, str):
             return None
 
-        return self.sf.countryNameFromCountryCode(srcIBAN[0:2])
+        return SpiderFootHelpers.countryNameFromCountryCode(srcIBAN[0:2])
 
-    def detectCountryFromData(self, srcData):
+    def detectCountryFromData(self, srcData: str) -> list:
         """Detect name of country from event data (WHOIS lookup, Geo Info, Physical Address, etc)
 
         Args:
@@ -138,7 +138,7 @@ def detectCountryFromData(self, srcData):
             return countries
 
         # Get dictionary of country codes and  country names
-        abbvCountryCodes = self.sf.getCountryCodeDict()
+        abbvCountryCodes = SpiderFootHelpers.countryCodes()
 
         # Look for countrycodes and country in source data
         for countryName in abbvCountryCodes.values():

modules/sfp_creditcard.py

@@ -11,7 +11,7 @@
 # Licence:     MIT
 # -------------------------------------------------------------------------------
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_creditcard(SpiderFootPlugin):
@@ -58,7 +58,7 @@ def handleEvent(self, event):
 
         self.debug(f"Received event, {eventName}, from {srcModuleName}")
 
-        creditCards = self.sf.parseCreditCards(eventData)
+        creditCards = SpiderFootHelpers.extractCreditCardsFromText(eventData)
 
         for creditCard in set(creditCards):
             self.info(f"Found credit card number: {creditCard}")

modules/sfp_crossref.py

@@ -15,7 +15,7 @@
 
 import re
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_crossref(SpiderFootPlugin):
@@ -124,7 +124,7 @@ def handleEvent(self, event):
             # fetch the base URL of the affiliate to check for a crossref.
             if eventName == "LINKED_URL_EXTERNAL" and self.opts['checkbase']:
                 # Check the base url to see if there is an affiliation
-                url = self.sf.urlBaseUrl(eventData)
+                url = SpiderFootHelpers.urlBaseUrl(eventData)
                 if url in self.fetched:
                     return
 

modules/sfp_email.py

@@ -11,7 +11,7 @@
 # Licence:     MIT
 # -------------------------------------------------------------------------------
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_email(SpiderFootPlugin):
@@ -56,7 +56,7 @@ def handleEvent(self, event):
 
         self.debug(f"Received event, {eventName}, from {srcModuleName}")
 
-        emails = self.sf.parseEmails(eventData)
+        emails = SpiderFootHelpers.extractEmailsFromText(eventData)
         for email in set(emails):
             evttype = "EMAILADDR"
             email = email.lower()

modules/sfp_emailformat.py

@@ -15,7 +15,7 @@
 
 from bs4 import BeautifulSoup
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_emailformat(SpiderFootPlugin):
@@ -90,7 +90,7 @@ def handleEvent(self, event):
             # fall back to raw page contents
             data = res["content"]
 
-        emails = self.sf.parseEmails(data)
+        emails = SpiderFootHelpers.extractEmailsFromText(data)
         for email in emails:
             # Skip unrelated emails
             mailDom = email.lower().split('@')[1]

modules/sfp_flickr.py

@@ -18,7 +18,7 @@
 import urllib.parse
 import urllib.request
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_flickr(SpiderFootPlugin):
@@ -203,7 +203,7 @@ def handleEvent(self, event):
 
             # Extract data
             for photo in photos.get('photo', list()):
-                emails = self.sf.parseEmails(str(photo))
+                emails = SpiderFootHelpers.extractEmailsFromText(str(photo))
                 for email in emails:
                     if email in self.results:
                         continue
@@ -224,7 +224,7 @@ def handleEvent(self, event):
                     self.notifyListeners(evt)
                     self.results[email] = True
 
-                links = self.sf.extractUrls(str(photo))
+                links = SpiderFootHelpers.extractUrlsFromText(str(photo))
                 for link in links:
                     if link in self.results:
                         continue

modules/sfp_gravatar.py

@@ -15,7 +15,7 @@
 import json
 import time
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_gravatar(SpiderFootPlugin):
@@ -163,7 +163,7 @@ def handleEvent(self, event):
                 em = email.get('value')
                 if not em:
                     continue
-                if self.sf.validEmail(em) and em != eventData:
+                if SpiderFootHelpers.validEmail(em) and em != eventData:
                     if em.split("@")[0] in self.opts['_genericusers'].split(","):
                         evttype = "EMAILADDR_GENERIC"
                     else:

modules/sfp_grep_app.py

@@ -18,7 +18,7 @@
 import urllib.parse
 import urllib.request
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_grep_app(SpiderFootPlugin):
@@ -196,7 +196,7 @@ def handleEvent(self, event):
                         self.notifyListeners(evt)
                         self.results[link] = True
 
-                emails = self.sf.parseEmails(snippet.replace('<mark>', '').replace('</mark>', ''))
+                emails = SpiderFootHelpers.extractEmailsFromText(snippet.replace('<mark>', '').replace('</mark>', ''))
                 if emails:
                     for email in emails:
                         if email in self.results:

modules/sfp_hashes.py

@@ -11,7 +11,7 @@
 # Licence:     MIT
 # -------------------------------------------------------------------------------
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_hashes(SpiderFootPlugin):
@@ -59,11 +59,11 @@ def handleEvent(self, event):
 
         self.debug(f"Received event, {eventName}, from {srcModuleName}")
 
-        hashes = self.sf.parseHashes(eventData)
+        hashes = SpiderFootHelpers.extractHashesFromText(eventData)
         for hashtup in hashes:
             hashalgo, hashval = hashtup
 
-            evt = SpiderFootEvent("HASH", "[" + hashalgo + "] " + hashval, self.__name__, event)
+            evt = SpiderFootEvent("HASH", f"[{hashalgo}] {hashval}", self.__name__, event)
             if event.moduleDataSource:
                 evt.moduleDataSource = event.moduleDataSource
             else:

modules/sfp_iban.py

@@ -11,7 +11,7 @@
 # Licence:     MIT
 # -------------------------------------------------------------------------------
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_iban(SpiderFootPlugin):
@@ -59,8 +59,8 @@ def handleEvent(self, event):
 
         self.debug(f"Received event, {eventName}, from {srcModuleName}")
 
-        ibanNumbers = self.sf.parseIBANNumbers(eventData)
-        for ibanNumber in set(ibanNumbers):
+        ibans = SpiderFootHelpers.extractIbansFromText(eventData)
+        for ibanNumber in set(ibans):
             self.info(f"Found IBAN number: {ibanNumber}")
             evt = SpiderFootEvent("IBAN_NUMBER", ibanNumber, self.__name__, event)
             if event.moduleDataSource:

modules/sfp_jsonwhoiscom.py

@@ -16,7 +16,7 @@
 import urllib.parse
 import urllib.request
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_jsonwhoiscom(SpiderFootPlugin):
@@ -215,7 +215,7 @@ def handleEvent(self, event):
         for contact in contacts:
             email = contact.get('email')
             if email:
-                if self.sf.validEmail(email):
+                if SpiderFootHelpers.validEmail(email):
                     emails.append(email)
 
             name = contact.get("name")
@@ -227,7 +227,7 @@ def handleEvent(self, event):
                 phone = phone.replace(" ", "").replace("-", "").replace("(", "").replace(")", "").replace(".", "")
                 phones.append(phone)
 
-            country = self.sf.countryNameFromCountryCode(contact.get('country_code'))
+            country = SpiderFootHelpers.countryNameFromCountryCode(contact.get('country_code'))
             location = ', '.join([_f for _f in [contact.get('address'), contact.get('city'), contact.get('state'), contact.get('zip'), country] if _f])
             if location:
                 locations.append(location)

modules/sfp_junkfiles.py

@@ -12,7 +12,7 @@
 
 import random
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_junkfiles(SpiderFootPlugin):
@@ -77,7 +77,7 @@ def checkValidity(self, junkUrl):
                                useragent=self.opts['_useragent'],
                                verify=False)
         if res['code'] != "404":
-            host = self.sf.urlBaseUrl(junkUrl)
+            host = SpiderFootHelpers.urlBaseUrl(junkUrl)
             self.skiphosts[host] = True
             return False
         return True
@@ -95,7 +95,7 @@ def handleEvent(self, event):
 
         self.results[eventData] = True
 
-        host = self.sf.urlBaseUrl(eventData)
+        host = SpiderFootHelpers.urlBaseUrl(eventData)
 
         if host in self.skiphosts:
             self.debug("Skipping " + host + " because it doesn't return 404s.")
@@ -137,7 +137,7 @@ def handleEvent(self, event):
                         evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event)
                         self.notifyListeners(evt)
 
-        base = self.sf.urlBaseDir(eventData)
+        base = SpiderFootHelpers.urlBaseDir(eventData)
         if not base or base in self.bases:
             return
 

modules/sfp_numverify.py

@@ -16,7 +16,7 @@
 import urllib.parse
 import urllib.request
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_numverify(SpiderFootPlugin):
@@ -160,7 +160,7 @@ def handleEvent(self, event):
         self.notifyListeners(evt)
 
         if data.get('country_code'):
-            country = self.sf.countryNameFromCountryCode(data.get('country_code'))
+            country = SpiderFootHelpers.countryNameFromCountryCode(data.get('country_code'))
             location = ', '.join([_f for _f in [data.get('location'), country] if _f])
             evt = SpiderFootEvent("GEOINFO", location, self.__name__, event)
             self.notifyListeners(evt)

modules/sfp_pastebin.py

@@ -13,7 +13,7 @@
 
 import re
 
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
 
 
 class sfp_pastebin(SpiderFootPlugin):
@@ -124,7 +124,7 @@ def handleEvent(self, event):
                 self.results[link] = True
 
             relevant_links = [
-                link for link in new_links if self.sf.urlBaseUrl(link).endswith(target)
+                link for link in new_links if SpiderFootHelpers.urlBaseUrl(link).endswith(target)
             ]
 
             for link in relevant_links: