diff --git a/sflib.py b/sflib.py index a2816e7c340..f90be992f0d 100644 --- a/sflib.py +++ b/sflib.py @@ -29,7 +29,6 @@ import urllib.request from copy import deepcopy from datetime import datetime -from pathlib import Path import cryptography import dns.resolver @@ -40,6 +39,7 @@ import urllib3 from bs4 import BeautifulSoup, SoupStrainer from publicsuffixlist import PublicSuffixList +from spiderfoot import SpiderFootHelpers # For hiding the SSL warnings coming from the requests lib urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # noqa: DUO131 @@ -299,19 +299,6 @@ def hashstring(self, string: str) -> str: s = str(string) return hashlib.sha256(s.encode('raw_unicode_escape')).hexdigest() - def cachePath(self) -> str: - """Returns the file system location of the cacha data files. - - Returns: - str: SpiderFoot cache file system path - """ - path = os.environ.get('SPIDERFOOT_CACHE') - if not path: - path = f"{Path.home()}/.spiderfoot/cache" - if not os.path.isdir(path): - os.mkdir(path) - return path - def cachePut(self, label: str, data: str) -> None: """Store data to the cache @@ -320,7 +307,7 @@ def cachePut(self, label: str, data: str) -> None: data (str): Data to cache """ pathLabel = hashlib.sha224(label.encode('utf-8')).hexdigest() - cacheFile = self.cachePath() + "/" + pathLabel + cacheFile = SpiderFootHelpers.cachePath() + "/" + pathLabel with io.open(cacheFile, "w", encoding="utf-8", errors="ignore") as fp: if isinstance(data, list): for line in data: @@ -349,7 +336,7 @@ def cacheGet(self, label: str, timeoutHrs: int) -> str: return None pathLabel = hashlib.sha224(label.encode('utf-8')).hexdigest() - cacheFile = self.cachePath() + "/" + pathLabel + cacheFile = SpiderFootHelpers.cachePath() + "/" + pathLabel try: (m, i, d, n, u, g, sz, atime, mtime, ctime) = os.stat(cacheFile) diff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.py index 91ce52eb53d..8ba4bb32de6 100644 --- a/spiderfoot/helpers.py +++ b/spiderfoot/helpers.py @@ -24,7 +24,21 @@ def dataPath() -> str: if not path: path = f"{Path.home()}/.spiderfoot/" if not os.path.isdir(path): - os.mkdir(path) + os.mkdirs(path, exist_ok=True) + return path + + @staticmethod + def cachePath() -> str: + """Returns the file system location of the cacha data files. + + Returns: + str: SpiderFoot cache file system path + """ + path = os.environ.get('SPIDERFOOT_CACHE') + if not path: + path = f"{Path.home()}/.spiderfoot/cache" + if not os.path.isdir(path): + os.mkdirs(path, exist_ok=True) return path @staticmethod diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.py index 094c69dac26..249da01f155 100644 --- a/test/unit/test_spiderfoot.py +++ b/test/unit/test_spiderfoot.py @@ -173,15 +173,6 @@ def test_hash_string_should_return_a_string(self): self.assertIsInstance(hash_string, str) self.assertEqual("aedfb92b3053a21a114f4f301a02a3c6ad5dff504d124dc2cee6117623eec706", hash_string) - def test_cache_path_should_return_a_string(self): - """ - Test cachePath(self) - """ - sf = SpiderFoot(dict()) - - cache_path = sf.cachePath() - self.assertIsInstance(cache_path, str) - def test_cache_get_should_return_a_string(self): """ Test cachePut(self, label, data) diff --git a/test/unit/test_spiderfoothelpers.py b/test/unit/test_spiderfoothelpers.py index ba9c9140108..3411c1734ec 100644 --- a/test/unit/test_spiderfoothelpers.py +++ b/test/unit/test_spiderfoothelpers.py @@ -11,6 +11,20 @@ class TestSpiderFootHelpers(unittest.TestCase): Test SpiderFootHelpers """ + def test_data_path_should_return_a_string(self): + """ + Test dataPath() + """ + data_path = SpiderFootHelpers.dataPath() + self.assertIsInstance(data_path, str) + + def test_cache_path_should_return_a_string(self): + """ + Test cachePath() + """ + cache_path = SpiderFootHelpers.cachePath() + self.assertIsInstance(cache_path, str) + def test_target_type(self): """ Test targetType(target)