Skip to content

Commit

Permalink
sfscan: validate configured proxy (smicallef#1228)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bcoles
bcoles authored Jul 11, 2021
1 parent 35c4fd8 commit 65b154c
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 51 deletions.
51 changes: 23 additions & 28 deletions sflib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import urllib.error
import urllib.parse
import urllib.request
import uuid
from copy import deepcopy
from datetime import datetime

Expand Down Expand Up @@ -185,15 +184,6 @@ def optValueToData(self, val):

return val

def genScanInstanceId(self):
"""Generate an globally unique ID for this scan.
Returns:
str: scan instance unique ID
"""

return str(uuid.uuid4()).split("-")[0].upper()

def _dblog(self, level, message, component=None):
"""Log a scan event.
Expand Down Expand Up @@ -366,8 +356,9 @@ def cachePath(self):
Returns:
str: SpiderFoot cache file system path
"""

path = self.myPath() + '/cache'
path = os.environ.get('SPIDERFOOT_CACHE')
if not path:
path = self.myPath() + '/cache'
if not os.path.isdir(path):
os.mkdir(path)
return path
Expand Down Expand Up @@ -2201,6 +2192,9 @@ def useProxyForUrl(self, url):
Returns:
bool: should the configured proxy be used?
Todo:
Allow using TOR only for .onion addresses
"""
host = self.urlFQDN(url).lower()

Expand All @@ -2217,7 +2211,7 @@ def useProxyForUrl(self, url):
if not proxy_port:
return False

# Never proxy requests to the proxy
# Never proxy requests to the proxy host
if host == proxy_host.lower():
return False

Expand Down Expand Up @@ -2264,7 +2258,7 @@ def fetchUrl(
timeout (int): timeout
useragent (str): user agent header
headers (str): headers
noLog (bool): do not log
noLog (bool): do not log request
postData (str): HTTP POST data
dontMangle (bool): do not mangle
sizeLimit (int): size threshold
Expand Down Expand Up @@ -2298,16 +2292,14 @@ def fetchUrl(
self.debug(f"Invalid URL scheme for URL: {url}")
return None

request_log = []

proxies = dict()
if self.useProxyForUrl(url):
proxy_url = f"socks5h://{self.opts['_socks2addr']}:{self.opts['_socks3port']}"
self.debug(f"Using proxy {proxy_url} for {url}")
proxies = {
'http': proxy_url,
'https': proxy_url
'http': self.socksProxy,
'https': self.socksProxy,
}
else:
self.debug(f"Not using proxy for {url}")

header = dict()
btime = time.time()
Expand All @@ -2322,9 +2314,14 @@ def fetchUrl(
for k in list(headers.keys()):
header[k] = str(headers[k])

request_log.append(f"proxy={self.socksProxy}")
request_log.append(f"user-agent={header['User-Agent']}")
request_log.append(f"timeout={timeout}")
request_log.append(f"cookies={cookies}")

if sizeLimit or headOnly:
if not noLog:
self.info(f"Fetching (HEAD only): {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")
self.info(f"Fetching (HEAD): {self.removeUrlCreds(url)} ({', '.join(request_log)})")

try:
hdr = self.getSession().head(
Expand Down Expand Up @@ -2361,7 +2358,7 @@ def fetchUrl(

if result['realurl'] != url:
if not noLog:
self.info(f"Fetching (HEAD only): {self.removeUrlCreds(result['realurl'])} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")
self.info(f"Fetching (HEAD): {self.removeUrlCreds(result['realurl'])} ({', '.join(request_log)})")

try:
hdr = self.getSession().head(
Expand All @@ -2388,14 +2385,10 @@ def fetchUrl(

return result

if not noLog:
if cookies:
self.info(f"Fetching (incl. cookies): {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")
else:
self.info(f"Fetching: {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")

try:
if postData:
if not noLog:
self.info(f"Fetching (POST): {self.removeUrlCreds(url)} ({', '.join(request_log)})")
res = self.getSession().post(
url,
data=postData,
Expand All @@ -2407,6 +2400,8 @@ def fetchUrl(
verify=verify
)
else:
if not noLog:
self.info(f"Fetching (GET): {self.removeUrlCreds(url)} ({', '.join(request_log)})")
res = self.getSession().get(
url,
headers=header,
Expand Down
65 changes: 43 additions & 22 deletions sfscan.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,31 +131,52 @@ def __init__(self, scanName, scanId, targetValue, targetType, moduleList, global

# Process global options that point to other places for data

# If a SOCKS server was specified, set it up
if self.__config['_socks1type']:
socksAddr = self.__config['_socks2addr']
socksPort = int(self.__config['_socks3port'])
socksUsername = self.__config['_socks4user'] or ''
socksPassword = self.__config['_socks5pwd'] or ''

proxy = f"{socksAddr}:{socksPort}"

if socksUsername or socksPassword:
proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy)

if self.__config['_socks1type'] == '4':
proxy = 'socks4://' + proxy
elif self.__config['_socks1type'] == '5':
proxy = 'socks5://' + proxy
elif self.__config['_socks1type'] == 'HTTP':
proxy = 'http://' + proxy
elif self.__config['_socks1type'] == 'TOR':
proxy = 'socks5h://' + proxy
# If a proxy server was specified, set it up
proxy_type = self.__config.get('_socks1type')
if proxy_type:
# TODO: allow DNS lookup to be configurable when using a proxy
# - proxy DNS lookup: socks5h:// and socks4a://
# - local DNS lookup: socks5:// and socks4://
if proxy_type == '4':
proxy_proto = 'socks4://'
elif proxy_type == '5':
proxy_proto = 'socks5://'
elif proxy_type == 'HTTP':
proxy_proto = 'http://'
elif proxy_type == 'TOR':
proxy_proto = 'socks5h://'
else:
raise ValueError(f"Invalid SOCKS proxy type: {self.__config['_socks1ttype']}")
self.__sf.status(f"Scan [{self.__scanId}] failed: Invalid proxy type: {proxy_type}")
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError(f"Invalid proxy type: {proxy_type}")

self.__sf.debug(f"SOCKS: {socksAddr}:{socksPort} ({socksUsername}:{socksPassword})")
proxy_host = self.__config.get('_socks2addr', '')

if not proxy_host:
self.__sf.status(f"Scan [{self.__scanId}] failed: Proxy type is set ({proxy_type}) but proxy address value is blank")
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError(f"Proxy type is set ({proxy_type}) but proxy address value is blank")

proxy_port = int(self.__config.get('_socks3port') or 0)

if not proxy_port:
if proxy_type == '4' or proxy_type == '5':
proxy_port = 1080
elif proxy_type.upper() == 'HTTP':
proxy_port = 8080
elif proxy_type.upper() == 'TOR':
proxy_port = 9050

proxy_username = self.__config.get('_socks4user', '')
proxy_password = self.__config.get('_socks5pwd', '')

if proxy_username or proxy_password:
proxy_auth = f"{proxy_username}:{proxy_password}"
proxy = f"{proxy_proto}{proxy_auth}@{proxy_host}:{proxy_port}"
else:
proxy = f"{proxy_proto}{proxy_host}:{proxy_port}"

self.__sf.debug(f"Using proxy: {proxy}")
self.__sf.socksProxy = proxy
else:
self.__sf.socksProxy = None
Expand Down
66 changes: 65 additions & 1 deletion test/unit/test_spiderfootscanner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def test_init_argument_targetType_of_invalid_type_should_raise_TypeError(self):
with self.assertRaises(TypeError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", invalid_type, module_list, self.default_options, start=False)

def test_init_argument_targetType_as_empty_string_should_raise_ValueError(self):
def test_init_argument_targetType_invalid_value_should_raise_ValueError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
Expand All @@ -127,6 +127,10 @@ def test_init_argument_targetType_as_empty_string_should_raise_ValueError(self):
with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", target_type, module_list, self.default_options, start=False)

target_type = "INVALID_TARGET_TYPE"
with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", target_type, module_list, self.default_options, start=False)

def test_init_argument_moduleList_of_invalid_type_should_raise_TypeError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
Expand Down Expand Up @@ -172,6 +176,66 @@ def test_init_argument_globalOpts_as_empty_dict_should_raise_ValueError(self):
with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, dict(), start=False)

def test_init_argument_globalOpts_proxy_invalid_proxy_type_should_raise_ValueError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
opts = self.default_options
opts['_socks1type'] = 'invalid proxy type'
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']

with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)

def test_init_argument_globalOpts_proxy_type_without_host_should_raise_ValueError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
opts = self.default_options
opts['_socks1type'] = 'HTTP'
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']

with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)

def test_init_argument_globalOpts_proxy_should_set_proxy(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
opts = self.default_options
opts['_socks1type'] = 'HTTP'
opts['_socks2addr'] = '127.0.0.1'
opts['_socks3port'] = '8080'
opts['_socks4user'] = 'user'
opts['_socks5pwd'] = 'password'
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']

SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)

self.assertEqual('TBD', 'TBD')

def test_init_argument_globalOpts_proxy_without_port_should_set_proxy(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
opts = self.default_options
opts['_socks1type'] = 'HTTP'
opts['_socks2addr'] = '127.0.0.1'
opts['_socks3port'] = ''
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']

SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)

self.assertEqual('TBD', 'TBD')

def test_attribute_scanId_should_return_scan_id_as_a_string(self):
opts = self.default_options
opts['__modules__'] = dict()
Expand Down

0 comments on commit 65b154c

Please sign in to comment.