forked from smicallef/spiderfoot
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Grayhat Warfare module (smicallef#1136)
* Grayhat Warfare Module
- Loading branch information
1 parent
99c2291
commit 88dcb8d
Showing
2 changed files
with
255 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,175 @@ | ||
# ------------------------------------------------------------------------------- | ||
# Name: sfp_grayhatwarfare | ||
# Purpose: Find bucket names matching the keyword extracted from a domain from Grayhat API. | ||
# | ||
# Author: <krishnasis@hotmail.com> | ||
# | ||
# Created: 24-01-2021 | ||
# Copyright: (c) Steve Micallef | ||
# Licence: GPL | ||
# ------------------------------------------------------------------------------- | ||
|
||
import json | ||
import time | ||
import urllib | ||
from spiderfoot import SpiderFootEvent, SpiderFootPlugin | ||
|
||
|
||
class sfp_grayhatwarfare(SpiderFootPlugin): | ||
|
||
meta = { | ||
'name': "Grayhat Warfare", | ||
'summary': "Find bucket names matching the keyword extracted from a domain from Grayhat API.", | ||
'flags': ["apikey"], | ||
'useCases': ["Footprint", "Investigate", "Passive"], | ||
'categories': ["Reputation Systems"], | ||
'dataSource': { | ||
'website': "https://buckets.grayhatwarfare.com/", | ||
'model': "FREE_AUTH_LIMITED", | ||
'references': [ | ||
"https://buckets.grayhatwarfare.com/docs/api/v1" | ||
], | ||
'favIcon': "https://buckets.grayhatwarfare.com/assets/template/images/favicon.png", | ||
'logo': "https://buckets.grayhatwarfare.com/assets/images/logo/logo-sm.png", | ||
'description': "It is a searchable database of open buckets." | ||
"Has up to million results of each bucket." | ||
"Full text search with binary logic (can search for keywords and also stopwords)", | ||
} | ||
} | ||
|
||
# Default options | ||
opts = { | ||
'api_key': '', | ||
'per_page': 1000, | ||
'max_pages': 2, | ||
'pause': 1 | ||
} | ||
|
||
# Option descriptions | ||
optdescs = { | ||
'api_key': 'Grayhat Warfare API key.', | ||
'per_page': 'Maximum number of results per page (Max: 1000).', | ||
'max_pages': 'Maximum number of pages to fetch.', | ||
'pause': 'Number of seconds to wait between each API call.' | ||
} | ||
|
||
results = None | ||
errorState = False | ||
|
||
def setup(self, sfc, userOpts=dict()): | ||
self.sf = sfc | ||
self.results = self.tempStorage() | ||
|
||
for opt in list(userOpts.keys()): | ||
self.opts[opt] = userOpts[opt] | ||
|
||
# What events is this module interested in for input | ||
def watchedEvents(self): | ||
return [ | ||
"DOMAIN_NAME", | ||
] | ||
|
||
# What events this module produces | ||
def producedEvents(self): | ||
return [ | ||
'CLOUD_STORAGE_BUCKET', | ||
'CLOUD_STORAGE_BUCKET_OPEN', | ||
'RAW_RIR_DATA' | ||
] | ||
|
||
# Query Grayhat Warfare | ||
def query(self, keyword, start): | ||
params = urllib.parse.urlencode({ | ||
'keywords': keyword.encode('raw_unicode_escape'), | ||
'access_token': self.opts['api_key'] | ||
}) | ||
|
||
headers = { | ||
'Accept': 'application/json', | ||
} | ||
|
||
res = self.sf.fetchUrl( | ||
f"https://buckets.grayhatwarfare.com/api/v1/buckets/{start}/{self.opts['per_page']}?{params}", | ||
headers=headers, | ||
timeout=15, | ||
useragent=self.opts['_useragent'], | ||
verify=True | ||
) | ||
|
||
time.sleep(self.opts['pause']) | ||
|
||
if res['code'] != "200": | ||
self.sf.error("Unable to fetch data from Grayhat Warfare API.") | ||
self.errorState = True | ||
return None | ||
|
||
if res['content'] is None: | ||
self.sf.debug('No response from Grayhat Warfare API.') | ||
return None | ||
|
||
try: | ||
return json.loads(res['content']) | ||
except Exception as e: | ||
self.sf.debug(f"Error processing JSON response: {e}") | ||
return None | ||
|
||
# Handle events sent to this module | ||
def handleEvent(self, event): | ||
eventName = event.eventType | ||
srcModuleName = event.module | ||
eventData = event.data | ||
|
||
if eventData in self.results: | ||
return None | ||
|
||
if self.errorState: | ||
return None | ||
|
||
self.results[eventData] = True | ||
|
||
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") | ||
|
||
if self.opts['api_key'] == "": | ||
self.sf.error("You enabled sfp_grayhatwarfare but did not set an API key!") | ||
self.errorState = True | ||
return | ||
|
||
currentIndex = 0 | ||
currentPage = 0 | ||
maxPages = self.opts['max_pages'] | ||
perPage = self.opts['per_page'] | ||
|
||
keyword = self.sf.domainKeyword(eventData, self.opts['_internettlds']) | ||
|
||
while currentPage < maxPages: | ||
currentIndex = currentPage * perPage | ||
if self.checkForStop(): | ||
return None | ||
|
||
if self.errorState: | ||
break | ||
|
||
data = self.query(keyword=keyword, start=currentIndex) | ||
|
||
if not data: | ||
return None | ||
|
||
for row in data.get('buckets'): | ||
bucketName = row.get('bucket') | ||
bucketKeyword = bucketName.split('.')[0] | ||
self.sf.debug(bucketKeyword) | ||
if bucketKeyword.startswith(keyword) or bucketKeyword.endswith(keyword): | ||
evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET', bucketName, self.__name__, event) | ||
self.notifyListeners(evt) | ||
|
||
evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET_OPEN', f"{bucketName}: {row.get('fileCount')} files found.", self.__name__, event) | ||
self.notifyListeners(evt) | ||
|
||
evt = SpiderFootEvent('RAW_RIR_DATA', str(row), self.__name__, event) | ||
self.notifyListeners(evt) | ||
|
||
currentPage += 1 | ||
if data.get('buckets_count') < perPage: | ||
break | ||
|
||
# End of sfp_grayhatwarfare class |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
# test_sfp_grayhatwarfare.py | ||
import unittest | ||
|
||
from modules.sfp_grayhatwarfare import sfp_grayhatwarfare | ||
from sflib import SpiderFoot | ||
from spiderfoot import SpiderFootEvent, SpiderFootTarget | ||
|
||
|
||
class TestModuleGrayhatWarfare(unittest.TestCase): | ||
""" | ||
Test modules.sfp_grayhatwarfare | ||
""" | ||
|
||
default_options = { | ||
'_debug': False, # Debug | ||
'__logging': True, # Logging in general | ||
'__outputfilter': None, # Event types to filter from modules' output | ||
'_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests | ||
'_dnsserver': '', # Override the default resolver | ||
'_fetchtimeout': 5, # number of seconds before giving up on a fetch | ||
'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat', | ||
'_internettlds_cache': 72, | ||
'_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www", | ||
'__version__': '3.3-DEV', | ||
'__database': 'spiderfoot.test.db', # note: test database file | ||
'__modules__': None, # List of modules. Will be set after start-up. | ||
'_socks1type': '', | ||
'_socks2addr': '', | ||
'_socks3port': '', | ||
'_socks4user': '', | ||
'_socks5pwd': '', | ||
'_torctlport': 9051, | ||
'__logstdout': False | ||
} | ||
|
||
def test_opts(self): | ||
module = sfp_grayhatwarfare() | ||
self.assertEqual(len(module.opts), len(module.optdescs)) | ||
|
||
def test_setup(self): | ||
""" | ||
Test setup(self, sfc, userOpts=dict()) | ||
""" | ||
sf = SpiderFoot(self.default_options) | ||
|
||
module = sfp_grayhatwarfare() | ||
module.setup(sf, dict()) | ||
|
||
def test_watchedEvents_should_return_list(self): | ||
module = sfp_grayhatwarfare() | ||
self.assertIsInstance(module.watchedEvents(), list) | ||
|
||
def test_producedEvents_should_return_list(self): | ||
module = sfp_grayhatwarfare() | ||
self.assertIsInstance(module.producedEvents(), list) | ||
|
||
@unittest.skip("todo") | ||
def test_handleEvent(self): | ||
""" | ||
Test handleEvent(self, event) | ||
""" | ||
sf = SpiderFoot(self.default_options) | ||
|
||
module = sfp_grayhatwarfare() | ||
module.setup(sf, dict()) | ||
|
||
target_value = 'example target value' | ||
target_type = 'DOMAIN_NAME' | ||
target = SpiderFootTarget(target_value, target_type) | ||
module.setTarget(target) | ||
|
||
event_type = 'ROOT' | ||
event_data = 'example data' | ||
event_module = '' | ||
source_event = '' | ||
evt = SpiderFootEvent(event_type, event_data, event_module, source_event) | ||
|
||
result = module.handleEvent(evt) | ||
|
||
self.assertIsNone(result) |