Skip to content

Commit

Permalink
Merge pull request sherlock-project#1205 from SethFalco/post-requests
Browse files Browse the repository at this point in the history 
feat: support post requests
  • Loading branch information
@sdushantha
sdushantha committed Dec 11, 2021
2 parents aef2711 + 737ce51 commit 96bee4f
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 48 deletions.
8 changes: 0 additions & 8 deletions removed_sites.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -645,14 +645,6 @@
"username_claimed": "TheMorozko",
"username_unclaimed": "noonewouldeverusethis7"
},
"Anilist": {
"errorType": "status_code",
"regexCheck": "^[A-Za-z0-9]{2,20}$",
"url": "https://anilist.co/user/{}/",
"urlMain": "https://anilist.co/",
"username_claimed": "Josh",
"username_unclaimed": "noonewouldeverusethi"
},
"Coil": {
"errorMsg": "Whoops, the thing you were looking for isn't here",
"errorType": "message",
Expand Down
14 changes: 0 additions & 14 deletions removed_sites.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1254,20 +1254,6 @@ As of 2021-11-21, 1337x seems to be down causing false positives.
}
```

### Anilist
As of 2021-11-26, Anilist is returning false positives.

```
"Anilist": {
"errorType": "status_code",
"regexCheck": "^[A-Za-z0-9]{2,20}$",
"url": "https://anilist.co/user/{}/",
"urlMain": "https://anilist.co/",
"username_claimed": "Josh",
"username_unclaimed": "noonewouldeverusethi"
}
```

### Coil
As of 2021-11-26, Coil is returning false positives.

Expand Down
20 changes: 18 additions & 2 deletions sherlock/resources/data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,22 @@
"username_claimed": "blue",
"username_unclaimed": "noonewouldeverusethis7"
},
"Anilist": {
"errorType": "status_code",
"regexCheck": "^[A-Za-z0-9]{2,20}$",
"request_method": "POST",
"request_payload": {
"query": "query($name:String){User(name:$name){id}}",
"variables": {
"name": "{}"
}
},
"url": "https://anilist.co/user/{}/",
"urlProbe": "https://graphql.anilist.co/",
"urlMain": "https://anilist.co/",
"username_claimed": "Josh",
"username_unclaimed": "noonewouldeverusethi"
},
"Apple Developer": {
"errorType": "status_code",
"url": "https://developer.apple.com/forums/profile/{}",
Expand Down Expand Up @@ -1792,7 +1808,7 @@
},
"Vero": {
"errorType": "status_code",
"request_head_only": false,
"request_method": "GET",
"url": "https://vero.co/{}",
"urlMain": "https://vero.co/",
"username_claimed": "blue",
Expand All @@ -1815,7 +1831,7 @@
},
"VirusTotal": {
"errorType": "status_code",
"request_head_only": false,
"request_method": "GET",
"url": "https://www.virustotal.com/ui/users/{}/trusted_users",
"urlMain": "https://www.virustotal.com/",
"urlProbe": "https://www.virustotal.com/ui/users/{}/avatar",
Expand Down
84 changes: 60 additions & 24 deletions sherlock/sherlock.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,21 @@ def get_response(request_future, error_type, social_network):
return response, error_context, expection_text


def interpolate_string(object, username):
"""Insert a string into the string properties of an object recursively."""

if isinstance(object, str):
return object.replace("{}", username)
elif isinstance(object, dict):
for key, value in object.items():
object[key] = interpolate_string(value, username)
elif isinstance(object, list):
for i in object:
object[i] = interpolate_string(object[i], username)

return object


def sherlock(username, site_data, query_notify,
tor=False, unique_tor=False,
proxy=None, timeout=None):
Expand Down Expand Up @@ -207,7 +222,7 @@ def sherlock(username, site_data, query_notify,
headers.update(net_info["headers"])

# URL of user on site (if it exists)
url = net_info["url"].format(username)
url = interpolate_string(net_info["url"], username)

# Don't make request if username is invalid for the site
regex_check = net_info.get("regexCheck")
Expand All @@ -225,25 +240,44 @@ def sherlock(username, site_data, query_notify,
# URL of user on site (if it exists)
results_site["url_user"] = url
url_probe = net_info.get("urlProbe")
request_method = net_info.get("request_method")
request_payload = net_info.get("request_payload")
request = None

if request_method is not None:
if request_method == "GET":
request = session.get
elif request_method == "HEAD":
request = session.head
elif request_method == "POST":
request = session.post
elif request_method == "PUT":
request = session.put
else:
raise RuntimeError( f"Unsupported request_method for {url}")

if request_payload is not None:
request_payload = interpolate_string(request_payload, username)

if url_probe is None:
# Probe URL is normal one seen by people out on the web.
url_probe = url
else:
# There is a special URL for probing existence separate
# from where the user profile normally can be found.
url_probe = url_probe.format(username)

if (net_info["errorType"] == 'status_code' and
net_info.get("request_head_only", True) == True):
# In most cases when we are detecting by status code,
# it is not necessary to get the entire body: we can
# detect fine with just the HEAD response.
request_method = session.head
else:
# Either this detect method needs the content associated
# with the GET response, or this specific website will
# not respond properly unless we request the whole page.
request_method = session.get
url_probe = interpolate_string(url_probe, username)

if request is None:
if net_info["errorType"] == 'status_code':
# In most cases when we are detecting by status code,
# it is not necessary to get the entire body: we can
# detect fine with just the HEAD response.
request = session.head
else:
# Either this detect method needs the content associated
# with the GET response, or this specific website will
# not respond properly unless we request the whole page.
request = session.get

if net_info["errorType"] == "response_url":
# Site forwards request to a different URL if username not
Expand All @@ -258,16 +292,18 @@ def sherlock(username, site_data, query_notify,
# This future starts running the request in a new thread, doesn't block the main thread
if proxy is not None:
proxies = {"http": proxy, "https": proxy}
future = request_method(url=url_probe, headers=headers,
proxies=proxies,
allow_redirects=allow_redirects,
timeout=timeout
)
future = request(url=url_probe, headers=headers,
proxies=proxies,
allow_redirects=allow_redirects,
timeout=timeout,
json=request_payload
)
else:
future = request_method(url=url_probe, headers=headers,
allow_redirects=allow_redirects,
timeout=timeout
)
future = request(url=url_probe, headers=headers,
allow_redirects=allow_redirects,
timeout=timeout,
json=request_payload
)

# Store future in data for access later
net_info["request_future"] = future
Expand Down Expand Up @@ -314,7 +350,7 @@ def sherlock(username, site_data, query_notify,
except:
http_status = "?"
try:
response_text = r.text.encode(r.encoding)
response_text = r.text.encode(r.encoding or "UTF-8")
except:
response_text = ""

Expand Down
1 change: 1 addition & 0 deletions sites.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
1. [Airliners](https://www.airliners.net/)
1. [Alik.cz](https://www.alik.cz/)
1. [AllMyLinks](https://allmylinks.com/)
1. [Anilist](https://anilist.co/)
1. [Apple Developer](https://developer.apple.com)
1. [Apple Discussions](https://discussions.apple.com)
1. [Archive.org](https://archive.org)
Expand Down

0 comments on commit 96bee4f

Please sign in to comment.