Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support post requests #1205

Merged
merged 1 commit into from
Dec 11, 2021
Merged

feat: support post requests #1205

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions removed_sites.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,14 +645,6 @@
"username_claimed": "TheMorozko",
"username_unclaimed": "noonewouldeverusethis7"
},
"Anilist": {
"errorType": "status_code",
"regexCheck": "^[A-Za-z0-9]{2,20}$",
"url": "https://anilist.co/user/{}/",
"urlMain": "https://anilist.co/",
"username_claimed": "Josh",
"username_unclaimed": "noonewouldeverusethi"
},
"Coil": {
"errorMsg": "Whoops, the thing you were looking for isn't here",
"errorType": "message",
Expand Down
14 changes: 0 additions & 14 deletions removed_sites.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1254,20 +1254,6 @@ As of 2021-11-21, 1337x seems to be down causing false positives.
}
```

### Anilist
As of 2021-11-26, Anilist is returning false positives.

```
"Anilist": {
"errorType": "status_code",
"regexCheck": "^[A-Za-z0-9]{2,20}$",
"url": "https://anilist.co/user/{}/",
"urlMain": "https://anilist.co/",
"username_claimed": "Josh",
"username_unclaimed": "noonewouldeverusethi"
}
```

### Coil
As of 2021-11-26, Coil is returning false positives.

Expand Down
20 changes: 18 additions & 2 deletions sherlock/resources/data.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,22 @@
"username_claimed": "blue",
"username_unclaimed": "noonewouldeverusethis7"
},
"Anilist": {
"errorType": "status_code",
"regexCheck": "^[A-Za-z0-9]{2,20}$",
"request_method": "POST",
"request_payload": {
"query": "query($name:String){User(name:$name){id}}",
"variables": {
"name": "{}"
}
},
"url": "https://anilist.co/user/{}/",
"urlProbe": "https://graphql.anilist.co/",
"urlMain": "https://anilist.co/",
"username_claimed": "Josh",
"username_unclaimed": "noonewouldeverusethi"
},
"Apple Developer": {
"errorType": "status_code",
"url": "https://developer.apple.com/forums/profile/{}",
Expand Down Expand Up @@ -1792,7 +1808,7 @@
},
"Vero": {
"errorType": "status_code",
"request_head_only": false,
"request_method": "GET",
"url": "https://vero.co/{}",
"urlMain": "https://vero.co/",
"username_claimed": "blue",
Expand All @@ -1815,7 +1831,7 @@
},
"VirusTotal": {
"errorType": "status_code",
"request_head_only": false,
"request_method": "GET",
"url": "https://www.virustotal.com/ui/users/{}/trusted_users",
"urlMain": "https://www.virustotal.com/",
"urlProbe": "https://www.virustotal.com/ui/users/{}/avatar",
Expand Down
84 changes: 60 additions & 24 deletions sherlock/sherlock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,21 @@ def get_response(request_future, error_type, social_network):
return response, error_context, expection_text


def interpolate_string(object, username):
"""Insert a string into the string properties of an object recursively."""

if isinstance(object, str):
return object.replace("{}", username)
elif isinstance(object, dict):
for key, value in object.items():
object[key] = interpolate_string(value, username)
elif isinstance(object, list):
for i in object:
object[i] = interpolate_string(object[i], username)

return object


def sherlock(username, site_data, query_notify,
tor=False, unique_tor=False,
proxy=None, timeout=None):
Expand Down Expand Up @@ -207,7 +222,7 @@ def sherlock(username, site_data, query_notify,
headers.update(net_info["headers"])

# URL of user on site (if it exists)
url = net_info["url"].format(username)
url = interpolate_string(net_info["url"], username)

# Don't make request if username is invalid for the site
regex_check = net_info.get("regexCheck")
Expand All @@ -225,25 +240,44 @@ def sherlock(username, site_data, query_notify,
# URL of user on site (if it exists)
results_site["url_user"] = url
url_probe = net_info.get("urlProbe")
request_method = net_info.get("request_method")
request_payload = net_info.get("request_payload")
request = None

if request_method is not None:
if request_method == "GET":
request = session.get
elif request_method == "HEAD":
request = session.head
elif request_method == "POST":
request = session.post
elif request_method == "PUT":
request = session.put
else:
raise RuntimeError( f"Unsupported request_method for {url}")

if request_payload is not None:
request_payload = interpolate_string(request_payload, username)

if url_probe is None:
# Probe URL is normal one seen by people out on the web.
url_probe = url
else:
# There is a special URL for probing existence separate
# from where the user profile normally can be found.
url_probe = url_probe.format(username)

if (net_info["errorType"] == 'status_code' and
net_info.get("request_head_only", True) == True):
# In most cases when we are detecting by status code,
# it is not necessary to get the entire body: we can
# detect fine with just the HEAD response.
request_method = session.head
else:
# Either this detect method needs the content associated
# with the GET response, or this specific website will
# not respond properly unless we request the whole page.
request_method = session.get
url_probe = interpolate_string(url_probe, username)

if request is None:
if net_info["errorType"] == 'status_code':
# In most cases when we are detecting by status code,
# it is not necessary to get the entire body: we can
# detect fine with just the HEAD response.
request = session.head
else:
# Either this detect method needs the content associated
# with the GET response, or this specific website will
# not respond properly unless we request the whole page.
request = session.get

if net_info["errorType"] == "response_url":
# Site forwards request to a different URL if username not
Expand All @@ -258,16 +292,18 @@ def sherlock(username, site_data, query_notify,
# This future starts running the request in a new thread, doesn't block the main thread
if proxy is not None:
proxies = {"http": proxy, "https": proxy}
future = request_method(url=url_probe, headers=headers,
proxies=proxies,
allow_redirects=allow_redirects,
timeout=timeout
)
future = request(url=url_probe, headers=headers,
proxies=proxies,
allow_redirects=allow_redirects,
timeout=timeout,
json=request_payload
)
else:
future = request_method(url=url_probe, headers=headers,
allow_redirects=allow_redirects,
timeout=timeout
)
future = request(url=url_probe, headers=headers,
allow_redirects=allow_redirects,
timeout=timeout,
json=request_payload
)

# Store future in data for access later
net_info["request_future"] = future
Expand Down Expand Up @@ -314,7 +350,7 @@ def sherlock(username, site_data, query_notify,
except:
http_status = "?"
try:
response_text = r.text.encode(r.encoding)
response_text = r.text.encode(r.encoding or "UTF-8")
except:
response_text = ""

Expand Down
1 change: 1 addition & 0 deletions sites.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
1. [Airliners](https://www.airliners.net/)
1. [Alik.cz](https://www.alik.cz/)
1. [AllMyLinks](https://allmylinks.com/)
1. [Anilist](https://anilist.co/)
1. [Apple Developer](https://developer.apple.com)
1. [Apple Discussions](https://discussions.apple.com)
1. [Archive.org](https://archive.org)
Expand Down