diff --git a/docs/supportedsites.md b/docs/supportedsites.md index d78dccd6a2..18f66dd354 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -178,7 +178,7 @@ Consider all listed sites to potentially be NSFW. DeviantArt https://www.deviantart.com/ - Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches + Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches OAuth diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 2dc8ebc476..a70710c823 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -846,55 +846,6 @@ def prepare(self, deviation): ) -class DeviantartPopularExtractor(DeviantartExtractor): - """Extractor for popular deviations""" - subcategory = "popular" - directory_fmt = ("{category}", "Popular", - "{popular[range]}", "{popular[search]}") - archive_fmt = "P_{popular[range]}_{popular[search]}_{index}.{extension}" - pattern = (r"(?:https?://)?www\.deviantart\.com/(?:" - r"(?:deviations/?)?\?order=(popular-[^/?#]+)" - r"|((?:[\w-]+/)*)(popular-[^/?#]+)" - r")/?(?:\?([^#]*))?") - example = "https://www.deviantart.com/popular-24-hours/" - - def __init__(self, match): - DeviantartExtractor.__init__(self, match) - self.user = "" - - trange1, path, trange2, query = match.groups() - query = text.parse_query(query) - self.search_term = query.get("q") - - trange = trange1 or trange2 or query.get("order", "") - if trange.startswith("popular-"): - trange = trange[8:] - self.time_range = { - "newest" : "now", - "most-recent" : "now", - "this-week" : "1week", - "this-month" : "1month", - "this-century": "alltime", - "all-time" : "alltime", - }.get(trange, "alltime") - - self.popular = { - "search": self.search_term or "", - "range" : trange or "all-time", - "path" : path.strip("/") if path else "", - } - - def deviations(self): - if self.time_range == "now": - return self.api.browse_newest(self.search_term, self.offset) - return self.api.browse_popular( - self.search_term, self.time_range, self.offset) - - def prepare(self, deviation): - DeviantartExtractor.prepare(self, deviation) - deviation["popular"] = self.popular - - class DeviantartTagExtractor(DeviantartExtractor): """Extractor for deviations from tag searches""" subcategory = "tag" @@ -1095,7 +1046,7 @@ def items(self): class DeviantartOAuthAPI(): """Interface for the DeviantArt OAuth API - Ref: https://www.deviantart.com/developers/http/v1/20160316 + https://www.deviantart.com/developers/http/v1/20160316 """ CLIENT_ID = "5388" CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1" @@ -1188,29 +1139,6 @@ def browse_posts_deviantsyouwatch(self, offset=0): "mature_content": self.mature} return self._pagination(endpoint, params, public=False, unpack=True) - def browse_newest(self, query=None, offset=0): - """Browse newest deviations""" - endpoint = "/browse/newest" - params = { - "q" : query, - "limit" : 120, - "offset" : offset, - "mature_content": self.mature, - } - return self._pagination(endpoint, params) - - def browse_popular(self, query=None, timerange=None, offset=0): - """Yield popular deviations""" - endpoint = "/browse/popular" - params = { - "q" : query, - "limit" : 120, - "timerange" : timerange, - "offset" : offset, - "mature_content": self.mature, - } - return self._pagination(endpoint, params) - def browse_tags(self, tag, offset=0): """ Browse a tag """ endpoint = "/browse/tags" @@ -1223,11 +1151,12 @@ def browse_tags(self, tag, offset=0): return self._pagination(endpoint, params) def browse_user_journals(self, username, offset=0): - """Yield all journal entries of a specific user""" - endpoint = "/browse/user/journals" - params = {"username": username, "offset": offset, "limit": 50, - "mature_content": self.mature, "featured": "false"} - return self._pagination(endpoint, params) + journals = filter( + lambda post: "/journal/" in post["url"], + self.user_profile_posts(username)) + if offset: + journals = util.advance(journals, offset) + return journals def collections(self, username, folder_id, offset=0): """Yield all Deviation-objects contained in a collection folder""" @@ -1339,18 +1268,6 @@ def gallery_folders(self, username, offset=0): "mature_content": self.mature} return self._pagination_list(endpoint, params) - @memcache(keyarg=1) - def user_profile(self, username): - """Get user profile information""" - endpoint = "/user/profile/" + username - return self._call(endpoint, fatal=False) - - def user_statuses(self, username, offset=0): - """Yield status updates of a specific user""" - endpoint = "/user/statuses/" - params = {"username": username, "offset": offset, "limit": 50} - return self._pagination(endpoint, params) - def user_friends(self, username, offset=0): """Get the users list of friends""" endpoint = "/user/friends/" + username @@ -1382,6 +1299,27 @@ def user_friends_unwatch(self, username): endpoint, method="POST", public=False, fatal=False, ).get("success") + @memcache(keyarg=1) + def user_profile(self, username): + """Get user profile information""" + endpoint = "/user/profile/" + username + return self._call(endpoint, fatal=False) + + def user_profile_posts(self, username): + endpoint = "/user/profile/posts" + params = {"username": username, "limit": 50, + "mature_content": self.mature} + return self._pagination(endpoint, params) + + def user_statuses(self, username, offset=0): + """Yield status updates of a specific user""" + statuses = filter( + lambda post: "/status-update/" in post["url"], + self.user_profile_posts(username)) + if offset: + statuses = util.advance(statuses, offset) + return statuses + def authenticate(self, refresh_token_key): """Authenticate the application by requesting an access token""" self.headers["Authorization"] = \ @@ -1470,7 +1408,7 @@ def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs): self.log.error(msg) return data - def _switch_tokens(self, results, params): + def _should_switch_tokens(self, results, params): if len(results) < params["limit"]: return True @@ -1502,7 +1440,7 @@ def _pagination(self, endpoint, params, results = [item["journal"] for item in results if "journal" in item] if extend: - if public and self._switch_tokens(results, params): + if public and self._should_switch_tokens(results, params): if self.refresh_token_key: self.log.debug("Switching to private access token") public = False @@ -1546,6 +1484,11 @@ def _pagination(self, endpoint, params, return params["offset"] = int(params["offset"]) + len(results) + def _pagination_list(self, endpoint, params, key="results"): + result = [] + result.extend(self._pagination(endpoint, params, False, key=key)) + return result + @staticmethod def _shared_content(results): """Return an iterable of shared deviations in 'results'""" @@ -1554,11 +1497,6 @@ def _shared_content(results): if "deviation" in item: yield item["deviation"] - def _pagination_list(self, endpoint, params, key="results"): - result = [] - result.extend(self._pagination(endpoint, params, False, key=key)) - return result - def _metadata(self, deviations): """Add extended metadata to each deviation object""" if len(deviations) <= self.limit: diff --git a/test/results/deviantart.py b/test/results/deviantart.py index 2aefee9460..9552ebd886 100644 --- a/test/results/deviantart.py +++ b/test/results/deviantart.py @@ -48,7 +48,7 @@ "allows_comments" : bool, "author" : { - "type" : "regular", + "type" : "premium", "usericon": str, "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", "username": "shimoda7", @@ -210,7 +210,7 @@ "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f", "author" : { - "type" : "regular", + "type" : "premium", "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4", "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", "username": "shimoda7", @@ -502,7 +502,7 @@ "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", "#category": ("", "deviantart", "journal"), "#class" : deviantart.DeviantartJournalExtractor, - "#sha1_url": "38db2a0d3a587a7e0f9dba7ff7d274610ebefe44", + "#sha1_url": "48aeed5631763d96f5391d2177ea72d9fdbee4e5", }, { @@ -616,30 +616,6 @@ "#sha1_url": "10a336bdee7b9692919461443a7dde44d495818c", }, -{ - "#url" : "https://www.deviantart.com/?order=popular-all-time", - "#category": ("", "deviantart", "popular"), - "#class" : deviantart.DeviantartPopularExtractor, - "#options" : {"original": False}, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://www.deviantart.com/popular-24-hours/?q=tree+house", - "#category": ("", "deviantart", "popular"), - "#class" : deviantart.DeviantartPopularExtractor, - "#options" : {"original": False}, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://www.deviantart.com/artisan/popular-all-time/?q=tree", - "#category": ("", "deviantart", "popular"), - "#class" : deviantart.DeviantartPopularExtractor, -}, - { "#url" : "https://www.deviantart.com/tag/nature", "#category": ("", "deviantart", "tag"), @@ -811,7 +787,7 @@ "#category": ("", "deviantart", "deviation"), "#class" : deviantart.DeviantartDeviationExtractor, "#pattern" : """text:\n""", - "#sha1_url": "d34b2c9f873423e665a1b8ced20fcb75951694a3", + "#sha1_url": "37302947642d1e53392ef8ee9b3f473a3c578e7c", }, { @@ -820,7 +796,7 @@ "#category": ("", "deviantart", "deviation"), "#class" : deviantart.DeviantartDeviationExtractor, "#pattern" : """text:\n""", - "#sha1_url": "e2e0044bd255304412179b6118536dbd9bb3bb0e", + "#sha1_url": "8ca1dc8df53d3707c778d08a604f9ad9ddba7469", }, { @@ -829,6 +805,7 @@ "#category": ("", "deviantart", "deviation"), "#class" : deviantart.DeviantartDeviationExtractor, "#count" : 0, + "#exception": exception.NotFoundError, }, { @@ -866,7 +843,7 @@ "#url" : "https://www.deviantart.com/view/706871727", "#category": ("", "deviantart", "deviation"), "#class" : deviantart.DeviantartDeviationExtractor, - "#sha1_content": "87dff6056fc9a2bf77f75317a1e00e18451b3c80", + "#sha1_content": "4d013515e72dec1e3977c82fd71ce4b15b8bd856", }, {