Skip to content

Commit

Permalink
feat: use Workers for manifest fetching & optimizations (#572)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kyrie25
kyrie25 committed Aug 7, 2023
1 parent a56ea3a commit ada537c
Show file tree
Hide file tree
Showing 7 changed files with 1,147 additions and 591 deletions.
3 changes: 1 addition & 2 deletions src/components/Card/Card.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import {
parseCSS,
injectUserCSS,
generateKey,
getAvailableTLD,
} from "../../logic/Utils";
import TrashIcon from "../Icons/TrashIcon";
import DownloadIcon from "../Icons/DownloadIcon";
Expand Down Expand Up @@ -379,7 +378,7 @@ class Card extends React.Component<CardProps, {
*/
async fetchAndInjectUserCSS(theme) {
try {
const tld = await getAvailableTLD();
const tld = window.sessionStorage.getItem("marketplace-request-tld") || undefined;
const userCSS = theme
? await parseCSS(this.props.item as CardItem, tld)
: undefined;
Expand Down
1 change: 1 addition & 0 deletions src/components/Grid.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,7 @@ class Grid extends React.Component<
// Checks for new Marketplace updates
fetch(LATEST_RELEASE).then(res => res.json()).then(
result => {
if (result.message) throw result;
this.setState({
version: result[0].name,
});
Expand Down
67 changes: 40 additions & 27 deletions src/extensions/extension.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ import {
parseCSS,
// TODO: there's a slightly different copy of this function in Card.ts?
injectUserCSS,
addToSessionStorage,
sleep,
addExtensionToSpicetifyConfig,
initAlbumArtBasedColor,
getAvailableTLD,
Expand All @@ -26,15 +24,14 @@ import {
getBlacklist,
fetchThemeManifest,
fetchExtensionManifest,
fetchAppManifest,
} from "../logic/FetchRemotes";

(async () => {
(async function init() {
while (!(Spicetify?.LocalStorage && Spicetify?.showNotification)) {
await new Promise(resolve => setTimeout(resolve, 100));
await new Promise(resolve => setTimeout(resolve, 10));
}

const tld = await getAvailableTLD();

// https://github.com/satya164/react-simple-code-editor/issues/86
const reactSimpleCodeEditorFix = document.createElement("script");
reactSimpleCodeEditorFix.innerHTML = "const global = globalThis;";
Expand All @@ -52,6 +49,8 @@ import {
version: MARKETPLACE_VERSION,
};

const tld = await getAvailableTLD();

const initializeExtension = (extensionKey: string) => {
const extensionManifest = getLocalStorageDataFromKey(extensionKey);
// Abort if no manifest found or no extension URL (i.e. a theme)
Expand Down Expand Up @@ -146,16 +145,30 @@ import {

console.log("Loaded Marketplace extension");

const installedSnippetKeys = getLocalStorageDataFromKey(LOCALSTORAGE_KEYS.installedSnippets, []);
const installedSnippets = installedSnippetKeys.map((key) => getLocalStorageDataFromKey(key));
initializeSnippets(installedSnippets);

if (!tld) {
if (window.navigator.onLine) {
console.error(new Error("Unable to connect to the CDN, please check your Internet configuration."));
Spicetify.showNotification("Marketplace is unable to connect to the CDN. Please check your Internet configuration.", true, 5000);
} else {
// Reload Marketplace extension in case the user couldn't connect to the CDN because they were offline
window.addEventListener("online", init, { once: true });
}

return;
}

window.sessionStorage.setItem("marketplace-request-tld", tld);

// Save to Spicetify.Config for use when removing a theme
Spicetify.Config.local_theme = Spicetify.Config.current_theme;
Spicetify.Config.local_color_scheme = Spicetify.Config.color_scheme;
const installedThemeKey = localStorage.getItem(LOCALSTORAGE_KEYS.themeInstalled);
if (installedThemeKey) initializeTheme(installedThemeKey);

const installedSnippetKeys = getLocalStorageDataFromKey(LOCALSTORAGE_KEYS.installedSnippets, []);
const installedSnippets = installedSnippetKeys.map((key) => getLocalStorageDataFromKey(key));
initializeSnippets(installedSnippets);

const installedExtensions = getLocalStorageDataFromKey(LOCALSTORAGE_KEYS.installedExtensions, []);
installedExtensions.forEach((extensionKey) => initializeExtension(extensionKey));
})();
Expand All @@ -169,16 +182,20 @@ import {
async function queryRepos(type: RepoType, pageNum = 1) {
const BLACKLIST = window.sessionStorage.getItem("marketplace:blacklist");

let url = `https://api.github.com/search/repositories?per_page=${ITEMS_PER_REQUEST}`;
if (type === "extension") url += `&q=${encodeURIComponent("topic:spicetify-extensions")}`;
else if (type === "theme") url += `&q=${encodeURIComponent("topic:spicetify-themes")}`;
let url = `https://api.github.com/search/repositories?per_page=${ITEMS_PER_REQUEST}&q=${encodeURIComponent(`topic:spicetify-${type}s`)}`;
if (pageNum) url += `&page=${pageNum}`;

const allRepos = await fetch(url).then(res => res.json()).catch(() => []);
if (!allRepos.items) {
Spicetify.showNotification("Too Many Requests, Cool Down.", true);
const allRepos = JSON.parse(window.sessionStorage.getItem(`spicetify-${type}s-page-${pageNum}`) || "null") || await fetch(url)
.then(res => res.json())
.catch(() => null);

if (!allRepos?.items) {
Spicetify.showNotification?.("Too Many Requests, Cool Down.", true);
return { items: [] };
}

window.sessionStorage.setItem(`spicetify-${type}s-page-${pageNum}`, JSON.stringify(allRepos));

const filteredResults = {
...allRepos,
page_count: allRepos.items.length,
Expand All @@ -199,7 +216,7 @@ async function loadPageRecursive(type: RepoType, pageNum: number) {
appendInformationToLocalStorage(pageOfRepos, type);

// Sets the amount of items that have thus been fetched
const soFarResults = ITEMS_PER_REQUEST * (pageNum - 1) + pageOfRepos.page_count;
const soFarResults = ITEMS_PER_REQUEST * pageNum + pageOfRepos.page_count;
console.debug({ pageOfRepos });
const remainingResults = pageOfRepos.total_count - soFarResults;

Expand All @@ -221,8 +238,9 @@ async function loadPageRecursive(type: RepoType, pageNum: number) {
// Begin by getting the themes and extensions from github
// const [extensionReposArray, themeReposArray] = await Promise.all([
await Promise.all([
loadPageRecursive("extension", 1),
loadPageRecursive("theme", 1),
loadPageRecursive("extension", 0),
loadPageRecursive("theme", 0),
loadPageRecursive("app", 0),
]);

// let extensionsNextPage = 1;
Expand All @@ -241,13 +259,8 @@ async function loadPageRecursive(type: RepoType, pageNum: number) {
async function appendInformationToLocalStorage(array, type: RepoType) {
// This system should make it so themes and extensions are stored concurrently
for (const repo of array.items) {
// console.log(repo);
const data = (type === "theme")
? await fetchThemeManifest(repo.contents_url, repo.default_branch, repo.stargazers_count)
: await fetchExtensionManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
if (data) {
addToSessionStorage(data);
await sleep(5000);
}
if (type === "theme") await fetchThemeManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
else if (type === "extension") await fetchExtensionManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
else if (type === "app") await fetchAppManifest(repo.contents_url, repo.default_branch, repo.stargazers_count);
}
}
85 changes: 51 additions & 34 deletions src/logic/FetchRemotes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,17 @@ export async function getTaggedRepos(tag: RepoTopic, page = 1, BLACKLIST:string[
// Sorting params (not implemented for Marketplace yet)
// if (sortConfig.by.match(/top|controversial/) && sortConfig.time) {
// url += `&t=${sortConfig.time}`
const allRepos = await fetch(url).then(res => res.json()).catch(() => []);
if (!allRepos.items) {
const allRepos = JSON.parse(window.sessionStorage.getItem(`${tag}-page-${page}`) || "null") || await fetch(url)
.then(res => res.json())
.catch(() => null);

if (!allRepos?.items) {
Spicetify.showNotification("Too Many Requests, Cool Down.", true);
return;
return { items: [] };
}

window.sessionStorage.setItem(`${tag}-page-${page}`, JSON.stringify(allRepos));

const filteredResults = {
...allRepos,
// Include count of all items on the page, since we're filtering the blacklist below,
Expand All @@ -41,6 +47,32 @@ export async function getTaggedRepos(tag: RepoTopic, page = 1, BLACKLIST:string[
return filteredResults;
}

// Workaround for not spamming console with 404s
const script = `
self.addEventListener('message', async (event) => {
const url = event.data;
const response = await fetch(url);
const data = await response.json().catch(() => null);
self.postMessage(data);
});
`;
const blob = new Blob([script], { type: "application/javascript" });
const workerURL = URL.createObjectURL(blob);

async function fetchRepoManifest(url: string) {
const worker = new Worker(workerURL);
return new Promise((resolver) => {
const resolve = (data) => {
worker.terminate();
resolver(data);
};

worker.postMessage(url);
worker.addEventListener("message", (event) => resolve(event.data), { once: true });
worker.addEventListener("error", () => resolve(null), { once: true });
});
}

// TODO: add try/catch here?
// TODO: can we add a return type here?
/**
Expand All @@ -51,17 +83,20 @@ export async function getTaggedRepos(tag: RepoTopic, page = 1, BLACKLIST:string[
* @returns The manifest object
*/
async function getRepoManifest(user: string, repo: string, branch: string) {
const sessionStorageItem = window.sessionStorage.getItem(`${user}-${repo}`);
const failedSessionStorageItems = window.sessionStorage.getItem("noManifests");
const key = `${user}-${repo}`;
const sessionStorageItem = window.sessionStorage.getItem(key);
const failedSessionStorageItems = JSON.parse(window.sessionStorage.getItem("noManifests") || "[]");
if (sessionStorageItem) return JSON.parse(sessionStorageItem);

const url = `https://raw.githubusercontent.com/${user}/${repo}/${branch}/manifest.json`;
if (failedSessionStorageItems?.includes(url)) return null;
if (failedSessionStorageItems.includes(url)) return null;

let manifest = await fetchRepoManifest(url);

if (!manifest) return addToSessionStorage([url], "noManifests");
if (!Array.isArray(manifest)) manifest = [manifest];

const manifest = await fetch(url).then(res => res.json()).catch(
() => addToSessionStorage([url], "noManifests"),
);
if (manifest) window.sessionStorage.setItem(`${user}-${repo}`, JSON.stringify(manifest));
addToSessionStorage(manifest, key);

return manifest;
}
Expand All @@ -77,16 +112,12 @@ async function getRepoManifest(user: string, repo: string, branch: string) {
export async function fetchExtensionManifest(contents_url: string, branch: string, stars: number, hideInstalled = false) {
try {
// TODO: use the original search full_name ("theRealPadster/spicetify-hide-podcasts") or something to get the url better?
let manifests;
const regex_result = contents_url.match(/https:\/\/api\.github\.com\/repos\/(?<user>.+)\/(?<repo>.+)\/contents/);
// TODO: err handling?
if (!regex_result || !regex_result.groups) return null;
const { user, repo } = regex_result.groups;

manifests = await getRepoManifest(user, repo, branch);

// If the manifest returned is not an array, initialize it as one
if (!Array.isArray(manifests)) manifests = [manifests];
const manifests = await getRepoManifest(user, repo, branch);

// Manifest is initially parsed
const parsedManifests: CardItem[] = manifests.reduce((accum, manifest) => {
Expand Down Expand Up @@ -129,9 +160,7 @@ export async function fetchExtensionManifest(contents_url: string, branch: strin
}, []);

return parsedManifests;
}
catch (err) {
// console.warn(contents_url, err);
} catch {
return null;
}
}
Expand All @@ -146,16 +175,12 @@ export async function fetchExtensionManifest(contents_url: string, branch: strin
*/
export async function fetchThemeManifest(contents_url: string, branch: string, stars: number) {
try {
let manifests;
const regex_result = contents_url.match(/https:\/\/api\.github\.com\/repos\/(?<user>.+)\/(?<repo>.+)\/contents/);
// TODO: err handling?
if (!regex_result || !regex_result.groups) return null;
const { user, repo } = regex_result.groups;

manifests = await getRepoManifest(user, repo, branch);

// If the manifest returned is not an array, initialize it as one
if (!Array.isArray(manifests)) manifests = [manifests];
const manifests = await getRepoManifest(user, repo, branch);

// Manifest is initially parsed
// const parsedManifests: ThemeCardItem[] = manifests.reduce((accum, manifest) => {
Expand Down Expand Up @@ -196,9 +221,7 @@ export async function fetchThemeManifest(contents_url: string, branch: string, s
return accum;
}, []);
return parsedManifests;
}
catch (err) {
// console.warn(contents_url, err);
} catch {
return null;
}
}
Expand All @@ -213,16 +236,12 @@ export async function fetchThemeManifest(contents_url: string, branch: string, s
export async function fetchAppManifest(contents_url: string, branch: string, stars: number) {
try {
// TODO: use the original search full_name ("theRealPadster/spicetify-hide-podcasts") or something to get the url better?
let manifests;
const regex_result = contents_url.match(/https:\/\/api\.github\.com\/repos\/(?<user>.+)\/(?<repo>.+)\/contents/);
// TODO: err handling?
if (!regex_result || !regex_result.groups) return null;
const { user, repo } = regex_result.groups;

manifests = await getRepoManifest(user, repo, branch);

// If the manifest returned is not an array, initialize it as one
if (!Array.isArray(manifests)) manifests = [manifests];
const manifests = await getRepoManifest(user, repo, branch);

// Manifest is initially parsed
const parsedManifests: CardItem[] = manifests.reduce((accum, manifest) => {
Expand Down Expand Up @@ -263,9 +282,7 @@ export async function fetchAppManifest(contents_url: string, branch: string, sta
}, []);

return parsedManifests;
}
catch (err) {
// console.warn(contents_url, err);
} catch {
return null;
}
}
Expand Down
27 changes: 17 additions & 10 deletions src/logic/Utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,9 +417,11 @@ export const initAlbumArtBasedColor = (scheme: ColourScheme) => {
});
};

export const parseCSS = async (themeData: CardItem, tld = "net") => {
export const parseCSS = async (themeData: CardItem, tld?: string) => {
if (!themeData.cssURL) throw new Error("No CSS URL provided");

tld ||= await getAvailableTLD();

const userCssUrl = isGithubRawUrl(themeData.cssURL)
// TODO: this should probably be the URL stored in localstorage actually (i.e. put this url in localstorage)
? `https://cdn.jsdelivr.${tld}/gh/${themeData.user}/${themeData.repo}@${themeData.branch}/${themeData.manifest.usercss}`
Expand Down Expand Up @@ -479,12 +481,13 @@ export const getParamsFromGithubRaw = (url: string) => {
export function addToSessionStorage(items, key?) {
if (!items) return;
items.forEach((item) => {
if (!key) key = `${items.user}-${items.repo}`;
const itemKey = key || `${item.user}-${item.repo}`;

// If the key already exists, it will append to it instead of overwriting it
const existing = window.sessionStorage.getItem(key);
const existing = window.sessionStorage.getItem(itemKey);
const parsed = existing ? JSON.parse(existing) : [];
parsed.push(item);
window.sessionStorage.setItem(key, JSON.stringify(parsed));
window.sessionStorage.setItem(itemKey, JSON.stringify(parsed));
});
}
export function getInvalidCSS(): string[] {
Expand Down Expand Up @@ -581,11 +584,15 @@ export const addExtensionToSpicetifyConfig = (main?: string) => {

// Make a ping to the jsdelivr CDN to check if the user has an internet connection
export async function getAvailableTLD() {
try {
const response = await fetch("https://cdn.jsdelivr.net", { redirect: "manual" });
if (response.type === "opaqueredirect") return "net";
return "xyz";
} catch (err) {
return "xyz";
const tlds = ["net", "xyz"];

for (const tld of tlds) {
try {
const response = await fetch(`https://cdn.jsdelivr.${tld}`, { redirect: "manual", cache: "no-cache" });
if (response.type === "opaqueredirect") return tld;
} catch (err) {
console.error(err);
continue;
}
}
}
8 changes: 2 additions & 6 deletions src/styles/components/_grid.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,8 @@
}

.marketplace-header__left {
position: fixed;
left: 16px;

@media (min-width: 1024px) {
left: 32px;
}
position: absolute;
left: 0;
}

.marketplace-grid {
Expand Down
Loading

0 comments on commit ada537c

Please sign in to comment.