From cd4c9c09f269e31e556d713bd6a3c20ef91f3872 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Wed, 15 Nov 2023 01:45:49 -0500 Subject: [PATCH] continue and implement .legacy plugin --- deno.jsonc | 7 +- source/engine/components/tests/context.ts | 14 +- .../engine/components/tests/context_test.ts | 90 ------------- source/engine/utils/deno/command_test.ts | 9 +- source/plugins/.legacy/mod.ts | 124 ++++++++++++++++++ source/plugins/.legacy/templates/legacy.ejs | 10 ++ source/plugins/.legacy/tests/list.yml | 32 +++++ source/plugins/webscraping/mod.ts | 1 + source/processors/publish.file/mod.ts | 2 +- source/processors/render.twemojis/mod.ts | 5 +- source/processors/transform.base64/mod.ts | 7 +- .../transform.base64/tests/img.http.ts | 5 + .../tests/img_invalid.http.ts | 5 + .../transform.base64/tests/list.yml | 85 ++++++++++++ 14 files changed, 294 insertions(+), 102 deletions(-) delete mode 100644 source/engine/components/tests/context_test.ts create mode 100644 source/plugins/.legacy/mod.ts create mode 100644 source/plugins/.legacy/templates/legacy.ejs create mode 100644 source/plugins/.legacy/tests/list.yml create mode 100644 source/processors/transform.base64/tests/img.http.ts create mode 100644 source/processors/transform.base64/tests/img_invalid.http.ts create mode 100644 source/processors/transform.base64/tests/list.yml diff --git a/deno.jsonc b/deno.jsonc index b393345402f..c56aa226e5a 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -40,7 +40,7 @@ }, "tasks": { // ' - "qq": "rm -rf .coverage && deno test --coverage=.coverage --no-check --unstable --trace-ops --allow-all source --filter='calendar' && deno coverage .coverage --include='^file:.*/calendar/mod.ts'", + "qq": "rm -rf .coverage && deno test --coverage=.coverage --no-check --unstable --trace-ops --allow-all source --filter='base64' && deno coverage .coverage --include='^file:.*/transform.base64/mod.ts'", "q": "rm -rf .coverage && deno test --coverage=.coverage --unstable --trace-ops --fail-fast --allow-all source && deno coverage .coverage --exclude='/dom/'", "make": "deno run --allow-env --allow-read --allow-write=.deno-make.json --allow-run=deno https://deno.land/x/make@1.2.0/mod.ts $0" @@ -79,6 +79,7 @@ "cdn.jsdelivr.net/gh/jdecked/twemoji@latest/assets" ], "run": [ + "docker", "$CHROME_BIN" ], "read": [ @@ -89,6 +90,7 @@ "metrics.config.yml" ], "write": [ + "$TMP", "$HOME/.config/chromium/SingletonLock" ], "env": true @@ -131,12 +133,14 @@ "cdn.jsdelivr.net/gh/jdecked/twemoji@latest/assets", // Testing "example.com", + "loremflickr.com", // Browser downloads "googlechromelabs.github.io/chrome-for-testing", "edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing" ], "run": [ "deno", + "docker", "$CHROME_BIN" ], "read": [ @@ -146,6 +150,7 @@ "write": [ ".test", ".cache", + "$TMP", "$HOME/.config/chromium/SingletonLock" ], "env": true diff --git a/source/engine/components/tests/context.ts b/source/engine/components/tests/context.ts index 06e6110fb2b..593adff2c15 100644 --- a/source/engine/components/tests/context.ts +++ b/source/engine/components/tests/context.ts @@ -1,3 +1,4 @@ +//Imports import { dir, test } from "@engine/utils/testing.ts" import { deepMerge } from "std/collections/deep_merge.ts" import { Browser } from "@engine/utils/browser.ts" @@ -66,11 +67,15 @@ export async function getPermissions(test: Awaited permission.startsWith("env:")).map((permission) => permission.replace("env:", "")), net: [...requested].filter((permission) => permission.startsWith("net:")).map((permission) => permission.replace("net:", "")), + run: [...requested].filter((permission) => permission.startsWith("run:")).map((permission) => permission.replace("run:", "")).filter((bin) => !["chrome"].includes(bin)), } as test if (requested.has("net:all")) { - delete permissions.net + permissions.net = "inherit" } - if (requested.has("run:chrome")) { + // TODO(@lowlighter): To remove when https://github.com/denoland/deno/issues/21123 fixed + if (permissions.run.length) { + permissions.run = "inherit" + } else if (requested.has("run:chrome")) { Object.assign( permissions, deepMerge(permissions, { @@ -82,7 +87,10 @@ export async function getPermissions(test: Awaited { - Logger.raw = raw - Browser.shareable = shareable - } - return { teardown } -} - -/** Compute required permissions for components testing */ -export async function getPermissions(test: Awaited>[0]) { - // Aggregate permissions from all plugins and processors - const requested = new Set() - const plugins = new Set() - const processors = new Set() - for (const plugin of test.plugins) { - const { id } = sugar(plugin, "plugin") as { id?: string } - if (id) { - plugins.add(id) - } - if (plugin.processors) { - for (const processor of plugin.processors) { - const { id } = sugar(processor, "processor") as { id: string } - processors.add(id) - } - } - } - await Promise.all([...plugins].map((id) => Plugin.load({ id }).then((plugin) => plugin.permissions.forEach((permission) => requested.add(permission))))) - await Promise.all([...processors].map((id) => Processor.load({ id }).then((processor) => processor.permissions.forEach((permission) => requested.add(permission))))) - - // Compute permissions - const permissions = { - read: [dir.source, dir.cache], - env: [...requested].filter((permission) => permission.startsWith("env:")).map((permission) => permission.replace("env:", "")), - net: [...requested].filter((permission) => permission.startsWith("net:")).map((permission) => permission.replace("net:", "")), - } as test - if (requested.has("run:chrome")) { - Object.assign( - permissions, - deepMerge(permissions, { - read: [dir.cache], - net: ["127.0.0.1", "localhost"], - env: ["CHROME_BIN", "CHROME_PATH", "CHROME_EXTRA_FLAGS"], - run: [env.get("CHROME_BIN")], - write: [`${env.get("HOME")}/.config/chromium/SingletonLock`], - }), - ) - } - if (requested.has("write")) { - Object.assign(permissions, deepMerge(permissions, { write: [dir.test] })) - } - - return { permissions } -} - -/** Exports */ -export { Plugin, Processor } diff --git a/source/engine/utils/deno/command_test.ts b/source/engine/utils/deno/command_test.ts index 5c4fc5c9261..45c1caba461 100644 --- a/source/engine/utils/deno/command_test.ts +++ b/source/engine/utils/deno/command_test.ts @@ -6,17 +6,18 @@ import { DevNull } from "@engine/utils/log_test.ts" const stdio = new DevNull() const log = new Logger(import.meta, { level: Logger.channels.trace, tags: { foo: "bar" }, stdio }) -// TODO(@lowlighter): Use `{ permissions: { run: ["deno"] } }` when https://github.com/denoland/deno/issues/21123 fixed +// TODO(@lowlighter): Use `["deno"]` when https://github.com/denoland/deno/issues/21123 fixed +const permissions = { run: "inherit" } as const -Deno.test(t(import.meta, "`command()` can execute commands"), async () => { +Deno.test(t(import.meta, "`command()` can execute commands"), { permissions }, async () => { await expect(command("deno --version")).to.be.eventually.containSubset({ success: true, code: 0 }) }) -Deno.test(t(import.meta, "`command()` returns stdio content instead if asked"), async () => { +Deno.test(t(import.meta, "`command()` returns stdio content instead if asked"), { permissions }, async () => { await expect(command("deno --version", { return: "stdout" })).to.eventually.include("deno") }) -Deno.test(t(import.meta, "`command()` returns stdio content instead if asked"), async () => { +Deno.test(t(import.meta, "`command()` returns stdio content instead if asked"), { permissions }, async () => { stdio.flush() await expect(command("deno --version", { log })).to.be.fulfilled expect(stdio.messages).to.not.be.empty diff --git a/source/plugins/.legacy/mod.ts b/source/plugins/.legacy/mod.ts new file mode 100644 index 00000000000..77e630cf006 --- /dev/null +++ b/source/plugins/.legacy/mod.ts @@ -0,0 +1,124 @@ +// Imports +import { is, parse, Plugin } from "@engine/components/plugin.ts" +import { Logger } from "@engine/utils/log.ts" +import { command } from "@engine/utils/deno/command.ts" +import { throws } from "@engine/utils/errors.ts" +import { read } from "@engine/utils/deno/io.ts" +import { encodeBase64 } from "std/encoding/base64.ts" + +/** Plugin */ +export default class extends Plugin { + /** Import meta */ + static readonly meta = import.meta + + /** Name */ + readonly name = "🏛️ Legacy plugins execution" + + /** Category */ + readonly category = "metrics" + + /** Description */ + readonly description = "Executes plugins using metrics v3 docker image" + + /** Supports */ + readonly supports = ["user", "organization", "repository"] + + /** Inputs */ + readonly inputs = is.object({ + version: is.string().regex(/^v3\.[0-9]{1,2}$/).default("v3.34").describe("Metrics version (v3.x)"), + inputs: is.record(is.unknown()).default(() => ({})).describe("Plugin inputs (as described from respective `action.yml`). Some core options are not supported and will have no effect"), + }) + + /** Outputs */ + readonly outputs = is.object({ + content: is.string().describe("Rendered content (base64 encoded)"), + }) + + /** Permissions */ + readonly permissions = ["run:docker", "write:tmp"] + + /** Action */ + protected async action() { + const { handle } = this.context + const { version, inputs } = await parse(this.inputs, this.context.args) + + // Prepare context + const context = { + fixed: { + use_prebuilt_image: true, + config_base64: true, + output_action: "none", + filename: "metrics.legacy", + config_output: "svg", + // TODO(@lowlighter): set mime type according to config_output + // auto / svg / png / jpeg / json / markdown / markdown-pdf / insights + }, + inherited: { + token: this.context.token.read(), + user: handle?.split("/")[0], + repo: handle?.split("/")[1], + template: this.context.template, + config_timezone: this.context.timezone, + retries: this.context.retries.attempts, + retries_delay: this.context.retries.delay, + plugins_errors_fatal: this.context.fatal, + debug: Logger.channels[this.context.logs] >= Logger.channels.debug, + use_mocked_data: this.context.mock, + github_api_rest: this.context.api, + github_api_graphql: this.context.api, + }, + editable: [ + /^base(?:_|$)/, + /^repositories(?:_|$)/, + /^users_ignored$/, + /^commits_authoring$/, + /^markdown$/, + /^optimize$/, + /^setup_community_templates$/, + /^query$/, + /^extras_(?:css|js)$/, + /^config_(?:order|twemoji|gemoji|octicon|display|animations|padding|presets)$/, + /^delay$/, + /^quota_required_(?:rest|graphql|search)$/, + /^verify$/, + /^debug_flags$/, + /^experimental_features$/, + /^plugin_\w+$/, + ], + inputs: {} as Record, + } + Object.assign(context.inputs, context.fixed, context.inherited) + + // Register user inputs + for (const [key, value] of Object.entries(inputs)) { + if (key in context.fixed) { + this.log.warn(`ignoring ${key}: cannot be overriden in this context`) + continue + } + if ((key in context.inherited) || (context.editable.some((regex) => regex.test(key)))) { + this.log.trace(`registering: ${key}=${value}`) + context.inputs[key] = value + continue + } + this.log.warn(`ignoring ${key}: not supported in this context`) + } + + // Execute docker image + const tmp = await Deno.makeTempDir({ prefix: "metrics_legacy_" }) + try { + const env = Object.fromEntries(Object.entries(context.inputs).map(([key, value]) => [`INPUT_${key.toLocaleUpperCase()}`, `${value ?? ""}`])) + this.log.trace(env) + const { success } = await command(`docker run --rm ${Object.keys(env).map((key) => `--env ${key}`).join(" ")} --volume=${tmp}:/renders ghcr.io/lowlighter/metrics:${version}`, { + log: this.log, + env, + }) + if (!success) { + throws("Failed to execute metrics") + } + this.context.template = "legacy" + return { content: encodeBase64(await read(`${tmp}/metrics.legacy`)) } + } finally { + await Deno.remove(tmp, { recursive: true }) + } + } +} diff --git a/source/plugins/.legacy/templates/legacy.ejs b/source/plugins/.legacy/templates/legacy.ejs new file mode 100644 index 00000000000..505cc62c58d --- /dev/null +++ b/source/plugins/.legacy/templates/legacy.ejs @@ -0,0 +1,10 @@ +
+ <% if (result instanceof Error) { %> +
+ + <%= result.message %> +
+ <% } else { %> + + <% } %> +
diff --git a/source/plugins/.legacy/tests/list.yml b/source/plugins/.legacy/tests/list.yml new file mode 100644 index 00000000000..ad161d0f147 --- /dev/null +++ b/source/plugins/.legacy/tests/list.yml @@ -0,0 +1,32 @@ +- name: supports `version` + plugins: +# - .legacy: +# version: v3.34 +# handle: octocat +# processors: +# - assert: +# html: +# select: .legacy img +# count: 1= + - .legacy: + version: v3.99 + handle: octocat + fatal: false + logs: none + processors: + - assert: + error: /failed to execute metrics/i + +#- name: supports `inputs` +# plugins: +# - .legacy: +# inputs: +# use_prebuilt_image: false +# dryrun: true +# base: header +# handle: octocat +# processors: +# - assert: +# html: +# select: .legacy img +# count: 1= diff --git a/source/plugins/webscraping/mod.ts b/source/plugins/webscraping/mod.ts index 2d8c8da9da0..27bf3ec7da9 100644 --- a/source/plugins/webscraping/mod.ts +++ b/source/plugins/webscraping/mod.ts @@ -51,6 +51,7 @@ export default class extends Plugin { /** Action */ protected async action() { if (this.context.mock) { + this.log.trace("replacing url as mock mode is enabled") this.context.args.url = new URL("tests/example.html", import.meta.url).href } const { url, select: selector, mode, viewport, wait, background } = await parse(this.inputs, this.context.args) diff --git a/source/processors/publish.file/mod.ts b/source/processors/publish.file/mod.ts index c8db0f07617..a3e469adc9c 100644 --- a/source/processors/publish.file/mod.ts +++ b/source/processors/publish.file/mod.ts @@ -27,7 +27,7 @@ export default class extends Processor { readonly supports = ["application/xml", "image/svg+xml", "image/png", "image/jpeg", "image/webp", "application/json", "text/html", "application/pdf", "text/plain"] /** Permissions */ - readonly permissions = ["write"] + readonly permissions = ["write:all"] /** Action */ protected async action(state: state) { diff --git a/source/processors/render.twemojis/mod.ts b/source/processors/render.twemojis/mod.ts index a214d5942f8..abcd4ec2d3e 100644 --- a/source/processors/render.twemojis/mod.ts +++ b/source/processors/render.twemojis/mod.ts @@ -22,12 +22,15 @@ export default class extends Processor { /** Permissions */ readonly permissions = ["net:cdn.jsdelivr.net/gh/jdecked/twemoji@latest/assets"] + /** Does this processor needs to perform requests ? */ + protected requesting = true + /** Action */ protected async action(state: state) { const result = await this.piped(state) const twemojis = new Map(parse(result.content).map(({ text: emoji, url }: { text: string; url: string }) => [emoji, url])) for (const [emoji, url] of twemojis) { - const svg = await fetch(url).then((response) => response.text()) + const svg = await this.requests.fetch(url, { type: "text" }) twemojis.set(emoji, svg.replace(/^ { + return fetch(faker.image.urlLoremFlickr()) +}) diff --git a/source/processors/transform.base64/tests/img_invalid.http.ts b/source/processors/transform.base64/tests/img_invalid.http.ts new file mode 100644 index 00000000000..a9f61accf1c --- /dev/null +++ b/source/processors/transform.base64/tests/img_invalid.http.ts @@ -0,0 +1,5 @@ +import { mock } from "@engine/utils/testing.ts" + +export default mock({}, () => { + return new Response(null) +}) diff --git a/source/processors/transform.base64/tests/list.yml b/source/processors/transform.base64/tests/list.yml new file mode 100644 index 00000000000..91eb455af13 --- /dev/null +++ b/source/processors/transform.base64/tests/list.yml @@ -0,0 +1,85 @@ +- name: transforms image to base64 + plugins: + - processors: + - inject.content: + content: + - transform.base64: + - assert: + html: + select: img + count: 1= + match: data:image/jpeg;base64, + raw: true + +- name: transforms and resize image to base64 + plugins: + - processors: + - inject.content: + content: + - transform.base64: + - assert: + html: + select: img + count: 1= + match: data:image/jpeg;base64, + raw: true + - processors: + - inject.content: + content: + - transform.base64: + - assert: + html: + select: img + count: 1= + match: data:image/jpeg;base64, + raw: true + +- name: ignores data urls + plugins: + - processors: + - inject.content: + content: + - transform.base64: + - assert: + html: + select: img + count: 1= + match: src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mOcOnfpfwAGfgLYttYINwAAAABJRU5ErkJggg==" + raw: true + +- name: ignores non `` tags + plugins: + - processors: + - inject.content: + content: not actually an image + - transform.base64: + select: .img + - assert: + html: + select: .img + count: 1= + match: /not actually an image/i + +- name: fallbacks on invalid urls + plugins: + - processors: + - inject.content: + content: + - transform.base64: + - assert: + html: + select: img + count: 1= + match: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mOcOnfpfwAGfgLYttYINwAAAABJRU5ErkJggg== + raw: true + - processors: + - inject.content: + content: + logs: none + - transform.base64: + - assert: + html: + select: img + count: 1= + match: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mOcOnfpfwAGfgLYttYINwAAAABJRU5ErkJggg== + raw: true \ No newline at end of file