Re-enable experimental support for esbuild (sourcegraph#28856)

client/web/dev/esbuild/build.ts

@@ -0,0 +1,122 @@
+import path from 'path'
+
+import * as esbuild from 'esbuild'
+import signale from 'signale'
+
+import { MONACO_LANGUAGES_AND_FEATURES } from '@sourcegraph/build-config'
+
+import { environmentConfig, ROOT_PATH, STATIC_ASSETS_PATH } from '../utils'
+
+import { manifestPlugin } from './manifestPlugin'
+import { monacoPlugin } from './monacoPlugin'
+import { packageResolutionPlugin } from './packageResolutionPlugin'
+import { stylePlugin } from './stylePlugin'
+import { workerPlugin } from './workerPlugin'
+
+const isEnterpriseBuild = environmentConfig.ENTERPRISE
+
+export const BUILD_OPTIONS: esbuild.BuildOptions = {
+    entryPoints: {
+        // Enterprise vs. OSS builds use different entrypoints. The enterprise entrypoint imports a
+        // strict superset of the OSS entrypoint.
+        'scripts/app': isEnterpriseBuild
+            ? path.join(ROOT_PATH, 'client/web/src/enterprise/main.tsx')
+            : path.join(ROOT_PATH, 'client/web/src/main.tsx'),
+    },
+    bundle: true,
+    format: 'esm',
+    logLevel: 'error',
+    splitting: true,
+    chunkNames: 'chunks/chunk-[name]-[hash]',
+    outdir: STATIC_ASSETS_PATH,
+    plugins: [
+        stylePlugin,
+        workerPlugin,
+        manifestPlugin,
+        packageResolutionPlugin({
+            path: require.resolve('path-browserify'),
+
+            // Needed because imports of rxjs/internal/... actually import a different variant of
+            // rxjs in the same package, which leads to observables from combineLatestOrDefault (and
+            // other places that use rxjs/internal/...) not being cross-compatible. See
+            // https://stackoverflow.com/questions/53758889/rxjs-subscribeto-js-observable-check-works-in-chrome-but-fails-in-chrome-incogn.
+            'rxjs/internal/OuterSubscriber': require.resolve('rxjs/_esm5/internal/OuterSubscriber'),
+            'rxjs/internal/util/subscribeToResult': require.resolve('rxjs/_esm5/internal/util/subscribeToResult'),
+            'rxjs/internal/util/subscribeToArray': require.resolve('rxjs/_esm5/internal/util/subscribeToArray'),
+            'rxjs/internal/Observable': require.resolve('rxjs/_esm5/internal/Observable'),
+        }),
+        monacoPlugin(MONACO_LANGUAGES_AND_FEATURES),
+        {
+            name: 'buildTimer',
+            setup: (build: esbuild.PluginBuild): void => {
+                let buildStarted: number
+                build.onStart(() => {
+                    buildStarted = Date.now()
+                })
+                build.onEnd(() => console.log(`# esbuild: build took ${Date.now() - buildStarted}ms`))
+            },
+        },
+        {
+            name: 'experimentalNotice',
+            setup: (): void => {
+                signale.info(
+                    'esbuild usage is experimental. See https://docs.sourcegraph.com/dev/background-information/web/build#esbuild.'
+                )
+            },
+        },
+    ],
+    define: {
+        ...Object.fromEntries(
+            Object.entries({ ...environmentConfig, SOURCEGRAPH_API_URL: undefined }).map(([key, value]) => [
+                `process.env.${key}`,
+                JSON.stringify(value),
+            ])
+        ),
+        global: 'window',
+    },
+    loader: {
+        '.yaml': 'text',
+        '.ttf': 'file',
+        '.png': 'file',
+    },
+    target: 'es2021',
+    sourcemap: true,
+
+    // TODO(sqs): When https://github.com/evanw/esbuild/pull/1458 is merged (or the issue is
+    // otherwise fixed), we can return to using tree shaking. Right now, esbuild's tree shaking has
+    // a bug where the NavBar CSS is not loaded because the @sourcegraph/wildcard uses `export *
+    // from` and has `"sideEffects": false` in its package.json.
+    ignoreAnnotations: true,
+    treeShaking: false,
+}
+
+// TODO(sqs): These Monaco Web Workers could be built as part of the main build if we switch to
+// using MonacoEnvironment#getWorker (from #getWorkerUrl), which would then let us use the worker
+// plugin (and in Webpack the worker-loader) to load these instead of needing to hardcode them as
+// build entrypoints.
+export const buildMonaco = async (): Promise<void> => {
+    await esbuild.build({
+        entryPoints: {
+            'scripts/editor.worker.bundle': 'monaco-editor/esm/vs/editor/editor.worker.js',
+            'scripts/json.worker.bundle': 'monaco-editor/esm/vs/language/json/json.worker.js',
+        },
+        format: 'iife',
+        target: 'es2021',
+        bundle: true,
+        outdir: STATIC_ASSETS_PATH,
+    })
+}
+
+export const build = async (): Promise<void> => {
+    await esbuild.build({
+        ...BUILD_OPTIONS,
+        outdir: STATIC_ASSETS_PATH,
+    })
+    await buildMonaco()
+}
+
+if (require.main === module) {
+    build()
+        .catch(error => console.error('Error:', error))
+        .finally(() => process.exit(0))
+}

client/web/dev/esbuild/manifestPlugin.ts

@@ -0,0 +1,36 @@
+import fs from 'fs'
+import path from 'path'
+
+import * as esbuild from 'esbuild'
+
+import { STATIC_ASSETS_PATH } from '../utils'
+import { WebpackManifest } from '../webpack/get-html-webpack-plugins'
+
+export const assetPathPrefix = '/.assets'
+
+export const getManifest = (): WebpackManifest => ({
+    'app.js': path.join(assetPathPrefix, 'scripts/app.js'),
+    'app.css': path.join(assetPathPrefix, 'scripts/app.css'),
+    isModule: true,
+})
+
+const writeManifest = async (manifest: WebpackManifest): Promise<void> => {
+    const manifestPath = path.join(STATIC_ASSETS_PATH, 'webpack.manifest.json')
+    await fs.promises.writeFile(manifestPath, JSON.stringify(manifest, null, 2))
+}
+
+/**
+ * An esbuild plugin to write a webpack.manifest.json file (just as Webpack does), for compatibility
+ * with our current Webpack build.
+ */
+export const manifestPlugin: esbuild.Plugin = {
+    name: 'manifest',
+    setup: build => {
+        build.onStart(async () => {
+            // The bug https://github.com/evanw/esbuild/issues/1384 means that onEnd isn't called in
+            // serve mode, so write it here instead of waiting for onEnd. This is OK because we
+            // don't actually need any information that's only available in onEnd.
+            await writeManifest(getManifest())
+        })
+    },
+}

client/web/dev/esbuild/monacoPlugin.ts

@@ -0,0 +1,61 @@
+import path from 'path'
+
+import * as esbuild from 'esbuild'
+import { EditorFeature, featuresArr } from 'monaco-editor-webpack-plugin/out/features'
+import { EditorLanguage, languagesArr } from 'monaco-editor-webpack-plugin/out/languages'
+
+import { MONACO_LANGUAGES_AND_FEATURES } from '@sourcegraph/build-config'
+
+import { ROOT_PATH } from '../utils'
+
+const monacoModulePath = (modulePath: string): string =>
+    require.resolve(path.join('monaco-editor/esm', modulePath), {
+        paths: [path.join(ROOT_PATH, 'node_modules')],
+    })
+
+/**
+ * An esbuild plugin that omits some unneeded features and languages from monaco-editor when
+ * bundling, to reduce bundle size and speed up builds. Similar to
+ * https://github.com/microsoft/monaco-editor-webpack-plugin.
+ */
+export const monacoPlugin = ({
+    languages,
+    features,
+}: Required<typeof MONACO_LANGUAGES_AND_FEATURES>): esbuild.Plugin => ({
+    name: 'monaco',
+    setup: build => {
+        for (const feature of features) {
+            if (feature.startsWith('!')) {
+                throw new Error('negated features (starting with "!") are not supported')
+            }
+        }
+
+        // Some feature exclusions don't work because their module exports a symbol needed by
+        // another feature.
+        const ALWAYS_ENABLED_FEATURES = new Set<EditorFeature>(['snippets'])
+
+        const skipLanguageModules = languagesArr
+            .filter(({ label }) => !languages.includes(label as EditorLanguage))
+            .flatMap(({ entry }) => entry || [])
+        const skipFeatureModules = featuresArr
+            .filter(
+                ({ label }) =>
+                    !features.includes(label as EditorFeature) && !ALWAYS_ENABLED_FEATURES.has(label as EditorFeature)
+            )
+            .flatMap(({ entry }) => entry || [])
+
+        const skipModulePaths = [...skipLanguageModules, ...skipFeatureModules].map(monacoModulePath)
+        const filter = new RegExp(`^(${skipModulePaths.join('|')})$`)
+
+        // For omitted features and languages, treat their modules as empty files.
+        //
+        // TODO(sqs): This is different from how
+        // https://github.com/microsoft/monaco-editor-webpack-plugin does it. The
+        // monaco-editor-webpack-plugin approach relies on injecting a different central module
+        // file, rather than zeroing out each feature/language module. Our approach necessitates the
+        // ALWAYS_ENABLED_FEATURES hack above. Our approach is fine for when esbuild is still an
+        // optional prototype build method for local dev, but this implementation should be fixed if
+        // we switch to esbuild by default.
+        build.onLoad({ filter }, () => ({ contents: '', loader: 'js' }))
+    },
+})

client/web/dev/esbuild/packageResolutionPlugin.ts

@@ -0,0 +1,17 @@
+import * as esbuild from 'esbuild'
+
+/**
+ * An esbuild plugin to redirect imports from one package to another (for example, from 'path' to
+ * 'path-browserify' to run in the browser).
+ */
+export const packageResolutionPlugin = (resolutions: { [fromModule: string]: string }): esbuild.Plugin => ({
+    name: 'packageResolution',
+    setup: build => {
+        const filter = new RegExp(`^(${Object.keys(resolutions).join('|')})$`)
+        build.onResolve({ filter, namespace: 'file' }, args =>
+            (args.kind === 'import-statement' || args.kind === 'require-call') && resolutions[args.path]
+                ? { path: resolutions[args.path] }
+                : undefined
+        )
+    },
+})

client/web/dev/esbuild/server.ts

@@ -0,0 +1,53 @@
+import path from 'path'
+
+import { serve } from 'esbuild'
+import express from 'express'
+import { createProxyMiddleware } from 'http-proxy-middleware'
+import signale from 'signale'
+
+import { STATIC_ASSETS_PATH } from '../utils'
+
+import { buildMonaco, BUILD_OPTIONS } from './build'
+import { assetPathPrefix } from './manifestPlugin'
+
+export const esbuildDevelopmentServer = async (
+    listenAddress: { host: string; port: number },
+    configureProxy: (app: express.Application) => void
+): Promise<void> => {
+    // One-time build (these files only change when the monaco-editor npm package is changed, which
+    // is rare enough to ignore here).
+    await buildMonaco()
+
+    // Start esbuild's server on a random local port.
+    const { host: esbuildHost, port: esbuildPort, wait: esbuildStopped } = await serve(
+        { host: 'localhost', servedir: STATIC_ASSETS_PATH },
+        BUILD_OPTIONS
+    )
+
+    // Start a proxy at :3080. Asset requests (underneath /.assets/) go to esbuild; all other
+    // requests go to the upstream.
+    const proxyApp = express()
+    proxyApp.use(
+        assetPathPrefix,
+        createProxyMiddleware({
+            target: { protocol: 'http:', host: esbuildHost, port: esbuildPort },
+            pathRewrite: { [`^${assetPathPrefix}`]: '' },
+            onProxyRes: (proxyResponse, request) => {
+                // Cache chunks because their filename includes a hash of the content.
+                const isCacheableChunk = path.basename(request.url).startsWith('chunk-')
+                proxyResponse.headers['Cache-Control'] = isCacheableChunk ? 'max-age=3600' : 'no-cache'
+            },
+            logLevel: 'error',
+        })
+    )
+    configureProxy(proxyApp)
+
+    const proxyServer = proxyApp.listen(listenAddress)
+    return await new Promise<void>((resolve, reject) => {
+        proxyServer.once('listening', () => {
+            signale.success('esbuild server is ready')
+            esbuildStopped.finally(() => proxyServer.close(error => (error ? reject(error) : resolve())))
+        })
+        proxyServer.once('error', error => reject(error))
+    })
+}