From fa70ff784db0fd75065fc78ee95a87a613201db6 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Sat, 14 Aug 2021 09:31:21 +0200 Subject: [PATCH] feat: build esm modules using ipjs (#865) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This builds up on @achingbrain 's work on https://github.com/ipfs/aegir/pull/863 with build improvements and full support This adds: - `ipjs` for ESM modules - auto-detected - `types` property transformation, allowing real time ts check in dev and path update for dist folder (TLDR no dist in the path) - `release` for ESM modules will navigate to the dist to publish its content - Dockerfile to bundlesize action per https://github.com/actions/runner/issues/772#issuecomment-883419149 as we need node14+ for ESM One of the problematic modules in skypack using aegir is `uint8arrays`. It is a CJS module that depends on a ESM first module (multiformats), which makes skypack to get bad dependency paths. I tested this out shipping `uint8arrays` https://github.com/achingbrain/uint8arrays/pull/22 PR and everything working smoothly 🎉 Original release: https://codepen.io/vascosantos/pen/KKmXoPV?editors=0011 Scoped release using `aegir`: https://codepen.io/vascosantos/pen/bGWoONq?editors=0011 (see browser built in console for errors) Co-authored-by: achingbrain --- .gitignore | 3 +- actions/bundle-size/.dockerignore | 2 + actions/bundle-size/Dockerfile | 6 +++ actions/bundle-size/action.yml | 8 +++- md/esm.md | 27 ++++++++++++++ package.json | 5 ++- src/build/index.js | 33 +++++++++++++++++ src/cmds/build.js | 12 ++++++ src/config/user.js | 4 +- src/release/publish.js | 24 +++++++----- src/types.d.ts | 15 ++++++++ test/build.js | 37 +++++++++++++++++++ .../node_modules/a-cjs-dep/package.json | 12 ++++++ .../node_modules/a-cjs-dep/src/index.d.ts | 2 + .../node_modules/a-cjs-dep/src/index.js | 4 ++ .../node_modules/an-esm-dep/package.json | 18 +++++++++ .../node_modules/an-esm-dep/src/index.cjs | 4 ++ .../node_modules/an-esm-dep/src/index.d.ts | 2 + .../node_modules/an-esm-dep/src/index.js | 4 ++ test/fixtures/esm/an-esm-project/package.json | 18 +++++++++ test/fixtures/esm/an-esm-project/src/index.js | 10 +++++ test/node.js | 1 + 22 files changed, 236 insertions(+), 15 deletions(-) create mode 100644 actions/bundle-size/.dockerignore create mode 100644 actions/bundle-size/Dockerfile create mode 100644 md/esm.md create mode 100644 test/build.js create mode 100644 test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/package.json create mode 100644 test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/src/index.d.ts create mode 100644 test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/src/index.js create mode 100644 test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/package.json create mode 100644 test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.cjs create mode 100644 test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.d.ts create mode 100644 test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.js create mode 100644 test/fixtures/esm/an-esm-project/package.json create mode 100644 test/fixtures/esm/an-esm-project/src/index.js diff --git a/.gitignore b/.gitignore index 0c4e49dfa..ea695cd71 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ package-lock.json yarn.lock -node_modules +/node_modules +/actions/bundle-size/node_modules /coverage /dist /docs diff --git a/actions/bundle-size/.dockerignore b/actions/bundle-size/.dockerignore new file mode 100644 index 000000000..d1e795834 --- /dev/null +++ b/actions/bundle-size/.dockerignore @@ -0,0 +1,2 @@ +** +!/dist \ No newline at end of file diff --git a/actions/bundle-size/Dockerfile b/actions/bundle-size/Dockerfile new file mode 100644 index 000000000..fa7b8fb83 --- /dev/null +++ b/actions/bundle-size/Dockerfile @@ -0,0 +1,6 @@ +# Just enough docker until github gets a new node16 runner +# see: https://github.com/actions/runner/issues/772 +FROM node:16-alpine +WORKDIR /usr/src/app +COPY dist/index.js . +CMD [ "node", "index.js" ] \ No newline at end of file diff --git a/actions/bundle-size/action.yml b/actions/bundle-size/action.yml index 47f8a23a7..f22578406 100644 --- a/actions/bundle-size/action.yml +++ b/actions/bundle-size/action.yml @@ -8,5 +8,9 @@ inputs: description: A directory to run the bundle check in required: false runs: - using: 'node12' - main: 'dist/index.js' + # TODO: we need node14.14 minimum. + # https://github.com/actions/runner/issues/772 + # using: 'node12' + # main: 'dist/index.js' + using: 'docker' + image: 'Dockerfile' diff --git a/md/esm.md b/md/esm.md new file mode 100644 index 000000000..bd030cc2d --- /dev/null +++ b/md/esm.md @@ -0,0 +1,27 @@ +# ESM support + +## Setup + +`aegir` leverages [ipjs](https://github.com/mikeal/ipjs) to output a build with `cjs` and `esm` for maximum compatibility. The general guidelines for writing a module in `esm` are detailed on the `ipjs` README. `aegir` will automatically identify a `esm` repo by the `module` property in `package.json`. + +## Electron testing + +Electron does [not support ESM](https://github.com/electron/electron/issues/21457) at the time of writing. When writing a module using ESM, we need to compile the tests to `cjs` and rely on them. For generating the build including the tests: + +```bash +aegir build --esm-tests +``` + +## Lerna Monorepo + +When using a lerna monorepo, local dependencies are symlinked by lerna on install. This means that an `esm` module will not use the resulting `dist` folder as symlink. This can become a problem if we are testing the `cjs` build of a module. + +To work around the above problem, we can use `publishConfig.directory = "dist"` in `package.json` to notice lerna about the symlink path. After running the `aegir build` command, it is necessary to run `lerna link` to update the symlinks. + +## Release + +When releasing an `esm` module, the published content will be the generated `dist` folder content, as indicated by [ipjs](https://github.com/mikeal/ipjs). + +## Examples + +TODO: List examples when merged (`ipfs-unixfs`, `uint8arrays`) \ No newline at end of file diff --git a/package.json b/package.json index 5909f574d..2f7220434 100644 --- a/package.json +++ b/package.json @@ -87,13 +87,14 @@ "esbuild-register": "^2.3.0", "eslint": "^7.23.0", "eslint-config-ipfs": "^2.0.0", - "execa": "^5.0.0", + "execa": "^5.1.1", "extract-zip": "^2.0.1", "fs-extra": "^10.0.0", "gh-pages": "^3.1.0", "git-authors-cli": "^1.0.33", "globby": "^11.0.3", "ipfs-utils": "^8.1.0", + "ipjs": "^5.0.5", "it-glob": "~0.0.10", "kleur": "^4.1.4", "lilconfig": "^2.0.2", @@ -119,7 +120,7 @@ "typedoc": "^0.21.2", "typescript": "^4.3.5", "update-notifier": "^5.0.0", - "yargs": "^17.0.1" + "yargs": "^17.1.1" }, "devDependencies": { "@types/bytes": "^3.1.0", diff --git a/src/build/index.js b/src/build/index.js index 955db16cd..a614f513b 100644 --- a/src/build/index.js +++ b/src/build/index.js @@ -56,11 +56,44 @@ const build = async (argv) => { return outfile } +/** + * Build command + * + * @param {GlobalOptions & BuildOptions} argv + */ +const buildEsm = async (argv) => { + const dist = path.join(process.cwd(), 'dist') + // @ts-ignore no types + const ipjs = await import('ipjs') + + await ipjs.default({ + dist, + onConsole: (/** @type {any[]} */...args) => console.info.apply(console, args), + cwd: process.cwd(), + main: argv.esmMain, + tests: argv.esmTests + }) +} + const tasks = new Listr([ { title: 'Clean ./dist', task: async () => del(path.join(process.cwd(), 'dist')) }, + { + title: 'Build ESM', + enabled: ctx => { + return pkg.type === 'module' + }, + /** + * + * @param {GlobalOptions & BuildOptions} ctx + * @param {Task} task + */ + task: async (ctx, task) => { + await buildEsm(ctx) + } + }, { title: 'Bundle', enabled: ctx => ctx.bundle, diff --git a/src/cmds/build.js b/src/cmds/build.js index 54b4e608b..c8ad1f429 100644 --- a/src/cmds/build.js +++ b/src/cmds/build.js @@ -37,6 +37,18 @@ module.exports = { type: 'boolean', describe: 'Build the Typescripts type declarations.', default: userConfig.build.types + }, + esmMain: { + alias: 'esm-main', + type: 'boolean', + describe: 'Include a main field in a built esm project', + default: userConfig.build.esmMain + }, + esmTests: { + alias: 'esm-tests', + type: 'boolean', + describe: 'Include tests in a built esm project', + default: userConfig.build.esmTests } }) }, diff --git a/src/config/user.js b/src/config/user.js index 0128c68e7..c07736154 100644 --- a/src/config/user.js +++ b/src/config/user.js @@ -40,7 +40,9 @@ const defaults = { bundlesize: false, bundlesizeMax: '100kB', types: true, - config: {} + config: {}, + esmMain: true, + esmTests: false }, // linter cmd options lint: { diff --git a/src/release/publish.js b/src/release/publish.js index 90f22198b..a511abf26 100644 --- a/src/release/publish.js +++ b/src/release/publish.js @@ -1,7 +1,7 @@ 'use strict' const execa = require('execa') -const { otp } = require('../utils') +const { otp, pkg, repoDirectory } = require('../utils') /** * @typedef {import('./../types').ReleaseOptions} ReleaseOptions * @typedef {import('listr').ListrTaskWrapper} ListrTask @@ -25,16 +25,22 @@ function publish (ctx, task) { task.title += ` (npm ${publishArgs.join(' ')})` } - return execa('npm', publishArgs) - .catch(async (error) => { - if (error.toString().includes('provide a one-time password')) { - const code = await otp() - task.title += '. Trying again with OTP.' - return await execa('npm', publishArgs.concat('--otp', code)) + // Publish from dist if ESM + const execaOptions = pkg.type === 'module' + ? { + cwd: `${repoDirectory}/dist` } + : {} - throw error - }) + return execa('npm', publishArgs, execaOptions).catch(async (error) => { + if (error.toString().includes('provide a one-time password')) { + const code = await otp() + task.title += '. Trying again with OTP.' + return await execa('npm', publishArgs.concat('--otp', code), execaOptions) + } + + throw error + }) } module.exports = publish diff --git a/src/types.d.ts b/src/types.d.ts index 14c0c4e55..9945f6727 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -99,6 +99,13 @@ interface GlobalOptions { fileConfig: Options } +interface ESMHooks { + onParse: () => {} + onParsed: () => {} + onDeflateStart: () => {} + onDeflateEnd: () => {} +} + interface BuildOptions { /** * Build the JS standalone bundle. @@ -120,6 +127,14 @@ interface BuildOptions { * esbuild build options */ config: esbuild.BuildOptions + /** + * Include tests in the ipjs output directory + */ + esmTests: boolean + /** + * Include a main field in the ipjs output package.json + */ + esmMain: boolean } interface TSOptions { diff --git a/test/build.js b/test/build.js new file mode 100644 index 000000000..7962f74e5 --- /dev/null +++ b/test/build.js @@ -0,0 +1,37 @@ +/* eslint-env mocha */ +'use strict' + +const { expect } = require('../utils/chai') +const execa = require('execa') +const { copy, existsSync } = require('fs-extra') +const { join } = require('path') +const bin = require.resolve('../') +const tempy = require('tempy') + +describe('build', () => { + describe('esm', () => { + let projectDir = '' + + before(async () => { + projectDir = tempy.directory() + + await copy(join(__dirname, 'fixtures', 'esm', 'an-esm-project'), projectDir) + }) + + it('should build an esm project', async function () { + this.timeout(20 * 1000) // slow ci is slow + + await execa(bin, ['build'], { + cwd: projectDir + }) + + expect(existsSync(join(projectDir, 'dist', 'esm'))).to.be.true() + expect(existsSync(join(projectDir, 'dist', 'cjs'))).to.be.true() + + const module = require(join(projectDir, 'dist')) + + expect(module).to.have.property('useHerp').that.is.a('function') + expect(module).to.have.property('useDerp').that.is.a('function') + }) + }) +}) diff --git a/test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/package.json b/test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/package.json new file mode 100644 index 000000000..e86e01db8 --- /dev/null +++ b/test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/package.json @@ -0,0 +1,12 @@ +{ + "name": "a-cjs-dep", + "version": "1.0.0", + "description": "", + "main": "src/index.js", + "types": "src", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC" +} diff --git a/test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/src/index.d.ts b/test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/src/index.d.ts new file mode 100644 index 000000000..41469fac9 --- /dev/null +++ b/test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/src/index.d.ts @@ -0,0 +1,2 @@ + +export default function herp(): void diff --git a/test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/src/index.js b/test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/src/index.js new file mode 100644 index 000000000..645524c54 --- /dev/null +++ b/test/fixtures/esm/an-esm-project/node_modules/a-cjs-dep/src/index.js @@ -0,0 +1,4 @@ + +module.exports = () => { + +} diff --git a/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/package.json b/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/package.json new file mode 100644 index 000000000..3c14ad382 --- /dev/null +++ b/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/package.json @@ -0,0 +1,18 @@ +{ + "name": "an-esm-dep", + "version": "1.0.0", + "description": "", + "type": "module", + "types": "src", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "exports": { + ".": { + "import": "./src/index.js", + "require": "./src/index.cjs" + } + } +} diff --git a/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.cjs b/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.cjs new file mode 100644 index 000000000..645524c54 --- /dev/null +++ b/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.cjs @@ -0,0 +1,4 @@ + +module.exports = () => { + +} diff --git a/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.d.ts b/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.d.ts new file mode 100644 index 000000000..d7dcc730d --- /dev/null +++ b/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.d.ts @@ -0,0 +1,2 @@ + +export default function derp(): void diff --git a/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.js b/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.js new file mode 100644 index 000000000..d913e1a52 --- /dev/null +++ b/test/fixtures/esm/an-esm-project/node_modules/an-esm-dep/src/index.js @@ -0,0 +1,4 @@ + +export default () => { + +} diff --git a/test/fixtures/esm/an-esm-project/package.json b/test/fixtures/esm/an-esm-project/package.json new file mode 100644 index 000000000..2e8633808 --- /dev/null +++ b/test/fixtures/esm/an-esm-project/package.json @@ -0,0 +1,18 @@ +{ + "name": "an-esm-project", + "version": "1.0.0", + "description": "", + "main": "src/index.js", + "type": "module", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + } +} diff --git a/test/fixtures/esm/an-esm-project/src/index.js b/test/fixtures/esm/an-esm-project/src/index.js new file mode 100644 index 000000000..32c3df472 --- /dev/null +++ b/test/fixtures/esm/an-esm-project/src/index.js @@ -0,0 +1,10 @@ +import herp from 'a-cjs-dep' +import derp from 'an-esm-dep' + +export const useHerp = () => { + herp() +} + +export const useDerp = () => { + derp() +} diff --git a/test/node.js b/test/node.js index 37fb17c47..ad3532e81 100644 --- a/test/node.js +++ b/test/node.js @@ -1,5 +1,6 @@ 'use strict' +require('./build') require('./lint') require('./fixtures') require('./dependants')