feat: map schema references from baseUrl to folder (#529)

CONTRIBUTING.md

@@ -14,13 +14,22 @@ Contributions are more than welcome. If you want to contribute, please make sure
 Before creating a Pull Request you should validate your code with ESLint.
 
 ```
-npm run eslint
+npm run lint
 ```
 
 ## Developing with Docker
 
 In case you want to quickly build a docker image locally to check if it works then run `npm run docker-build`
 
+In case you don't have a local node environment you can run 
+
+```
+docker build -t asyncapi/generator:latest .
+docker run -v `pwd`:`pwd` -w `pwd` -it --rm --entrypoint /bin/sh  asyncapi/generator:latest
+npm install
+./cli.js --help
+```
+
 ## Conventional commits
 
 This project follows [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/#summary) specification. Releasing to GitHub and NPM is done with the support of [semantic-release](https://semantic-release.gitbook.io/semantic-release/).
@@ -38,4 +47,4 @@ What about MAJOR release? just add `!` to the prefix, like `fix!: ` or `refactor
 
 Prefix that follows specification is not enough though. Remember that the title must be clear and descriptive with usage of [imperative mood](https://chris.beams.io/posts/git-commit/#imperative).
 
-Happy contributing :heart:
+Happy contributing :heart:

README.md

@@ -117,6 +117,7 @@ Options:
   -p, --param <name=value>       additional param to pass to templates
   --force-write                  force writing of the generated files to given directory even if it is a git repo with unstaged files or not empty dir (defaults to false)
   --watch-template               watches the template directory and the AsyncAPI document, and re-generate the files when changes occur. Ignores the output directory. This flag should be used only for template development.
+  --map-base-url <url:folder>    maps all schema references from base url to local folder
   -h, --help                     display help for command
 ```
 
@@ -191,6 +192,13 @@ It creates a symbolic link to the target directory (`~/my-template` in this case
 ag asyncapi.yaml https://github.com/asyncapi/html-template.git
 ```
 
+**Map schema references from baseUrl to local folder:**
+```bash
+ag test/docs/apiwithref.json @asyncapi/html-template -o ./build/ --force-write --map-base-url https://schema.example.com/crm/:./test/docs/
+```
+
+The parameter `--map-base-url` maps external schema references to local folders.
+
 ### CLI usage with Docker
 
 Install [Docker](https://docs.docker.com/get-docker/) first. Thanks to Docker you do not need Node.js even though the generator is written with it.

cli.js

@@ -19,6 +19,7 @@ let template;
 const params = {};
 const noOverwriteGlobs = [];
 const disabledHooks = {};
+const mapBaseUrlToFolder = {};
 
 const parseOutput = dir => path.resolve(dir);
 
@@ -41,6 +42,25 @@ const disableHooksParser = v => {
   }
 };
 
+const mapBaseUrlParser = v => {
+  // Example value for regular expression: https://schema.example.com/crm/:./test/docs/
+  // it splits on last occurrence of : into the groups all, url and folder
+  const re = /(.*):(.*)/g;
+  let mapping = [];
+  if ((mapping = re.exec(v))===null || mapping.length!==3) {
+    throw new Error('Invalid --map-base-url flag. A mapping <url>:<folder> with delimiter : expected.');
+  }
+
+  // Folder is without trailing slash, so make sure that url has also no trailing slash:
+  mapBaseUrlToFolder.url = mapping[1].replace(/\/$/, '');
+  mapBaseUrlToFolder.folder = path.resolve(mapping[2]);
+
+  const isURL = /^https?:/;
+  if (!isURL.test(mapBaseUrlToFolder.url.toLowerCase())) {
+    throw new Error('Invalid --map-base-url flag. The mapping <url>:<folder> requires a valid http/https url and valid folder with delimiter `:`.');
+  }
+};
+
 const showError = err => {
   console.error(red('Something went wrong:'));
   console.error(red(err.stack || err.message));
@@ -67,6 +87,7 @@ program
   .option('-p, --param <name=value>', 'additional param to pass to templates', paramParser)
   .option('--force-write', 'force writing of the generated files to given directory even if it is a git repo with unstaged files or not empty dir (defaults to false)')
   .option('--watch-template', 'watches the template directory and the AsyncAPI document, and re-generate the files when changes occur. Ignores the output directory. This flag should be used only for template development.')
+  .option('--map-base-url <url:folder>','maps all schema references from base url to local folder',mapBaseUrlParser)
   .parse(process.argv);
 
 if (!asyncapiDocPath) {
@@ -105,7 +126,7 @@ xfs.mkdirp(program.output, async err => {
     if (!await isLocalTemplate(path.resolve(Generator.DEFAULT_TEMPLATES_DIR, templateName))) {
       console.warn(`WARNING: ${template} is a remote template. Changes may be lost on subsequent installations.`);
     }
-    
+
     watcher.watch(watcherHandler, (paths) => {
       showErrorAndExit({ message: `[WATCHER] Could not find the file path ${paths}, are you sure it still exists? If it has been deleted or moved please rerun the generator.` });
     });
@@ -125,7 +146,8 @@ function generate(targetDir) {
         disabledHooks,
         forceWrite: program.forceWrite,
         install: program.install,
-        debug: program.debug
+        debug: program.debug,
+        mapBaseUrlToFolder
       });
 
       if (isAsyncapiDocLocal) {

docs/api.md

@@ -17,6 +17,7 @@
         * [.install](#Generator+install) : <code>Boolean</code>
         * [.templateConfig](#Generator+templateConfig) : <code>Object</code>
         * [.hooks](#Generator+hooks) : <code>Object</code>
+        * [.mapBaseUrlToFolder](#Generator+mapBaseUrlToFolder) : <code>Object</code>
         * [.templateParams](#Generator+templateParams) : <code>Object</code>
         * [.originalAsyncAPI](#Generator+originalAsyncAPI) : <code>String</code>
         * [.asyncapi](#Generator+asyncapi) : <code>AsyncAPIDocument</code>
@@ -48,6 +49,7 @@ Instantiates a new Generator object.
 | [options.forceWrite] | <code>Boolean</code> | <code>false</code> | Force writing of the generated files to given directory even if it is a git repo with unstaged files or not empty dir. Default is set to false. |
 | [options.install] | <code>Boolean</code> | <code>false</code> | Install the template and its dependencies, even when the template has already been installed. |
 | [options.debug] | <code>Boolean</code> | <code>false</code> | Enable more specific errors in the console. At the moment it only shows specific errors about filters. Keep in mind that as a result errors about template are less descriptive. |
+| [options.mapBaseUrlToFolder] | <code>Object.&lt;String, String&gt;</code> |  | Optional parameter to map schema references from a base url to a local base folder e.g. url=https://schema.example.com/crm/  folder=./test/docs/ . |
 
 **Example**  
 ```js
@@ -128,6 +130,12 @@ The template configuration.
 ### generator.hooks : <code>Object</code>
 Hooks object with hooks functions grouped by the hook type.
 
+**Kind**: instance property of [<code>Generator</code>](#Generator)  
+<a name="Generator+mapBaseUrlToFolder"></a>
+
+### generator.mapBaseUrlToFolder : <code>Object</code>
+Maps schema URL to folder.
+
 **Kind**: instance property of [<code>Generator</code>](#Generator)  
 <a name="Generator+templateParams"></a>
 

lib/generator.js

@@ -29,7 +29,8 @@ const {
   isJsFile,
   registerSourceMap,
   registerTypeScript,
-  getTemplateDetails
+  getTemplateDetails,
+  getMapBaseUrlToFolderResolver
 } = require('./utils');
 const { registerFilters } = require('./filtersRegistry');
 const { registerHooks } = require('./hooksRegistry');
@@ -45,7 +46,7 @@ const DEFAULT_TEMPLATES_DIR = path.resolve(ROOT_DIR, 'node_modules');
 
 const TRANSPILED_TEMPLATE_LOCATION = '__transpiled';
 const TEMPLATE_CONTENT_DIRNAME = 'template';
-const GENERATOR_OPTIONS = ['debug', 'disabledHooks', 'entrypoint', 'forceWrite', 'install', 'noOverwriteGlobs', 'output', 'templateParams'];
+const GENERATOR_OPTIONS = ['debug', 'disabledHooks', 'entrypoint', 'forceWrite', 'install', 'noOverwriteGlobs', 'output', 'templateParams', 'mapBaseUrlToFolder'];
 
 const logMessage = require('./logMessages');
 
@@ -89,8 +90,9 @@ class Generator {
    * @param {Boolean} [options.forceWrite=false] Force writing of the generated files to given directory even if it is a git repo with unstaged files or not empty dir. Default is set to false.
    * @param {Boolean} [options.install=false] Install the template and its dependencies, even when the template has already been installed.
    * @param {Boolean} [options.debug=false] Enable more specific errors in the console. At the moment it only shows specific errors about filters. Keep in mind that as a result errors about template are less descriptive.
+   * @param {Object<String, String>} [options.mapBaseUrlToFolder] Optional parameter to map schema references from a base url to a local base folder e.g. url=https://schema.example.com/crm/  folder=./test/docs/ .
    */
-  constructor(templateName, targetDir, { templateParams = {}, entrypoint, noOverwriteGlobs, disabledHooks, output = 'fs', forceWrite = false, install = false, debug = false } = {}) {
+  constructor(templateName, targetDir, { templateParams = {}, entrypoint, noOverwriteGlobs, disabledHooks, output = 'fs', forceWrite = false, install = false, debug = false, mapBaseUrlToFolder = {} } = {}) {
     const invalidOptions = getInvalidOptions(GENERATOR_OPTIONS, arguments[arguments.length - 1] || []);
     if (invalidOptions.length) throw new Error(`These options are not supported by the generator: ${invalidOptions.join(', ')}`);
     if (!templateName) throw new Error('No template name has been specified.');
@@ -119,6 +121,8 @@ class Generator {
     this.templateConfig = {};
     /** @type {Object} Hooks object with hooks functions grouped by the hook type. */
     this.hooks = {};
+    /** @type {Object} Maps schema URL to folder. */
+    this.mapBaseUrlToFolder = mapBaseUrlToFolder;
 
     // Load template configuration
     /** @type {Object} The template parameters. The structure for this object is based on each individual template. */
@@ -284,8 +288,12 @@ class Generator {
    */
   async generateFromURL(asyncapiURL) {
     const doc = await fetchSpec(asyncapiURL);
+    const parserOptions = {};
+    if (this.mapBaseUrlToFolder.url) {
+      parserOptions.resolve = {resolver: getMapBaseUrlToFolderResolver(this.mapBaseUrlToFolder)};
+    }
 
-    return this.generateFromString(doc);
+    return this.generateFromString(doc, parserOptions);
   }
 
   /**
@@ -312,7 +320,12 @@ class Generator {
    */
   async generateFromFile(asyncapiFile) {
     const doc = await readFile(asyncapiFile, { encoding: 'utf8' });
-    return this.generateFromString(doc, { path: asyncapiFile });
+    const parserOptions = { path: asyncapiFile };
+    if (this.mapBaseUrlToFolder.url) {
+      parserOptions.resolve = {resolver: getMapBaseUrlToFolderResolver(this.mapBaseUrlToFolder)};
+    }
+
+    return this.generateFromString(doc, parserOptions);
   }
 
   /**
@@ -475,7 +488,7 @@ class Generator {
   /**
    * Makes sure that during directory structure generation ignored dirs are not modified
    * @private
-   * 
+   *
    * @param  {String} root Dir name.
    * @param  {String} stats Information about the file.
    * @param  {Function} next Callback function
@@ -492,7 +505,7 @@ class Generator {
   /**
    * Makes sure that during directory structure generation ignored dirs are not modified
    * @private
-   * 
+   *
    * @param  {AsyncAPIDocument} asyncapiDocument AsyncAPI document to use as the source.
    * @param  {String} objectMap Map of schemas of type object
    * @param  {String} root Dir name.
@@ -591,7 +604,7 @@ class Generator {
 
   /**
    * Renders a template and writes the result into a file.
-   * 
+   *
    * @private
    * @param {AsyncAPIDocument} asyncapiDocument AsyncAPI document to pass to the template.
    * @param {String} templateFilePath Path to the input file being rendered.

lib/utils.js

@@ -61,8 +61,8 @@ utils.isLocalTemplate = async (templatePath) => {
 
 /**
  * Returns whether or not the template is a react template
- * 
- * @param {object} templateConfig 
+ *
+ * @param {object} templateConfig
  * @returns {boolean} Whether the template is a React template or not.
  */
 utils.isReactTemplate = (templateConfig) => {
@@ -71,7 +71,7 @@ utils.isReactTemplate = (templateConfig) => {
 
 /**
  * Fetches an AsyncAPI document from the given URL and return its content as string
- * 
+ *
  * @param {String} link URL where the AsyncAPI document is located.
  * @returns {Promise<String>} Content of fetched file.
  */
@@ -85,7 +85,7 @@ utils.fetchSpec = (link) => {
 
 /**
  * Checks if given string is URL and if not, we assume it is file path
- * 
+ *
  * @param {String} str information representing file path or url
  * @returns {Boolean}
  */
@@ -95,7 +95,7 @@ utils.isFilePath = (str) => {
 
 /**
  * Checks if given file path is JS file
- * 
+ *
  * @param {String} filename information representing file path
  * @returns {Boolean}
  */
@@ -106,7 +106,7 @@ utils.isJsFile = (filepath) => {
 
 /**
  * Get version of the generator
- * 
+ *
  * @returns {String}
  */
 utils.getGeneratorVersion = () => {
@@ -127,26 +127,26 @@ utils.getInvalidOptions = (generatorOptions, options) => {
 /**
  * Determine whether the given function is asynchronous.
  * @private
- * @param {*} fn to check 
+ * @param {*} fn to check
  * @returns {Boolean} is function asynchronous
  */
-utils.isAsyncFunction = (fn) => {  
+utils.isAsyncFunction = (fn) => {
   return fn && fn.constructor && fn.constructor.name === 'AsyncFunction';
 };
 
 /**
  * Register `source-map-support` package.
  * This package provides source map support for stack traces in Node - also for transpiled code from TS.
- * 
+ *
  * @private
  */
 utils.registerSourceMap = () => {
   require('source-map-support').install();
 };
 
 /**
- * Register TypeScript transpiler. It enables transpilation of TS filters and hooks on the fly. 
- * 
+ * Register TypeScript transpiler. It enables transpilation of TS filters and hooks on the fly.
+ *
  * @private
  */
 utils.registerTypeScript = () => {
@@ -155,7 +155,7 @@ utils.registerTypeScript = () => {
 
 /**
  * Check if given template is installed and return details about it located in the package.json
- * 
+ *
  * @private
  * @param {String} name name of the template
  * @param {String} PACKAGE_JSON_FILENAME standard name of the package.json file
@@ -166,12 +166,12 @@ utils.getTemplateDetails = (name, PACKAGE_JSON_FILENAME) => {
   let pkgPath;
   let pkgPackageJsonPath;
   let installedPkg;
-  
+
   //first trying to resolve package by its path
   //this could be done at the end, without additional if but the advantage if below approach is that we explicitly know resolving was successful by the template file path and we can log proper debug info
   if (utils.isFileSystemPath(name)) {
     pkgPath = path.resolve(name);
-    
+
     log.debug(logMessage.NODE_MODULES_INSTALL);
   } else {
     //first trying to resolve package in local dependencies of generator or the project that uses generator library
@@ -197,12 +197,49 @@ utils.getTemplateDetails = (name, PACKAGE_JSON_FILENAME) => {
       }
     }
   }
-  
+
   if (pkgPath) {
     installedPkg = require(path.join(pkgPath, PACKAGE_JSON_FILENAME));
     //we add path to returned object only because later we want to use it in debug logs
     installedPkg.pkgPath = pkgPath;
   }
 
-  return installedPkg; 
+  return installedPkg;
+};
+
+/**
+ * Creates a custom resolver that maps urlToFolder.url to urlToFolder.folder
+ * Building your custom resolver is explained here: https://apitools.dev/json-schema-ref-parser/docs/plugins/resolvers.html
+ *
+ * @private
+ * @param  {object} urlToFolder to resolve url e.g. https://schema.example.com/crm/ to a folder e.g. ./test/docs/.
+ * @return {{read(*, *, *): Promise<unknown>, canRead(*): boolean, order: number}}
+ */
+utils.getMapBaseUrlToFolderResolver = (urlToFolder) => {
+  return {
+    order: 1,
+    canRead (file) {
+      return true;
+    },
+    read(file, callback, $refs) {
+      const baseUrl = urlToFolder.url;
+      const baseDir = urlToFolder.folder;
+
+      return new Promise(((resolve, reject) => {
+        let localpath = file.url;
+        localpath = localpath.replace(baseUrl,baseDir);
+        try {
+          fs.readFile(localpath, (err, data) => {
+            if (err) {
+              reject(`Error opening file "${localpath}"`);
+            } else {
+              resolve(data);
+            }
+          });
+        } catch (err) {
+          reject(`Error opening file "${localpath}"`);
+        }
+      }));
+    }
+  };
 };