Introduce AutorestV2 for generation

.github/workflows/generate-schemas.yml

@@ -24,6 +24,8 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+        with: 
+          submodules: recursive
 
       - name: Clone azure-rest-api-specs
         uses: actions/checkout@v4
@@ -41,6 +43,18 @@ jobs:
         run: npm ci
         working-directory: generator
 
+      - name: Build bicep-types
+        run: |
+          npm ci
+          npm run build
+        working-directory: bicep-types-az/bicep-types/src/bicep-types
+
+      - name: Build autorest.bicep
+        run: |
+          npm ci
+          npm run build
+        working-directory: bicep-types-az/src/autorest.bicep
+
       - name: Run generator
         run: |
           rm -Rf "$GITHUB_WORKSPACE/schemas"

.github/workflows/generate-single.yml

@@ -14,11 +14,15 @@ on:
 jobs:
   update-schemas:
     name: Update Schemas
+    permissions:
+      contents: write
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+        with: 
+          submodules: recursive
 
       - name: Clone azure-rest-api-specs
         uses: actions/checkout@v4
@@ -36,26 +40,29 @@ jobs:
         run: npm ci
         working-directory: generator
 
+      - name: Build bicep-types
+        run: |
+          npm ci
+          npm run build
+        working-directory: bicep-types-az/bicep-types/src/bicep-types
+
+      - name: Build autorest.bicep
+        run: |
+          npm ci
+          npm run build
+        working-directory: bicep-types-az/src/autorest.bicep
+
       - name: Run generator
         run: |
           npm run generate-single -- \
             --specs-dir "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" \
             --base-path '${{ github.event.inputs.single_path }}'
         working-directory: generator
 
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v6
+      - name: Push to autogenerate-batch branch
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
-          committer: GitHub <noreply@github.com>
-          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
-          signoff: false
-          branch: autogenerate-${{ github.event.inputs.single_path }}
-          delete-branch: true
-          title: |
-            Update Generated Schemas (${{ github.event.inputs.single_path }})
-          body:  |
-            Update Generated Schemas (${{ github.event.inputs.single_path }})
-          commit-message: |
-            Update Generated Schemas (${{ github.event.inputs.single_path }})
-          labels: autogenerate
-          draft: false
+          commit_message: Update Generated Schemas (${{ github.event.inputs.single_path }})
+          branch: autogenerate-single${{ github.event.inputs.single_path }}
+          push_options: '--force'
+          create_branch: true

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "bicep-types-az"]
+	path = bicep-types-az
+	url = https://github.com/Azure/bicep-types-az

generator/autogenlist.ts

@@ -32,10 +32,9 @@ const disabledProviders: AutoGenConfig[] = [
         disabledForAutogen: true,
     },
     {
-        // Disabled until the unexpected character error in the swagger spec is fixed
         basePath: 'cdn/resource-manager',
         namespace: 'Microsoft.Cdn',
-        disabledForAutogen: true,
+        useAutorestV2: true,
     },
     {
         // Disabled until the enum mismatch in the swagger spec is fixed

generator/autorest.ts

@@ -0,0 +1,120 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import path from 'path';
+import os from 'os';
+import { findRecursive, lowerCaseContains, executeCmd, fileExists } from './utils';
+import * as constants from './constants';
+import { ReadmeTag, AutoGenConfig, CodeBlock } from './models';
+import * as cm from '@ts-common/commonmark-to-markdown'
+import * as yaml from 'js-yaml'
+import { readFile, writeFile } from 'fs/promises';
+
+const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest';
+export const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/;
+
+async function execAutoRest(tmpFolder: string, params: string[]) {
+    await executeCmd(__dirname, `${__dirname}/node_modules/.bin/${autorestBinary}`, params);
+    if (!fileExists(tmpFolder)) {
+        return [];
+    }
+
+    return await findRecursive(tmpFolder, p => path.extname(p) === '.json');
+}
+
+export async function runAutorest(readme: string, tmpFolder: string) {
+    const autoRestParams = [
+        `--version=${constants.autorestCoreVersion}`,
+        `--use=@autorest/azureresourceschema@${constants.azureresourceschemaVersion}`,
+        '--azureresourceschema',
+        `--output-folder=${tmpFolder}`,
+        '--multiapi',
+        '--pass-thru:subset-reducer',
+        '--pass-thru:schema-validator-swagger',
+        readme,
+    ];
+
+    if (constants.autoRestVerboseOutput) {
+        autoRestParams.push('--verbose');
+    }
+
+    return await execAutoRest(tmpFolder, autoRestParams);
+}
+
+
+
+export async function generateAutorestConfig(readme: string, autoGenConfig: AutoGenConfig) {
+    const content = (await readFile(readme)).toString();
+    const markdownEx = cm.parse(content);
+    const fileSet = new Set<string>();
+    for (const node of cm.iterate(markdownEx.markDown)) {
+        // We're only interested in yaml code blocks
+        if (node.type !== 'code_block' || !node.info || !node.literal ||
+            !node.info.trim().startsWith('yaml')) {
+            continue;
+        }
+
+        const DOC = (yaml.load(node.literal) as CodeBlock);
+        if (DOC) {
+            const inputFile = DOC['input-file'];
+            if (typeof inputFile === 'string') {
+                fileSet.add(inputFile);
+            } else if (inputFile instanceof Array) {
+                for (const i of inputFile) {
+                    fileSet.add(i);
+                }
+            }
+        }
+    }
+
+    let readmeTag = {} as ReadmeTag;
+    for (const inputFile of fileSet) {
+        const pathComponents = inputFile.split("/");
+
+        if (!autoGenConfig.useNamespaceFromConfig &&
+            !lowerCaseContains(pathComponents, autoGenConfig.namespace)) {
+            continue;
+        }
+
+        const apiVersion = pathComponents.filter(p => p.match(apiVersionRegex) !== null)[0];
+        if (!apiVersion) {
+            continue;
+        }
+
+        readmeTag[apiVersion] ??= readmeTag[apiVersion] || [];
+        readmeTag[apiVersion].push(inputFile);
+    }
+
+    if (autoGenConfig.readmeTag) {
+        readmeTag = {...readmeTag, ...autoGenConfig.readmeTag };
+    }
+
+    const schemaReadmeContent = compositeSchemaReadme(readmeTag);
+
+    const schemaReadme = readme.replace(/\.md$/i, '.azureresourceschema.md');
+
+    await writeFile(schemaReadme, schemaReadmeContent);
+}
+
+function compositeSchemaReadme(readmeTag: ReadmeTag): string {
+    let content =
+`## AzureResourceSchema
+
+### AzureResourceSchema multi-api
+
+\`\`\` yaml $(azureresourceschema) && $(multiapi)
+${yaml.dump({ 'batch': Object.keys(readmeTag).map(apiVersion => ({ 'tag': `schema-${apiVersion}`})) }, { lineWidth: 1000 })}
+\`\`\`
+
+`
+    for (const apiVersion of Object.keys(readmeTag)) {
+        content +=
+`
+### Tag: schema-${apiVersion} and azureresourceschema
+
+\`\`\` yaml $(tag) == 'schema-${apiVersion}' && $(azureresourceschema)
+${yaml.dump({ 'input-file': readmeTag[apiVersion] }, { lineWidth: 1000})}
+\`\`\`
+`
+    }
+    return content;
+}

generator/autorestV2.ts

@@ -0,0 +1,127 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import path from 'path';
+import os from 'os';
+import { findRecursive, executeCmd, fileExists } from './utils';
+import * as constants from './constants';
+import { readFile, writeFile } from 'fs/promises';
+import * as markdown from '@ts-common/commonmark-to-markdown'
+import * as yaml from 'js-yaml'
+
+const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest';
+
+const rootDir = `${__dirname}/../`;
+const extensionDir = path.resolve(`${rootDir}/bicep-types-az/src/autorest.bicep/`);
+
+export async function generateAutorestV2Config(readmePath: string, bicepReadmePath: string) {
+  // We expect a path format convention of <provider>/(any/number/of/intervening/folders)/<yyyy>-<mm>-<dd>(|-preview)/<filename>.json
+  // This information is used to generate individual tags in the generated autorest configuration
+  // eslint-disable-next-line no-useless-escape
+  const pathRegex = /^(\$\(this-folder\)\/|)([^\/]+)(?:\/[^\/]+)*\/(\d{4}-\d{2}-\d{2}(|-preview))\/.*\.json$/i;
+
+  const readmeContents = await readFile(readmePath, { encoding: 'utf8' });
+  const readmeMarkdown = markdown.parse(readmeContents);
+
+  const inputFiles = new Set<string>();
+  // we need to look for all autorest configuration elements containing input files, and collect that list of files. These will look like (e.g.):
+  // ```yaml $(tag) == 'someTag'
+  // input-file:
+  // - path/to/file.json
+  // - path/to/other_file.json
+  // ```
+  for (const node of markdown.iterate(readmeMarkdown.markDown)) {
+    // We're only interested in yaml code blocks
+    if (node.type !== 'code_block' || !node.info || !node.literal ||
+      !node.info.trim().startsWith('yaml')) {
+      continue;
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const yamlData = yaml.load(node.literal) as any;
+    if (yamlData) {
+      // input-file may be a single string or an array of strings
+      const inputFile = yamlData['input-file'];
+      if (typeof inputFile === 'string') {
+        inputFiles.add(inputFile);
+      } else if (inputFile instanceof Array) {
+        for (const i of inputFile) {
+          inputFiles.add(i);
+        }
+      }
+    }
+  }
+
+  const filesByTag: Record<string, string[]> = {};
+  for (const file of inputFiles) {
+    const normalizedFile = normalizeJsonPath(file);
+    const match = pathRegex.exec(normalizedFile);
+    if (match) {
+      // Generate a unique tag. We can't process all of the different API versions in one autorest pass
+      // because there are constraints on naming uniqueness (e.g. naming of definitions), so we want to pass over
+      // each API version separately.
+      const tagName = `${match[2].toLowerCase()}-${match[3].toLowerCase()}`;
+      if (!filesByTag[tagName]) {
+        filesByTag[tagName] = [];
+      }
+
+      filesByTag[tagName].push(normalizedFile);
+    } else {
+      console.warn(`WARNING: Unable to parse swagger path "${file}"`);
+    }
+  }
+
+  let generatedContent = `##Bicep
+
+### Bicep multi-api
+\`\`\`yaml $(bicep) && $(multiapi)
+${yaml.dump({ 'batch': Object.keys(filesByTag).map(tag => ({ 'tag': tag })) }, { lineWidth: 1000 })}
+\`\`\`
+`;
+
+  for (const tag of Object.keys(filesByTag)) {
+    generatedContent += `### Tag: ${tag} and bicep
+\`\`\`yaml $(tag) == '${tag}' && $(bicep)
+${yaml.dump({ 'input-file': filesByTag[tag] }, { lineWidth: 1000})}
+\`\`\`
+`;
+  }
+
+  await writeFile(bicepReadmePath, generatedContent);
+}
+
+function normalizeJsonPath(jsonPath: string) {
+  // eslint-disable-next-line no-useless-escape
+  return path.normalize(jsonPath).replace(/[\\\/]/g, '/');
+}
+
+async function execAutoRest(tmpFolder: string, params: string[]) {
+    await executeCmd(__dirname, `${__dirname}/node_modules/.bin/${autorestBinary}`, params);
+    if (!fileExists(tmpFolder)) {
+        return [];
+    }
+
+    return await findRecursive(tmpFolder, p => path.extname(p) === '.json');
+}
+
+export async function runAutorestV2(readme: string, tmpFolder: string) {
+    const autoRestParams = [
+        `--use=@autorest/modelerfour`,
+        `--use=${extensionDir}`,
+        '--bicep',
+        `--output-folder=${tmpFolder}`,
+        '--multiapi',
+        '--title=none',
+        // This is necessary to avoid failures such as "ERROR: Semantic violation: Discriminator must be a required property." blocking type generation.
+        // In an ideal world, we'd raise issues in https://github.com/Azure/azure-rest-api-specs and force RP teams to fix them, but this isn't very practical
+        // as new validations are added continuously, and there's often quite a lag before teams will fix them - we don't want to be blocked by this in generating types.
+        `--skip-semantics-validation`,
+        `--arm-schema=true`,
+        readme,
+    ];
+
+    if (constants.autoRestVerboseOutput) {
+        autoRestParams.push('--verbose');
+    }
+
+    return await execAutoRest(tmpFolder, autoRestParams);
+}