diff --git a/.github/workflows/generate-schemas.yml b/.github/workflows/generate-schemas.yml index 56fe1ee560..2504869cc9 100644 --- a/.github/workflows/generate-schemas.yml +++ b/.github/workflows/generate-schemas.yml @@ -24,6 +24,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v4 + with: + submodules: recursive - name: Clone azure-rest-api-specs uses: actions/checkout@v4 @@ -41,6 +43,18 @@ jobs: run: npm ci working-directory: generator + - name: Build bicep-types + run: | + npm ci + npm run build + working-directory: bicep-types-az/bicep-types/src/bicep-types + + - name: Build autorest.bicep + run: | + npm ci + npm run build + working-directory: bicep-types-az/src/autorest.bicep + - name: Run generator run: | rm -Rf "$GITHUB_WORKSPACE/schemas" diff --git a/.github/workflows/generate-single.yml b/.github/workflows/generate-single.yml index 785a983ebe..bf2401f6f3 100644 --- a/.github/workflows/generate-single.yml +++ b/.github/workflows/generate-single.yml @@ -14,11 +14,15 @@ on: jobs: update-schemas: name: Update Schemas + permissions: + contents: write runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v4 + with: + submodules: recursive - name: Clone azure-rest-api-specs uses: actions/checkout@v4 @@ -36,6 +40,18 @@ jobs: run: npm ci working-directory: generator + - name: Build bicep-types + run: | + npm ci + npm run build + working-directory: bicep-types-az/bicep-types/src/bicep-types + + - name: Build autorest.bicep + run: | + npm ci + npm run build + working-directory: bicep-types-az/src/autorest.bicep + - name: Run generator run: | npm run generate-single -- \ @@ -43,19 +59,10 @@ jobs: --base-path '${{ github.event.inputs.single_path }}' working-directory: generator - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 + - name: Push to autogenerate-batch branch + uses: stefanzweifel/git-auto-commit-action@v5 with: - committer: GitHub - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: autogenerate-${{ github.event.inputs.single_path }} - delete-branch: true - title: | - Update Generated Schemas (${{ github.event.inputs.single_path }}) - body: | - Update Generated Schemas (${{ github.event.inputs.single_path }}) - commit-message: | - Update Generated Schemas (${{ github.event.inputs.single_path }}) - labels: autogenerate - draft: false \ No newline at end of file + commit_message: Update Generated Schemas (${{ github.event.inputs.single_path }}) + branch: autogenerate-single${{ github.event.inputs.single_path }} + push_options: '--force' + create_branch: true \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..cfb8c7a43a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "bicep-types-az"] + path = bicep-types-az + url = https://github.com/Azure/bicep-types-az diff --git a/bicep-types-az b/bicep-types-az new file mode 160000 index 0000000000..4f20acbf0e --- /dev/null +++ b/bicep-types-az @@ -0,0 +1 @@ +Subproject commit 4f20acbf0edb6873b0d232b517f618c43d1a6b0f diff --git a/generator/autogenlist.ts b/generator/autogenlist.ts index ef2420238b..37ce358a0d 100644 --- a/generator/autogenlist.ts +++ b/generator/autogenlist.ts @@ -32,10 +32,9 @@ const disabledProviders: AutoGenConfig[] = [ disabledForAutogen: true, }, { - // Disabled until the unexpected character error in the swagger spec is fixed basePath: 'cdn/resource-manager', namespace: 'Microsoft.Cdn', - disabledForAutogen: true, + useAutorestV2: true, }, { // Disabled until the enum mismatch in the swagger spec is fixed diff --git a/generator/autorest.ts b/generator/autorest.ts new file mode 100644 index 0000000000..fbfcc028ca --- /dev/null +++ b/generator/autorest.ts @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +import path from 'path'; +import os from 'os'; +import { findRecursive, lowerCaseContains, executeCmd, fileExists } from './utils'; +import * as constants from './constants'; +import { ReadmeTag, AutoGenConfig, CodeBlock } from './models'; +import * as cm from '@ts-common/commonmark-to-markdown' +import * as yaml from 'js-yaml' +import { readFile, writeFile } from 'fs/promises'; + +const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest'; +export const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/; + +async function execAutoRest(tmpFolder: string, params: string[]) { + await executeCmd(__dirname, `${__dirname}/node_modules/.bin/${autorestBinary}`, params); + if (!fileExists(tmpFolder)) { + return []; + } + + return await findRecursive(tmpFolder, p => path.extname(p) === '.json'); +} + +export async function runAutorest(readme: string, tmpFolder: string) { + const autoRestParams = [ + `--version=${constants.autorestCoreVersion}`, + `--use=@autorest/azureresourceschema@${constants.azureresourceschemaVersion}`, + '--azureresourceschema', + `--output-folder=${tmpFolder}`, + '--multiapi', + '--pass-thru:subset-reducer', + '--pass-thru:schema-validator-swagger', + readme, + ]; + + if (constants.autoRestVerboseOutput) { + autoRestParams.push('--verbose'); + } + + return await execAutoRest(tmpFolder, autoRestParams); +} + + + +export async function generateAutorestConfig(readme: string, autoGenConfig: AutoGenConfig) { + const content = (await readFile(readme)).toString(); + const markdownEx = cm.parse(content); + const fileSet = new Set(); + for (const node of cm.iterate(markdownEx.markDown)) { + // We're only interested in yaml code blocks + if (node.type !== 'code_block' || !node.info || !node.literal || + !node.info.trim().startsWith('yaml')) { + continue; + } + + const DOC = (yaml.load(node.literal) as CodeBlock); + if (DOC) { + const inputFile = DOC['input-file']; + if (typeof inputFile === 'string') { + fileSet.add(inputFile); + } else if (inputFile instanceof Array) { + for (const i of inputFile) { + fileSet.add(i); + } + } + } + } + + let readmeTag = {} as ReadmeTag; + for (const inputFile of fileSet) { + const pathComponents = inputFile.split("/"); + + if (!autoGenConfig.useNamespaceFromConfig && + !lowerCaseContains(pathComponents, autoGenConfig.namespace)) { + continue; + } + + const apiVersion = pathComponents.filter(p => p.match(apiVersionRegex) !== null)[0]; + if (!apiVersion) { + continue; + } + + readmeTag[apiVersion] ??= readmeTag[apiVersion] || []; + readmeTag[apiVersion].push(inputFile); + } + + if (autoGenConfig.readmeTag) { + readmeTag = {...readmeTag, ...autoGenConfig.readmeTag }; + } + + const schemaReadmeContent = compositeSchemaReadme(readmeTag); + + const schemaReadme = readme.replace(/\.md$/i, '.azureresourceschema.md'); + + await writeFile(schemaReadme, schemaReadmeContent); +} + +function compositeSchemaReadme(readmeTag: ReadmeTag): string { + let content = +`## AzureResourceSchema + +### AzureResourceSchema multi-api + +\`\`\` yaml $(azureresourceschema) && $(multiapi) +${yaml.dump({ 'batch': Object.keys(readmeTag).map(apiVersion => ({ 'tag': `schema-${apiVersion}`})) }, { lineWidth: 1000 })} +\`\`\` + +` + for (const apiVersion of Object.keys(readmeTag)) { + content += +` +### Tag: schema-${apiVersion} and azureresourceschema + +\`\`\` yaml $(tag) == 'schema-${apiVersion}' && $(azureresourceschema) +${yaml.dump({ 'input-file': readmeTag[apiVersion] }, { lineWidth: 1000})} +\`\`\` +` + } + return content; +} \ No newline at end of file diff --git a/generator/autorestV2.ts b/generator/autorestV2.ts new file mode 100644 index 0000000000..617f88f6a5 --- /dev/null +++ b/generator/autorestV2.ts @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +import path from 'path'; +import os from 'os'; +import { findRecursive, executeCmd, fileExists } from './utils'; +import * as constants from './constants'; +import { readFile, writeFile } from 'fs/promises'; +import * as markdown from '@ts-common/commonmark-to-markdown' +import * as yaml from 'js-yaml' + +const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest'; + +const rootDir = `${__dirname}/../`; +const extensionDir = path.resolve(`${rootDir}/bicep-types-az/src/autorest.bicep/`); + +export async function generateAutorestV2Config(readmePath: string, bicepReadmePath: string) { + // We expect a path format convention of /(any/number/of/intervening/folders)/--
(|-preview)/.json + // This information is used to generate individual tags in the generated autorest configuration + // eslint-disable-next-line no-useless-escape + const pathRegex = /^(\$\(this-folder\)\/|)([^\/]+)(?:\/[^\/]+)*\/(\d{4}-\d{2}-\d{2}(|-preview))\/.*\.json$/i; + + const readmeContents = await readFile(readmePath, { encoding: 'utf8' }); + const readmeMarkdown = markdown.parse(readmeContents); + + const inputFiles = new Set(); + // we need to look for all autorest configuration elements containing input files, and collect that list of files. These will look like (e.g.): + // ```yaml $(tag) == 'someTag' + // input-file: + // - path/to/file.json + // - path/to/other_file.json + // ``` + for (const node of markdown.iterate(readmeMarkdown.markDown)) { + // We're only interested in yaml code blocks + if (node.type !== 'code_block' || !node.info || !node.literal || + !node.info.trim().startsWith('yaml')) { + continue; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const yamlData = yaml.load(node.literal) as any; + if (yamlData) { + // input-file may be a single string or an array of strings + const inputFile = yamlData['input-file']; + if (typeof inputFile === 'string') { + inputFiles.add(inputFile); + } else if (inputFile instanceof Array) { + for (const i of inputFile) { + inputFiles.add(i); + } + } + } + } + + const filesByTag: Record = {}; + for (const file of inputFiles) { + const normalizedFile = normalizeJsonPath(file); + const match = pathRegex.exec(normalizedFile); + if (match) { + // Generate a unique tag. We can't process all of the different API versions in one autorest pass + // because there are constraints on naming uniqueness (e.g. naming of definitions), so we want to pass over + // each API version separately. + const tagName = `${match[2].toLowerCase()}-${match[3].toLowerCase()}`; + if (!filesByTag[tagName]) { + filesByTag[tagName] = []; + } + + filesByTag[tagName].push(normalizedFile); + } else { + console.warn(`WARNING: Unable to parse swagger path "${file}"`); + } + } + + let generatedContent = `##Bicep + +### Bicep multi-api +\`\`\`yaml $(bicep) && $(multiapi) +${yaml.dump({ 'batch': Object.keys(filesByTag).map(tag => ({ 'tag': tag })) }, { lineWidth: 1000 })} +\`\`\` +`; + + for (const tag of Object.keys(filesByTag)) { + generatedContent += `### Tag: ${tag} and bicep +\`\`\`yaml $(tag) == '${tag}' && $(bicep) +${yaml.dump({ 'input-file': filesByTag[tag] }, { lineWidth: 1000})} +\`\`\` +`; + } + + await writeFile(bicepReadmePath, generatedContent); +} + +function normalizeJsonPath(jsonPath: string) { + // eslint-disable-next-line no-useless-escape + return path.normalize(jsonPath).replace(/[\\\/]/g, '/'); +} + +async function execAutoRest(tmpFolder: string, params: string[]) { + await executeCmd(__dirname, `${__dirname}/node_modules/.bin/${autorestBinary}`, params); + if (!fileExists(tmpFolder)) { + return []; + } + + return await findRecursive(tmpFolder, p => path.extname(p) === '.json'); +} + +export async function runAutorestV2(readme: string, tmpFolder: string) { + const autoRestParams = [ + `--use=@autorest/modelerfour`, + `--use=${extensionDir}`, + '--bicep', + `--output-folder=${tmpFolder}`, + '--multiapi', + '--title=none', + // This is necessary to avoid failures such as "ERROR: Semantic violation: Discriminator must be a required property." blocking type generation. + // In an ideal world, we'd raise issues in https://github.com/Azure/azure-rest-api-specs and force RP teams to fix them, but this isn't very practical + // as new validations are added continuously, and there's often quite a lag before teams will fix them - we don't want to be blocked by this in generating types. + `--skip-semantics-validation`, + `--arm-schema=true`, + readme, + ]; + + if (constants.autoRestVerboseOutput) { + autoRestParams.push('--verbose'); + } + + return await execAutoRest(tmpFolder, autoRestParams); +} \ No newline at end of file diff --git a/generator/generate.ts b/generator/generate.ts index 38f94ecbf6..3ab42020df 100644 --- a/generator/generate.ts +++ b/generator/generate.ts @@ -2,14 +2,14 @@ // Licensed under the MIT License. import path from 'path'; import os from 'os'; -import { findRecursive, findDirRecursive, executeCmd, rmdirRecursive, lowerCaseCompare, lowerCaseCompareLists, lowerCaseStartsWith, readJsonFile, writeJsonFile, safeMkdir, safeUnlink, fileExists, lowerCaseEquals } from './utils'; +import { findDirRecursive, rmdirRecursive, lowerCaseCompare, lowerCaseCompareLists, lowerCaseStartsWith, readJsonFile, writeJsonFile, safeMkdir, safeUnlink, lowerCaseEquals } from './utils'; import * as constants from './constants'; -import { prepareReadme } from './specs'; import colors from 'colors'; import { ScopeType, AutoGenConfig } from './models'; import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference } from 'lodash'; +import { generateAutorestV2Config, runAutorestV2 } from './autorestV2'; +import { generateAutorestConfig, runAutorest } from './autorest'; -const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest'; export const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/; export interface SchemaConfiguration { @@ -44,21 +44,29 @@ export async function detectProviderNamespaces(readme: string) { } export async function generateSchemas(readme: string, autoGenConfig: AutoGenConfig): Promise { - await prepareReadme(readme, autoGenConfig); + if (autoGenConfig.useAutorestV2) { + const bicepReadmePath = `${path.dirname(readme)}/readme.bicep.md`; + await generateAutorestV2Config(readme, bicepReadmePath); + } else { + await generateAutorestConfig(readme, autoGenConfig); + } const schemaConfigs: SchemaConfiguration[] = []; const tmpFolder = path.join(os.tmpdir(), Math.random().toString(36).substr(2)); try { - const generatedSchemas = await generateSchema(readme, tmpFolder); + const generatedSchemas = autoGenConfig.useAutorestV2 ? + await runAutorestV2(readme, tmpFolder) : + await runAutorest(readme, tmpFolder); for (const schemaPath of generatedSchemas) { - const namespace = path.basename(schemaPath.substring(0, schemaPath.lastIndexOf(path.extname(schemaPath)))); - if (!lowerCaseEquals(autoGenConfig.namespace, namespace)) { + const contents = await readJsonFile(schemaPath); + const namespace = contents.title as string; + if (!lowerCaseEquals(autoGenConfig!.namespace, namespace)) { continue; } - const generatedSchemaConfig = await handleGeneratedSchema(readme, schemaPath, autoGenConfig); + const generatedSchemaConfig = await handleGeneratedSchema(readme, schemaPath, namespace, autoGenConfig); schemaConfigs.push(generatedSchemaConfig); } @@ -70,13 +78,7 @@ export async function generateSchemas(readme: string, autoGenConfig: AutoGenConf return schemaConfigs; } -async function handleGeneratedSchema(readme: string, schemaPath: string, autoGenConfig?: AutoGenConfig) { - const namespace = path.basename(schemaPath.substring(0, schemaPath.lastIndexOf(path.extname(schemaPath)))); - - if (autoGenConfig && autoGenConfig.namespace.toLowerCase() !== namespace.toLowerCase()) { - throw new Error(`Encountered unexpected namespace ${namespace} in readme ${readme}`); - } - +async function handleGeneratedSchema(readme: string, schemaPath: string, namespace: string, autoGenConfig?: AutoGenConfig) { const apiVersion = path.basename(path.resolve(`${schemaPath}/..`)); const schemaConfig = await generateSchemaConfig(schemaPath, namespace, apiVersion, autoGenConfig); @@ -91,34 +93,6 @@ async function handleGeneratedSchema(readme: string, schemaPath: string, autoGen return schemaConfig; } -async function execAutoRest(tmpFolder: string, params: string[]) { - await executeCmd(__dirname, `${__dirname}/node_modules/.bin/${autorestBinary}`, params); - if (!fileExists(tmpFolder)) { - return []; - } - - return await findRecursive(tmpFolder, p => path.extname(p) === '.json'); -} - -async function generateSchema(readme: string, tmpFolder: string) { - const autoRestParams = [ - `--version=${constants.autorestCoreVersion}`, - `--use=@autorest/azureresourceschema@${constants.azureresourceschemaVersion}`, - '--azureresourceschema', - `--output-folder=${tmpFolder}`, - '--multiapi', - '--pass-thru:subset-reducer', - '--pass-thru:schema-validator-swagger', - readme, - ]; - - if (constants.autoRestVerboseOutput) { - autoRestParams.push('--verbose'); - } - - return await execAutoRest(tmpFolder, autoRestParams); -} - // eslint-disable-next-line @typescript-eslint/no-explicit-any function getSchemaRefs(output: any, scopeType: ScopeType, resourceDefinitionsPath: string): SchemaReference[] { const resourceDefs = output[resourceDefinitionsPath] || {}; diff --git a/generator/models.ts b/generator/models.ts index ef57931dae..f2e9a45c29 100644 --- a/generator/models.ts +++ b/generator/models.ts @@ -11,6 +11,7 @@ export enum ScopeType { } export interface AutoGenConfig { + useAutorestV2?: true, disabledForAutogen?: true, basePath: string, namespace: string, diff --git a/generator/specs.ts b/generator/specs.ts index 05b7f43ca3..88303138fc 100644 --- a/generator/specs.ts +++ b/generator/specs.ts @@ -1,14 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. import path from 'path'; -import { findRecursive, lowerCaseContains } from './utils'; -import { ReadmeTag, AutoGenConfig, CodeBlock } from './models'; +import { findRecursive } from './utils'; import * as constants from './constants' -import * as cm from '@ts-common/commonmark-to-markdown' -import * as yaml from 'js-yaml' import { existsSync } from 'fs'; -import { readFile, writeFile } from 'fs/promises'; -import { apiVersionRegex } from './generate'; export async function resolveAbsolutePath(localPath: string) { if (path.isAbsolute(localPath)) { @@ -68,81 +63,4 @@ function isExcludedBasePath(basePath: string) { return constants.excludedBasePathPrefixes .map(prefix => prefix.toLowerCase()) .some(prefix => basePath.toLowerCase().startsWith(prefix)); -} - -export async function prepareReadme(readme: string, autoGenConfig: AutoGenConfig) { - const content = (await readFile(readme)).toString(); - const markdownEx = cm.parse(content); - const fileSet = new Set(); - for (const node of cm.iterate(markdownEx.markDown)) { - // We're only interested in yaml code blocks - if (node.type !== 'code_block' || !node.info || !node.literal || - !node.info.trim().startsWith('yaml')) { - continue; - } - - const DOC = (yaml.load(node.literal) as CodeBlock); - if (DOC) { - const inputFile = DOC['input-file']; - if (typeof inputFile === 'string') { - fileSet.add(inputFile); - } else if (inputFile instanceof Array) { - for (const i of inputFile) { - fileSet.add(i); - } - } - } - } - - let readmeTag = {} as ReadmeTag; - for (const inputFile of fileSet) { - const pathComponents = inputFile.split("/"); - - if (!autoGenConfig.useNamespaceFromConfig && - !lowerCaseContains(pathComponents, autoGenConfig.namespace)) { - continue; - } - - const apiVersion = pathComponents.filter(p => p.match(apiVersionRegex) !== null)[0]; - if (!apiVersion) { - continue; - } - - readmeTag[apiVersion] ??= readmeTag[apiVersion] || []; - readmeTag[apiVersion].push(inputFile); - } - - if (autoGenConfig.readmeTag) { - readmeTag = {...readmeTag, ...autoGenConfig.readmeTag }; - } - - const schemaReadmeContent = compositeSchemaReadme(readmeTag); - - const schemaReadme = readme.replace(/\.md$/i, '.azureresourceschema.md'); - - await writeFile(schemaReadme, schemaReadmeContent); -} - -function compositeSchemaReadme(readmeTag: ReadmeTag): string { - let content = -`## AzureResourceSchema - -### AzureResourceSchema multi-api - -\`\`\` yaml $(azureresourceschema) && $(multiapi) -${yaml.dump({ 'batch': Object.keys(readmeTag).map(apiVersion => ({ 'tag': `schema-${apiVersion}`})) }, { lineWidth: 1000 })} -\`\`\` - -` - for (const apiVersion of Object.keys(readmeTag)) { - content += -` -### Tag: schema-${apiVersion} and azureresourceschema - -\`\`\` yaml $(tag) == 'schema-${apiVersion}' && $(azureresourceschema) -${yaml.dump({ 'input-file': readmeTag[apiVersion] }, { lineWidth: 1000})} -\`\`\` -` - } - return content; } \ No newline at end of file