From 88bb91f3a0211c0db703746b1c1147c09c33d43b Mon Sep 17 00:00:00 2001 From: Byron Hulcher Date: Mon, 25 Jul 2022 19:33:59 -0400 Subject: [PATCH 1/4] [Enterprise Search] Remove Ingest Attachment message from App Search Crawler (#137106) --- .../components/crawler/crawler_overview.tsx | 31 ------------------- .../translations/translations/fr-FR.json | 3 -- .../translations/translations/ja-JP.json | 3 -- .../translations/translations/zh-CN.json | 3 -- 4 files changed, 40 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_overview.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_overview.tsx index 79460ac486f1f0..13a13c25a5ad85 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_overview.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_overview.tsx @@ -13,9 +13,6 @@ import { EuiFlexGroup, EuiFlexItem, EuiLink, EuiSpacer, EuiText, EuiTitle } from import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n-react'; - -import { docLinks } from '../../../shared/doc_links'; import { WEB_CRAWLER_DOCS_URL, WEB_CRAWLER_LOG_DOCS_URL } from '../../routes'; import { getEngineBreadcrumbs } from '../engine'; import { AppSearchPageTemplate } from '../layout'; @@ -46,34 +43,6 @@ export const CrawlerOverview: React.FC = () => { pageHeader={{ pageTitle: CRAWLER_TITLE, rightSideItems: [, ], - description: ( - - {i18n.translate( - 'xpack.enterpriseSearch.appSearch.crawler.ingestionPluginDocumentationLink', - { defaultMessage: 'Elasticsearch ingest attachment plugin' } - )} - - ), - deploymentSettingsDocumentationLink: ( - - {i18n.translate( - 'xpack.enterpriseSearch.appSearch.crawler.deploymentSettingsDocumentationLink', - { defaultMessage: 'review your deployment settings' } - )} - - ), - }} - /> - ), }} isLoading={dataLoading} > diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index d041a0b0738dc3..5c7d7b551ec18e 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -10880,7 +10880,6 @@ "xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.deleteDomainButtonLabel": "Supprimer le domaine", "xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.description": "Retirez ce domaine de votre robot d'indexation. Cela supprimera également tous les points d'entrée et toutes les règles d'indexation que vous avez configurés. {cannotUndoMessage}.", "xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.title": "Supprimer le domaine", - "xpack.enterpriseSearch.appSearch.crawler.deploymentSettingsDocumentationLink": "vérifier vos paramètres de déploiement", "xpack.enterpriseSearch.appSearch.crawler.domainsTable.action.add.successMessage": "Le domaine {domainUrl} a été ajouté avec succès", "xpack.enterpriseSearch.appSearch.crawler.domainsTable.action.delete.buttonLabel": "Supprimer ce domaine", "xpack.enterpriseSearch.appSearch.crawler.domainsTable.action.manage.buttonLabel": "Gérer ce domaine", @@ -10901,13 +10900,11 @@ "xpack.enterpriseSearch.appSearch.crawler.entryPointsTable.learnMoreLinkText": "Découvrez plus d'informations sur les points d'entrée.", "xpack.enterpriseSearch.appSearch.crawler.entryPointsTable.title": "Points d'entrée", "xpack.enterpriseSearch.appSearch.crawler.entryPointsTable.urlTableHead": "URL", - "xpack.enterpriseSearch.appSearch.crawler.ingestionPluginDocumentationLink": "Plug-in Ingest Attachment Elasticsearch", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.automaticCrawlingButtonLabel": "Indexation automatique", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.automaticCrawlingTitle": "Indexation automatique", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.manageCrawlsButtonLabel": "Gérer les indexations", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.reApplyCrawlRules.successMessage": "Les règles d'indexation sont en train d'être réappliquées dans l'arrière-plan", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.reApplyCrawlRulesButtonLabel": "Réappliquer les règles d'indexation", - "xpack.enterpriseSearch.appSearch.crawler.pdfExtractionMessage": "Intéressé par l'extraction d'autres types de contenu ? Installez la {ingestPluginDocumentationLink} et la {deploymentSettingsDocumentationLink}.", "xpack.enterpriseSearch.appSearch.crawler.simplifiedSelectable.deselectAllButtonLabel": "Tout désélectionner", "xpack.enterpriseSearch.appSearch.crawler.simplifiedSelectable.selectAllButtonLabel": "Tout sélectionner", "xpack.enterpriseSearch.appSearch.crawler.sitemapsTable.addButtonLabel": "Ajouter un plan du site", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 9d8b54cecbccb1..fe46cfda3399e8 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -10871,7 +10871,6 @@ "xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.deleteDomainButtonLabel": "ドメインを削除", "xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.description": "このドメインをクローラーから削除します。これにより、設定したすべてのエントリポイントとクロールルールも削除されます。{cannotUndoMessage}。", "xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.title": "ドメインを削除", - "xpack.enterpriseSearch.appSearch.crawler.deploymentSettingsDocumentationLink": "デプロイ設定を確認", "xpack.enterpriseSearch.appSearch.crawler.domainsTable.action.add.successMessage": "ドメイン'{domainUrl}'が正常に追加されました", "xpack.enterpriseSearch.appSearch.crawler.domainsTable.action.delete.buttonLabel": "このドメインを削除", "xpack.enterpriseSearch.appSearch.crawler.domainsTable.action.manage.buttonLabel": "このドメインを管理", @@ -10892,13 +10891,11 @@ "xpack.enterpriseSearch.appSearch.crawler.entryPointsTable.learnMoreLinkText": "エントリポイントの詳細。", "xpack.enterpriseSearch.appSearch.crawler.entryPointsTable.title": "エントリポイント", "xpack.enterpriseSearch.appSearch.crawler.entryPointsTable.urlTableHead": "URL", - "xpack.enterpriseSearch.appSearch.crawler.ingestionPluginDocumentationLink": "Elasticsearchインジェクト関連付けプラグイン", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.automaticCrawlingButtonLabel": "自動クローリング", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.automaticCrawlingTitle": "自動クローリング", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.manageCrawlsButtonLabel": "クロールの管理", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.reApplyCrawlRules.successMessage": "クロールルールはバックグラウンドで再適用されています", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.reApplyCrawlRulesButtonLabel": "クロールルールを再適用", - "xpack.enterpriseSearch.appSearch.crawler.pdfExtractionMessage": "追加のコンテンツタイプの抽出に関心がある場合{ingestPluginDocumentationLink}と{deploymentSettingsDocumentationLink}をインストールします。", "xpack.enterpriseSearch.appSearch.crawler.simplifiedSelectable.deselectAllButtonLabel": "すべて選択解除", "xpack.enterpriseSearch.appSearch.crawler.simplifiedSelectable.selectAllButtonLabel": "すべて選択", "xpack.enterpriseSearch.appSearch.crawler.sitemapsTable.addButtonLabel": "サイトマップを追加", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index a2bd52a8eb4407..8968f2f5b6ddb7 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -10887,7 +10887,6 @@ "xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.deleteDomainButtonLabel": "删除域", "xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.description": "从您的网络爬虫移除此域。这还将您已设置的所有入口点和爬网规则。{cannotUndoMessage}。", "xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.title": "删除域", - "xpack.enterpriseSearch.appSearch.crawler.deploymentSettingsDocumentationLink": "复查您的部署设置", "xpack.enterpriseSearch.appSearch.crawler.domainsTable.action.add.successMessage": "已成功添加域“{domainUrl}”", "xpack.enterpriseSearch.appSearch.crawler.domainsTable.action.delete.buttonLabel": "删除此域", "xpack.enterpriseSearch.appSearch.crawler.domainsTable.action.manage.buttonLabel": "管理此域", @@ -10908,13 +10907,11 @@ "xpack.enterpriseSearch.appSearch.crawler.entryPointsTable.learnMoreLinkText": "详细了解入口点。", "xpack.enterpriseSearch.appSearch.crawler.entryPointsTable.title": "入口点", "xpack.enterpriseSearch.appSearch.crawler.entryPointsTable.urlTableHead": "URL", - "xpack.enterpriseSearch.appSearch.crawler.ingestionPluginDocumentationLink": "Elasticsearch 采集附件插件", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.automaticCrawlingButtonLabel": "自动爬网", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.automaticCrawlingTitle": "自动爬网", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.manageCrawlsButtonLabel": "管理爬网", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.reApplyCrawlRules.successMessage": "正在后台重新应用爬网规则", "xpack.enterpriseSearch.appSearch.crawler.manageCrawlsPopover.reApplyCrawlRulesButtonLabel": "重新应用爬网规则", - "xpack.enterpriseSearch.appSearch.crawler.pdfExtractionMessage": "对提取其他内容类型感兴趣？安装 {ingestPluginDocumentationLink} 和 {deploymentSettingsDocumentationLink}。", "xpack.enterpriseSearch.appSearch.crawler.simplifiedSelectable.deselectAllButtonLabel": "取消全选", "xpack.enterpriseSearch.appSearch.crawler.simplifiedSelectable.selectAllButtonLabel": "全选", "xpack.enterpriseSearch.appSearch.crawler.sitemapsTable.addButtonLabel": "添加站点地图", From 20f9cf9fd45d7f3298a3cb0846403682e2ef6c93 Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 25 Jul 2022 18:49:17 -0500 Subject: [PATCH 2/4] [eslint] add rule for validating cross-boundary imports (#137116) --- dev_docs/operations/operations_landing.mdx | 6 + nav-kibana-dev.docnav.json | 6 + package.json | 6 + packages/BUILD.bazel | 6 + .../kbn-eslint-plugin-imports/BUILD.bazel | 1 + .../src/helpers/repo_source_classifier.ts | 31 +++ .../src/helpers/source.ts | 24 +++ .../kbn-eslint-plugin-imports/src/index.ts | 2 + .../src/rules/exports_moved_packages.ts | 2 +- .../src/rules/no_boundary_crossing.test.ts | 159 +++++++++++++++ .../src/rules/no_boundary_crossing.ts | 146 +++++++++++++ .../src/rules/no_unresolvable_imports.ts | 14 +- .../src/rules/no_unused_imports.ts | 2 +- .../src/rules/uniform_imports.ts | 13 +- packages/kbn-get-repo-files/BUILD.bazel | 118 +++++++++++ packages/kbn-get-repo-files/README.mdx | 10 + packages/kbn-get-repo-files/jest.config.js | 13 ++ packages/kbn-get-repo-files/package.json | 10 + .../kbn-get-repo-files/src/get_repo_files.ts | 67 ++++++ packages/kbn-get-repo-files/src/index.ts | 9 + packages/kbn-get-repo-files/tsconfig.json | 18 ++ .../BUILD.bazel | 128 ++++++++++++ .../kbn-repo-source-classifier-cli/README.md | 3 + .../jest.config.js | 13 ++ .../package.json | 10 + .../src/index.ts | 90 ++++++++ .../src/type_tree.ts | 104 ++++++++++ .../tsconfig.json | 18 ++ .../kbn-repo-source-classifier/BUILD.bazel | 119 +++++++++++ .../kbn-repo-source-classifier/README.mdx | 31 +++ .../kbn-repo-source-classifier/jest.config.js | 13 ++ .../kbn-repo-source-classifier/package.json | 10 + .../kbn-repo-source-classifier/src/config.ts | 59 ++++++ .../kbn-repo-source-classifier/src/index.ts | 10 + .../src/module_id.ts | 19 ++ .../src/module_type.ts | 16 ++ .../src/pkg_info.ts | 16 ++ .../src/repo_path.ts | 102 ++++++++++ .../src/repo_source_classifier.ts | 192 ++++++++++++++++++ .../kbn-repo-source-classifier/tsconfig.json | 18 ++ packages/kbn-test/BUILD.bazel | 1 + .../src/jest/configs/get_all_jest_paths.ts | 62 ++---- scripts/classify_source.js | 11 + yarn.lock | 24 +++ 44 files changed, 1670 insertions(+), 62 deletions(-) create mode 100644 packages/kbn-eslint-plugin-imports/src/helpers/repo_source_classifier.ts create mode 100644 packages/kbn-eslint-plugin-imports/src/helpers/source.ts create mode 100644 packages/kbn-eslint-plugin-imports/src/rules/no_boundary_crossing.test.ts create mode 100644 packages/kbn-eslint-plugin-imports/src/rules/no_boundary_crossing.ts create mode 100644 packages/kbn-get-repo-files/BUILD.bazel create mode 100644 packages/kbn-get-repo-files/README.mdx create mode 100644 packages/kbn-get-repo-files/jest.config.js create mode 100644 packages/kbn-get-repo-files/package.json create mode 100644 packages/kbn-get-repo-files/src/get_repo_files.ts create mode 100644 packages/kbn-get-repo-files/src/index.ts create mode 100644 packages/kbn-get-repo-files/tsconfig.json create mode 100644 packages/kbn-repo-source-classifier-cli/BUILD.bazel create mode 100644 packages/kbn-repo-source-classifier-cli/README.md create mode 100644 packages/kbn-repo-source-classifier-cli/jest.config.js create mode 100644 packages/kbn-repo-source-classifier-cli/package.json create mode 100644 packages/kbn-repo-source-classifier-cli/src/index.ts create mode 100644 packages/kbn-repo-source-classifier-cli/src/type_tree.ts create mode 100644 packages/kbn-repo-source-classifier-cli/tsconfig.json create mode 100644 packages/kbn-repo-source-classifier/BUILD.bazel create mode 100644 packages/kbn-repo-source-classifier/README.mdx create mode 100644 packages/kbn-repo-source-classifier/jest.config.js create mode 100644 packages/kbn-repo-source-classifier/package.json create mode 100644 packages/kbn-repo-source-classifier/src/config.ts create mode 100644 packages/kbn-repo-source-classifier/src/index.ts create mode 100644 packages/kbn-repo-source-classifier/src/module_id.ts create mode 100644 packages/kbn-repo-source-classifier/src/module_type.ts create mode 100644 packages/kbn-repo-source-classifier/src/pkg_info.ts create mode 100644 packages/kbn-repo-source-classifier/src/repo_path.ts create mode 100644 packages/kbn-repo-source-classifier/src/repo_source_classifier.ts create mode 100644 packages/kbn-repo-source-classifier/tsconfig.json create mode 100644 scripts/classify_source.js diff --git a/dev_docs/operations/operations_landing.mdx b/dev_docs/operations/operations_landing.mdx index 2e2cef5c9bbbbc..2533df2f194b97 100644 --- a/dev_docs/operations/operations_landing.mdx +++ b/dev_docs/operations/operations_landing.mdx @@ -22,6 +22,7 @@ layout: landing \ No newline at end of file diff --git a/nav-kibana-dev.docnav.json b/nav-kibana-dev.docnav.json index 96f4074d9a7f78..2ab23a70c26fa5 100644 --- a/nav-kibana-dev.docnav.json +++ b/nav-kibana-dev.docnav.json @@ -531,6 +531,12 @@ }, { "id": "kibDevDocsOpsDevCliRunner" + }, + { + "id": "kibDevDocsOpsGetRepoFiles" + }, + { + "id": "kibDevDocsOpsRepoSourceClassifier" } ] } diff --git a/package.json b/package.json index 590a66cc451f2d..f106a959642407 100644 --- a/package.json +++ b/package.json @@ -604,6 +604,7 @@ "@kbn/expect": "link:bazel-bin/packages/kbn-expect", "@kbn/find-used-node-modules": "link:bazel-bin/packages/kbn-find-used-node-modules", "@kbn/generate": "link:bazel-bin/packages/kbn-generate", + "@kbn/get-repo-files": "link:bazel-bin/packages/kbn-get-repo-files", "@kbn/import-resolver": "link:bazel-bin/packages/kbn-import-resolver", "@kbn/jest-serializers": "link:bazel-bin/packages/kbn-jest-serializers", "@kbn/optimizer": "link:bazel-bin/packages/kbn-optimizer", @@ -611,6 +612,8 @@ "@kbn/performance-testing-dataset-extractor": "link:bazel-bin/packages/kbn-performance-testing-dataset-extractor", "@kbn/plugin-generator": "link:bazel-bin/packages/kbn-plugin-generator", "@kbn/plugin-helpers": "link:bazel-bin/packages/kbn-plugin-helpers", + "@kbn/repo-source-classifier": "link:bazel-bin/packages/kbn-repo-source-classifier", + "@kbn/repo-source-classifier-cli": "link:bazel-bin/packages/kbn-repo-source-classifier-cli", "@kbn/scalability-simulation-generator": "link:bazel-bin/packages/kbn-scalability-simulation-generator", "@kbn/some-dev-log": "link:bazel-bin/packages/kbn-some-dev-log", "@kbn/sort-package-json": "link:bazel-bin/packages/kbn-sort-package-json", @@ -851,6 +854,7 @@ "@types/kbn__field-types": "link:bazel-bin/packages/kbn-field-types/npm_module_types", "@types/kbn__find-used-node-modules": "link:bazel-bin/packages/kbn-find-used-node-modules/npm_module_types", "@types/kbn__generate": "link:bazel-bin/packages/kbn-generate/npm_module_types", + "@types/kbn__get-repo-files": "link:bazel-bin/packages/kbn-get-repo-files/npm_module_types", "@types/kbn__handlebars": "link:bazel-bin/packages/kbn-handlebars/npm_module_types", "@types/kbn__hapi-mocks": "link:bazel-bin/packages/kbn-hapi-mocks/npm_module_types", "@types/kbn__home-sample-data-cards": "link:bazel-bin/packages/home/sample_data_cards/npm_module_types", @@ -876,6 +880,8 @@ "@types/kbn__plugin-generator": "link:bazel-bin/packages/kbn-plugin-generator/npm_module_types", "@types/kbn__plugin-helpers": "link:bazel-bin/packages/kbn-plugin-helpers/npm_module_types", "@types/kbn__react-field": "link:bazel-bin/packages/kbn-react-field/npm_module_types", + "@types/kbn__repo-source-classifier": "link:bazel-bin/packages/kbn-repo-source-classifier/npm_module_types", + "@types/kbn__repo-source-classifier-cli": "link:bazel-bin/packages/kbn-repo-source-classifier-cli/npm_module_types", "@types/kbn__rule-data-utils": "link:bazel-bin/packages/kbn-rule-data-utils/npm_module_types", "@types/kbn__scalability-simulation-generator": "link:bazel-bin/packages/kbn-scalability-simulation-generator/npm_module_types", "@types/kbn__securitysolution-autocomplete": "link:bazel-bin/packages/kbn-securitysolution-autocomplete/npm_module_types", diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel index d376efa4c00282..fea0d789820a4f 100644 --- a/packages/BUILD.bazel +++ b/packages/BUILD.bazel @@ -145,6 +145,7 @@ filegroup( "//packages/kbn-find-used-node-modules:build", "//packages/kbn-flot-charts:build", "//packages/kbn-generate:build", + "//packages/kbn-get-repo-files:build", "//packages/kbn-handlebars:build", "//packages/kbn-hapi-mocks:build", "//packages/kbn-i18n-react:build", @@ -165,6 +166,8 @@ filegroup( "//packages/kbn-plugin-generator:build", "//packages/kbn-plugin-helpers:build", "//packages/kbn-react-field:build", + "//packages/kbn-repo-source-classifier-cli:build", + "//packages/kbn-repo-source-classifier:build", "//packages/kbn-rule-data-utils:build", "//packages/kbn-scalability-simulation-generator:build", "//packages/kbn-securitysolution-autocomplete:build", @@ -361,6 +364,7 @@ filegroup( "//packages/kbn-field-types:build_types", "//packages/kbn-find-used-node-modules:build_types", "//packages/kbn-generate:build_types", + "//packages/kbn-get-repo-files:build_types", "//packages/kbn-handlebars:build_types", "//packages/kbn-hapi-mocks:build_types", "//packages/kbn-i18n-react:build_types", @@ -381,6 +385,8 @@ filegroup( "//packages/kbn-plugin-generator:build_types", "//packages/kbn-plugin-helpers:build_types", "//packages/kbn-react-field:build_types", + "//packages/kbn-repo-source-classifier-cli:build_types", + "//packages/kbn-repo-source-classifier:build_types", "//packages/kbn-rule-data-utils:build_types", "//packages/kbn-scalability-simulation-generator:build_types", "//packages/kbn-securitysolution-autocomplete:build_types", diff --git a/packages/kbn-eslint-plugin-imports/BUILD.bazel b/packages/kbn-eslint-plugin-imports/BUILD.bazel index 8e878d4f595a09..73badddbcb2817 100644 --- a/packages/kbn-eslint-plugin-imports/BUILD.bazel +++ b/packages/kbn-eslint-plugin-imports/BUILD.bazel @@ -57,6 +57,7 @@ TYPES_DEPS = [ "//packages/kbn-utils:npm_module_types", "//packages/kbn-dev-utils:npm_module_types", # only required for the tests, which are excluded except on windows "//packages/kbn-import-resolver:npm_module_types", + "//packages/kbn-repo-source-classifier:npm_module_types", "@npm//dedent", # only required for the tests, which are excluded except on windows "@npm//@types/eslint", "@npm//@types/jest", diff --git a/packages/kbn-eslint-plugin-imports/src/helpers/repo_source_classifier.ts b/packages/kbn-eslint-plugin-imports/src/helpers/repo_source_classifier.ts new file mode 100644 index 00000000000000..9fd7e7e541a7ab --- /dev/null +++ b/packages/kbn-eslint-plugin-imports/src/helpers/repo_source_classifier.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ImportResolver } from '@kbn/import-resolver'; +import { RepoSourceClassifier } from '@kbn/repo-source-classifier'; + +const cache = new WeakMap(); + +/** + * Gets the instance of RepoSourceClassifier that should be used. We cache these instances + * key'd off of ImportResolver instances because the caches maintained by the RepoSourceClassifer + * should live the same amount of time. Both classes assume that the files on disk are + * relatively "stable" for the lifetime of the object and once the files are assumed + * to have change that a new object will be created and the old version with the old + * caches will be thrown away and garbage collected. + */ +export function getRepoSourceClassifier(resolver: ImportResolver) { + const cached = cache.get(resolver); + if (cached) { + return cached; + } + + const classifier = new RepoSourceClassifier(resolver); + cache.set(resolver, classifier); + return classifier; +} diff --git a/packages/kbn-eslint-plugin-imports/src/helpers/source.ts b/packages/kbn-eslint-plugin-imports/src/helpers/source.ts new file mode 100644 index 00000000000000..8bb98832f31185 --- /dev/null +++ b/packages/kbn-eslint-plugin-imports/src/helpers/source.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Rule } from 'eslint'; + +/** + * Get the path of the sourcefile being linted + */ +export function getSourcePath(context: Rule.RuleContext) { + const sourceFilename = context.getPhysicalFilename + ? context.getPhysicalFilename() + : context.getFilename(); + + if (!sourceFilename) { + throw new Error('unable to determine sourceFilename for file being linted'); + } + + return sourceFilename; +} diff --git a/packages/kbn-eslint-plugin-imports/src/index.ts b/packages/kbn-eslint-plugin-imports/src/index.ts index ca1050f9ba4060..64ae97ca760014 100644 --- a/packages/kbn-eslint-plugin-imports/src/index.ts +++ b/packages/kbn-eslint-plugin-imports/src/index.ts @@ -11,6 +11,7 @@ import { NoUnresolvableImportsRule } from './rules/no_unresolvable_imports'; import { UniformImportsRule } from './rules/uniform_imports'; import { ExportsMovedPackagesRule } from './rules/exports_moved_packages'; import { NoUnusedImportsRule } from './rules/no_unused_imports'; +import { NoBoundaryCrossingRule } from './rules/no_boundary_crossing'; /** * Custom ESLint rules, add `'@kbn/eslint-plugin-imports'` to your eslint config to use them @@ -21,4 +22,5 @@ export const rules = { uniform_imports: UniformImportsRule, exports_moved_packages: ExportsMovedPackagesRule, no_unused_imports: NoUnusedImportsRule, + no_boundary_crossing: NoBoundaryCrossingRule, }; diff --git a/packages/kbn-eslint-plugin-imports/src/rules/exports_moved_packages.ts b/packages/kbn-eslint-plugin-imports/src/rules/exports_moved_packages.ts index 2962ab4a5cb124..1dfde85dc1f2ef 100644 --- a/packages/kbn-eslint-plugin-imports/src/rules/exports_moved_packages.ts +++ b/packages/kbn-eslint-plugin-imports/src/rules/exports_moved_packages.ts @@ -220,7 +220,7 @@ export const ExportsMovedPackagesRule: Rule.RuleModule = { }, ], docs: { - url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.md#kbnimportsexports_moved_packages', + url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.mdx#kbnimportsexports_moved_packages', }, }, diff --git a/packages/kbn-eslint-plugin-imports/src/rules/no_boundary_crossing.test.ts b/packages/kbn-eslint-plugin-imports/src/rules/no_boundary_crossing.test.ts new file mode 100644 index 00000000000000..7830b01f68f8e5 --- /dev/null +++ b/packages/kbn-eslint-plugin-imports/src/rules/no_boundary_crossing.test.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { RuleTester } from 'eslint'; +import { NoBoundaryCrossingRule } from './no_boundary_crossing'; +import { ModuleType } from '@kbn/repo-source-classifier'; +import dedent from 'dedent'; + +const make = (from: ModuleType, to: ModuleType, imp = 'import') => ({ + filename: `${from}.ts`, + code: dedent` + ${imp} '${to}' + `, +}); + +jest.mock('../get_import_resolver', () => { + return { + getImportResolver() { + return { + resolve(req: string) { + return { + type: 'file', + absolute: { + type: req, + }, + }; + }, + }; + }, + }; +}); + +jest.mock('../helpers/repo_source_classifier', () => { + return { + getRepoSourceClassifier() { + return { + classify(r: string | { type: string }) { + return { + type: typeof r === 'string' ? (r.endsWith('.ts') ? r.slice(0, -3) : r) : r.type, + }; + }, + }; + }, + }; +}); + +const tsTester = [ + '@typescript-eslint/parser', + new RuleTester({ + parser: require.resolve('@typescript-eslint/parser'), + parserOptions: { + sourceType: 'module', + ecmaVersion: 2018, + ecmaFeatures: { + jsx: true, + }, + }, + }), +] as const; + +const babelTester = [ + '@babel/eslint-parser', + new RuleTester({ + parser: require.resolve('@babel/eslint-parser'), + parserOptions: { + sourceType: 'module', + ecmaVersion: 2018, + requireConfigFile: false, + babelOptions: { + presets: ['@kbn/babel-preset/node_preset'], + }, + }, + }), +] as const; + +for (const [name, tester] of [tsTester, babelTester]) { + describe(name, () => { + tester.run('@kbn/imports/no_boundary_crossing', NoBoundaryCrossingRule, { + valid: [ + make('common package', 'common package'), + make('server package', 'common package'), + make('browser package', 'common package'), + make('server package', 'server package'), + make('browser package', 'browser package'), + make('tests or mocks', 'common package'), + make('tests or mocks', 'browser package'), + make('tests or mocks', 'server package'), + make('tests or mocks', 'tests or mocks'), + make('browser package', 'server package', 'import type { Foo } from'), + make('server package', 'browser package', 'import type { Foo } from'), + make('common package', 'browser package', 'import type { Foo } from'), + ], + + invalid: [ + { + ...make('common package', 'server package'), + errors: [ + { + line: 1, + messageId: 'TYPE_MISMATCH', + data: { + importedType: 'server package', + ownType: 'common package', + suggestion: ` ${dedent` + Suggestions: + - Remove the import statement. + - Limit your imports to "common package" or "static" code. + - Covert to a type-only import. + - Reach out to #kibana-operations for help. + `}`, + }, + }, + ], + }, + { + ...make('server package', 'tests or mocks'), + errors: [ + { + line: 1, + messageId: 'TYPE_MISMATCH', + }, + ], + }, + { + ...make('browser package', 'tests or mocks'), + errors: [ + { + line: 1, + messageId: 'TYPE_MISMATCH', + }, + ], + }, + { + ...make('common package', 'server package'), + errors: [ + { + line: 1, + messageId: 'TYPE_MISMATCH', + }, + ], + }, + { + ...make('common package', 'browser package'), + errors: [ + { + line: 1, + messageId: 'TYPE_MISMATCH', + }, + ], + }, + ], + }); + }); +} diff --git a/packages/kbn-eslint-plugin-imports/src/rules/no_boundary_crossing.ts b/packages/kbn-eslint-plugin-imports/src/rules/no_boundary_crossing.ts new file mode 100644 index 00000000000000..4f3defc21d2968 --- /dev/null +++ b/packages/kbn-eslint-plugin-imports/src/rules/no_boundary_crossing.ts @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import { TSESTree } from '@typescript-eslint/typescript-estree'; +import * as Bt from '@babel/types'; +import { Rule } from 'eslint'; +import ESTree from 'estree'; +import { ModuleType } from '@kbn/repo-source-classifier'; + +import { visitAllImportStatements, Importer } from '../helpers/visit_all_import_statements'; +import { getSourcePath } from '../helpers/source'; +import { getRepoSourceClassifier } from '../helpers/repo_source_classifier'; +import { getImportResolver } from '../get_import_resolver'; + +const IMPORTABLE_FROM: Record = { + 'non-package': ['non-package', 'server package', 'browser package', 'common package', 'static'], + 'server package': ['common package', 'server package', 'static'], + 'browser package': ['common package', 'browser package', 'static'], + 'common package': ['common package', 'static'], + + static: [], + 'tests or mocks': '*', + tooling: '*', +}; + +const toList = (strings: string[]) => { + const items = strings.map((s) => `"${s}"`); + const list = items.slice(0, -1).join(', '); + const last = items.at(-1); + return !list.length ? last ?? '' : `${list} or ${last}`; +}; + +const formatSuggestions = (suggestions: string[]) => { + const s = suggestions.map((l) => l.trim()).filter(Boolean); + if (!s.length) { + return ''; + } + + return ` Suggestions:\n - ${s.join('\n - ')}`; +}; + +const isTypeOnlyImport = (importer: Importer) => { + // handle babel nodes + if (Bt.isImportDeclaration(importer)) { + return ( + importer.importKind === 'type' || + importer.specifiers.some((s) => ('importKind' in s ? s.importKind === 'type' : false)) + ); + } + + if (importer.type === TSESTree.AST_NODE_TYPES.ImportDeclaration) { + return ( + importer.importKind === 'type' || + importer.specifiers.some( + (s) => s.type === TSESTree.AST_NODE_TYPES.ImportSpecifier && s.importKind === 'type' + ) + ); + } + + if (Bt.isExportNamedDeclaration(importer)) { + return ( + importer.exportKind === 'type' || + importer.specifiers.some((s) => (Bt.isExportSpecifier(s) ? s.exportKind === 'type' : false)) + ); + } + + if (importer.type === TSESTree.AST_NODE_TYPES.ExportNamedDeclaration) { + return ( + importer.exportKind === 'type' || importer.specifiers.some((s) => s.exportKind === 'type') + ); + } + + return false; +}; + +export const NoBoundaryCrossingRule: Rule.RuleModule = { + meta: { + docs: { + url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.mdx#kbnimportsno_unused_imports', + }, + messages: { + TYPE_MISMATCH: `"{{importedType}}" code can not be imported from "{{ownType}}" code.{{suggestion}}`, + }, + }, + create(context) { + const resolver = getImportResolver(context); + const classifier = getRepoSourceClassifier(resolver); + const sourcePath = getSourcePath(context); + const ownDirname = Path.dirname(sourcePath); + + const self = classifier.classify(sourcePath); + const importable = IMPORTABLE_FROM[self.type]; + + if (importable === '*') { + // don't check imports in files which can import anything + return {}; + } + + return visitAllImportStatements((req, { node, importer }) => { + if ( + req === null || + // we can ignore imports using the raw-loader, they will need to be resolved but can be managed on a case by case basis + req.startsWith('!!raw-loader') || + // type only imports can stretch across all the boundaries + isTypeOnlyImport(importer) + ) { + return; + } + + const result = resolver.resolve(req, ownDirname); + if (result?.type !== 'file' || result.nodeModule) { + return; + } + + const imported = classifier.classify(result.absolute); + + if (!importable.includes(imported.type)) { + context.report({ + node: node as ESTree.Node, + messageId: 'TYPE_MISMATCH', + data: { + ownType: self.type, + importedType: imported.type, + suggestion: formatSuggestions([ + self.type.endsWith(' package') && imported.type === 'tests or mocks' + ? 'To expose mocks to other packages, they should be in their own package that is consumed by this package.' + : '', + `Remove the import statement.`, + importable.length > 0 ? `Limit your imports to ${toList(importable)} code.` : '', + `Covert to a type-only import.`, + `Reach out to #kibana-operations for help.`, + ]), + }, + }); + return; + } + }); + }, +}; diff --git a/packages/kbn-eslint-plugin-imports/src/rules/no_unresolvable_imports.ts b/packages/kbn-eslint-plugin-imports/src/rules/no_unresolvable_imports.ts index e5b324ea0a3e80..9990ec38530e5e 100644 --- a/packages/kbn-eslint-plugin-imports/src/rules/no_unresolvable_imports.ts +++ b/packages/kbn-eslint-plugin-imports/src/rules/no_unresolvable_imports.ts @@ -10,28 +10,22 @@ import Path from 'path'; import { Rule } from 'eslint'; import { report } from '../helpers/report'; +import { getSourcePath } from '../helpers/source'; import { getImportResolver } from '../get_import_resolver'; import { visitAllImportStatements } from '../helpers/visit_all_import_statements'; export const NoUnresolvableImportsRule: Rule.RuleModule = { meta: { docs: { - url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.md#kbnimportsno_unresolvable_imports', + url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.mdx#kbnimportsno_unresolvable_imports', }, }, create(context) { const resolver = getImportResolver(context); - - const sourceFilename = context.getPhysicalFilename - ? context.getPhysicalFilename() - : context.getFilename(); - - if (!sourceFilename) { - throw new Error('unable to determine sourceFilename for file being linted'); - } + const sourcePath = getSourcePath(context); return visitAllImportStatements((req, { node }) => { - if (req !== null && !resolver.resolve(req, Path.dirname(sourceFilename))) { + if (req !== null && !resolver.resolve(req, Path.dirname(sourcePath))) { report(context, { node, message: `Unable to resolve import [${req}]`, diff --git a/packages/kbn-eslint-plugin-imports/src/rules/no_unused_imports.ts b/packages/kbn-eslint-plugin-imports/src/rules/no_unused_imports.ts index 88153f6f914e56..e3c196c5d560e9 100644 --- a/packages/kbn-eslint-plugin-imports/src/rules/no_unused_imports.ts +++ b/packages/kbn-eslint-plugin-imports/src/rules/no_unused_imports.ts @@ -80,7 +80,7 @@ export const NoUnusedImportsRule: Rule.RuleModule = { meta: { fixable: 'code', docs: { - url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.md#kbnimportsno_unused_imports', + url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.mdx#kbnimportsno_unused_imports', }, }, create(context) { diff --git a/packages/kbn-eslint-plugin-imports/src/rules/uniform_imports.ts b/packages/kbn-eslint-plugin-imports/src/rules/uniform_imports.ts index 75487e62cd6137..5d6daad4cfdee5 100644 --- a/packages/kbn-eslint-plugin-imports/src/rules/uniform_imports.ts +++ b/packages/kbn-eslint-plugin-imports/src/rules/uniform_imports.ts @@ -14,6 +14,7 @@ import { getRelativeImportReq, getPackageRelativeImportReq } from '@kbn/import-r import { report } from '../helpers/report'; import { visitAllImportStatements } from '../helpers/visit_all_import_statements'; +import { getSourcePath } from '../helpers/source'; import { getImportResolver } from '../get_import_resolver'; // TODO: get rid of all the special cases in here by moving more things to packages @@ -27,19 +28,15 @@ export const UniformImportsRule: Eslint.Rule.RuleModule = { meta: { fixable: 'code', docs: { - url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.md#kbnimportsuniform_imports', + url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.mdx#kbnimportsuniform_imports', }, }, create(context) { const resolver = getImportResolver(context); - const sourceFilename = context.getPhysicalFilename - ? context.getPhysicalFilename() - : context.getFilename(); - - const sourceDirname = Path.dirname(sourceFilename); - - const ownPackageId = resolver.getPackageIdForPath(sourceFilename); + const sourcePath = getSourcePath(context); + const sourceDirname = Path.dirname(sourcePath); + const ownPackageId = resolver.getPackageIdForPath(sourcePath); return visitAllImportStatements((req, { node, type }) => { if (!req) { diff --git a/packages/kbn-get-repo-files/BUILD.bazel b/packages/kbn-get-repo-files/BUILD.bazel new file mode 100644 index 00000000000000..ac476aa0757446 --- /dev/null +++ b/packages/kbn-get-repo-files/BUILD.bazel @@ -0,0 +1,118 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-get-repo-files" +PKG_REQUIRE_NAME = "@kbn/get-repo-files" + +SOURCE_FILES = glob( + [ + "src/**/*.ts", + ], + exclude = [ + "**/*.test.*", + "**/*.stories.*", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "@npm//@types/node", + "@npm//@types/jest", + "@npm//execa", + "//packages/kbn-utils:npm_module_types", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + declaration_map = True, + emit_declaration_only = True, + out_dir = "target_types", + root_dir = "src", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +pkg_npm_types( + name = "npm_module_types", + srcs = SRCS, + deps = [":tsc_types"], + package_name = PKG_REQUIRE_NAME, + tsconfig = ":tsconfig", + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-get-repo-files/README.mdx b/packages/kbn-get-repo-files/README.mdx new file mode 100644 index 00000000000000..9ec3f1bb0128d5 --- /dev/null +++ b/packages/kbn-get-repo-files/README.mdx @@ -0,0 +1,10 @@ +--- +id: kibDevDocsOpsGetRepoFiles +slug: /kibana-dev-docs/ops/get-repo-files +title: "@kbn/get-repo-files" +description: 'A tool which lists the files under source-control' +date: 2022-07-25 +tags: ['kibana', 'dev', 'contributor', 'operations', 'packages', 'scripts', 'repo', 'files'] +--- + +This package exposes a helper to retreive a list of the files checked into the repository. It does this using the `git ls-files` CLI with some post processing to detemine if unstaged changes represent files which are modified (have edits) or deleted. diff --git a/packages/kbn-get-repo-files/jest.config.js b/packages/kbn-get-repo-files/jest.config.js new file mode 100644 index 00000000000000..d1b290f59a7ae3 --- /dev/null +++ b/packages/kbn-get-repo-files/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-get-repo-files'], +}; diff --git a/packages/kbn-get-repo-files/package.json b/packages/kbn-get-repo-files/package.json new file mode 100644 index 00000000000000..8cd19049fd1979 --- /dev/null +++ b/packages/kbn-get-repo-files/package.json @@ -0,0 +1,10 @@ +{ + "name": "@kbn/get-repo-files", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "license": "SSPL-1.0 OR Elastic License 2.0", + "kibana": { + "devOnly": true + } +} diff --git a/packages/kbn-get-repo-files/src/get_repo_files.ts b/packages/kbn-get-repo-files/src/get_repo_files.ts new file mode 100644 index 00000000000000..f63018b99e0d76 --- /dev/null +++ b/packages/kbn-get-repo-files/src/get_repo_files.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fs from 'fs'; + +import execa from 'execa'; +import { REPO_ROOT } from '@kbn/utils'; + +interface RepoPath { + /** repo-relative path to the file */ + repoRel: string; + /** absolute path to the file */ + abs: string; +} + +/** + * List the files in the repo, only including files which are manged by version + * control or "untracked" (new, not committed, and not ignored). + * @param include limit the list to specfic absolute paths + * @param exclude exclude specific absolute paths + */ +export async function getRepoFiles(include?: string[], exclude?: string[]) { + const flags = [ + include?.map((p) => Path.relative(REPO_ROOT, p)) ?? [], + exclude?.map((p) => `--exclude=${Path.relative(REPO_ROOT, p)}`) ?? [], + ].flat(); + + const proc = await execa('git', ['ls-files', '-comt', '--exclude-standard', ...flags], { + cwd: REPO_ROOT, + stdio: ['ignore', 'pipe', 'pipe'], + buffer: true, + }); + + const paths = new Map(); + const files = new Set(); + + for (const line of proc.stdout.split('\n').map((l) => l.trim())) { + if (!line) { + continue; + } + + const repoRel = line.slice(2); // trim the single char status and separating space from the line + const existingPath = paths.get(repoRel); + const path = existingPath ?? { repoRel, abs: Path.resolve(REPO_ROOT, repoRel) }; + if (!existingPath) { + paths.set(repoRel, path); + } + + if (line.startsWith('C ')) { + // this line indicates that the previous path is changed in the working + // tree, so we need to determine if it was deleted and remove it if so + if (!Fs.existsSync(path.abs)) { + files.delete(path); + } + } else { + files.add(path); + } + } + + return files; +} diff --git a/packages/kbn-get-repo-files/src/index.ts b/packages/kbn-get-repo-files/src/index.ts new file mode 100644 index 00000000000000..7e633d1c9b3816 --- /dev/null +++ b/packages/kbn-get-repo-files/src/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { getRepoFiles } from './get_repo_files'; diff --git a/packages/kbn-get-repo-files/tsconfig.json b/packages/kbn-get-repo-files/tsconfig.json new file mode 100644 index 00000000000000..789c6b3111115d --- /dev/null +++ b/packages/kbn-get-repo-files/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "rootDir": "src", + "stripInternal": false, + "types": [ + "jest", + "node" + ] + }, + "include": [ + "src/**/*" + ] +} diff --git a/packages/kbn-repo-source-classifier-cli/BUILD.bazel b/packages/kbn-repo-source-classifier-cli/BUILD.bazel new file mode 100644 index 00000000000000..6f4db611221cd1 --- /dev/null +++ b/packages/kbn-repo-source-classifier-cli/BUILD.bazel @@ -0,0 +1,128 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-repo-source-classifier-cli" +PKG_REQUIRE_NAME = "@kbn/repo-source-classifier-cli" + +SOURCE_FILES = glob( + [ + "src/**/*.ts", + ], + exclude = [ + "**/*.test.*", + "**/*.stories.*", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ + "//packages/kbn-dev-cli-runner", + "//packages/kbn-dev-cli-errors", + "//packages/kbn-import-resolver", + "//packages/kbn-repo-source-classifier", + "//packages/kbn-get-repo-files", + "//packages/kbn-utils", +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "@npm//@types/node", + "@npm//@types/jest", + "//packages/kbn-dev-cli-runner:npm_module_types", + "//packages/kbn-dev-cli-errors:npm_module_types", + "//packages/kbn-import-resolver:npm_module_types", + "//packages/kbn-repo-source-classifier:npm_module_types", + "//packages/kbn-get-repo-files:npm_module_types", + "//packages/kbn-utils:npm_module_types", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + declaration_map = True, + emit_declaration_only = True, + out_dir = "target_types", + root_dir = "src", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +pkg_npm_types( + name = "npm_module_types", + srcs = SRCS, + deps = [":tsc_types"], + package_name = PKG_REQUIRE_NAME, + tsconfig = ":tsconfig", + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-repo-source-classifier-cli/README.md b/packages/kbn-repo-source-classifier-cli/README.md new file mode 100644 index 00000000000000..bd85562b702511 --- /dev/null +++ b/packages/kbn-repo-source-classifier-cli/README.md @@ -0,0 +1,3 @@ +# @kbn/repo-source-classifier-cli + +CLI for debugging the repo source classifier, run to see how the classifier identifies your files. Use `node scripts/classify_source --help` for more info. \ No newline at end of file diff --git a/packages/kbn-repo-source-classifier-cli/jest.config.js b/packages/kbn-repo-source-classifier-cli/jest.config.js new file mode 100644 index 00000000000000..4fd0edec3b7a72 --- /dev/null +++ b/packages/kbn-repo-source-classifier-cli/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-repo-source-classifier-cli'], +}; diff --git a/packages/kbn-repo-source-classifier-cli/package.json b/packages/kbn-repo-source-classifier-cli/package.json new file mode 100644 index 00000000000000..47ca9f6598f93d --- /dev/null +++ b/packages/kbn-repo-source-classifier-cli/package.json @@ -0,0 +1,10 @@ +{ + "name": "@kbn/repo-source-classifier-cli", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "license": "SSPL-1.0 OR Elastic License 2.0", + "kibana": { + "devOnly": true + } +} diff --git a/packages/kbn-repo-source-classifier-cli/src/index.ts b/packages/kbn-repo-source-classifier-cli/src/index.ts new file mode 100644 index 00000000000000..0e741b98bb6d6f --- /dev/null +++ b/packages/kbn-repo-source-classifier-cli/src/index.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import { RepoSourceClassifier } from '@kbn/repo-source-classifier'; +import { ImportResolver } from '@kbn/import-resolver'; +import { REPO_ROOT } from '@kbn/utils'; +import { getRepoFiles } from '@kbn/get-repo-files'; +import { run } from '@kbn/dev-cli-runner'; +import { createFlagError } from '@kbn/dev-cli-errors'; + +import { TypeTree } from './type_tree'; + +run( + async ({ flags }) => { + const resolver = ImportResolver.create(REPO_ROOT); + const classifier = new RepoSourceClassifier(resolver); + + const include = flags._.length ? flags._ : [process.cwd()]; + let exclude; + if (flags.exclude) { + if (Array.isArray(flags.exclude)) { + exclude = flags.exclude; + } else if (typeof flags.exclude === 'string') { + exclude = [flags.exclude]; + } else { + throw createFlagError('expected --exclude value to be a string'); + } + } + + const typeFlags = String(flags.types) + .split(',') + .map((f) => f.trim()) + .filter(Boolean); + + const includeTypes: string[] = []; + const excludeTypes: string[] = []; + for (const type of typeFlags) { + if (type.startsWith('!')) { + excludeTypes.push(type.slice(1)); + } else { + includeTypes.push(type); + } + } + + const tree = new TypeTree(); + const cwd = process.cwd(); + for (const { abs } of await getRepoFiles(include, exclude)) { + const { type } = classifier.classify(abs); + if ((includeTypes.length && !includeTypes.includes(type)) || excludeTypes.includes(type)) { + continue; + } + + tree.add(type, Path.relative(cwd, abs)); + } + + if (!!flags.flat) { + for (const file of tree.toList()) { + process.stdout.write(`${file}\n`); + } + } else { + process.stdout.write(tree.print({ expand: !!flags.expand })); + } + }, + { + description: 'run the repo-source-classifier on the source files and produce a report', + usage: `node scripts/classify_source <...paths>`, + flags: { + string: ['exclude', 'types'], + boolean: ['expand', 'flat'], + help: ` + <...paths> include paths to select specific files which should be reported + by default all files in the cwd are classified. Can be specified + multiple times + --exclude exclude specific paths from the classification. Can be specified + multiple times + --types limit the types reported to the types in this comma separated list + to exclude a type prefix it with ! + --expand prevent collapsing entries that are of the same type + --flat just print file names + `, + }, + } +); diff --git a/packages/kbn-repo-source-classifier-cli/src/type_tree.ts b/packages/kbn-repo-source-classifier-cli/src/type_tree.ts new file mode 100644 index 00000000000000..749a9057de478e --- /dev/null +++ b/packages/kbn-repo-source-classifier-cli/src/type_tree.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ModuleType } from '@kbn/repo-source-classifier'; +import normalizePath from 'normalize-path'; + +type RecursiveTypes = Map; + +interface PrintOpts { + expand: boolean; +} + +export class TypeTree { + private dirs = new Map(); + private files = new Map(); + + constructor(public readonly path: string[] = []) {} + + add(type: ModuleType, rel: string) { + const segs = normalizePath(rel).split('/').filter(Boolean); + + let node: TypeTree = this; + const path = []; + for (const dirSeg of segs.slice(0, -1)) { + path.push(dirSeg); + const existing = node.dirs.get(dirSeg); + if (existing) { + node = existing; + } else { + const newDir = new TypeTree([...node.path, dirSeg]); + node.dirs.set(dirSeg, newDir); + node = newDir; + } + } + + const filename = segs.at(-1); + if (!filename) { + throw new Error(`invalid rel path [${rel}]`); + } + + node.files.set(filename, type); + } + + flatten(options: PrintOpts): ModuleType | RecursiveTypes { + const entries: RecursiveTypes = new Map([ + ...[...this.dirs].map(([name, dir]) => [name, dir.flatten(options)] as const), + ...this.files, + ]); + + if (!options.expand) { + const types = new Set(entries.values()); + const [firstType] = types; + if (types.size === 1 && typeof firstType === 'string') { + return firstType; + } + } + + return entries; + } + + print(options: PrintOpts) { + const tree = this.flatten(options); + + if (typeof tree === 'string') { + return `${this.path.join('/')}: ${tree}`; + } + + const lines: string[] = []; + const print = (prefix: string, types: RecursiveTypes) => { + for (const [name, childTypes] of types) { + if (typeof childTypes === 'string') { + lines.push(`${prefix}${name}: ${childTypes}`); + } else { + lines.push(`${prefix}${name}/`); + print(` ${prefix}`, childTypes); + } + } + }; + + print('', tree); + return lines.join('\n') + '\n'; + } + + toList() { + const files: string[] = []; + const getFiles = (tree: TypeTree) => { + for (const dir of tree.dirs.values()) { + getFiles(dir); + } + for (const filename of tree.files.keys()) { + files.push([...tree.path, filename].join('/')); + } + }; + + getFiles(this); + + return files.sort((a, b) => a.localeCompare(b)); + } +} diff --git a/packages/kbn-repo-source-classifier-cli/tsconfig.json b/packages/kbn-repo-source-classifier-cli/tsconfig.json new file mode 100644 index 00000000000000..789c6b3111115d --- /dev/null +++ b/packages/kbn-repo-source-classifier-cli/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "rootDir": "src", + "stripInternal": false, + "types": [ + "jest", + "node" + ] + }, + "include": [ + "src/**/*" + ] +} diff --git a/packages/kbn-repo-source-classifier/BUILD.bazel b/packages/kbn-repo-source-classifier/BUILD.bazel new file mode 100644 index 00000000000000..f3b1b08026b68c --- /dev/null +++ b/packages/kbn-repo-source-classifier/BUILD.bazel @@ -0,0 +1,119 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-repo-source-classifier" +PKG_REQUIRE_NAME = "@kbn/repo-source-classifier" + +SOURCE_FILES = glob( + [ + "src/**/*.ts", + ], + exclude = [ + "**/*.test.*", + "**/*.stories.*", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "@npm//@types/node", + "@npm//@types/jest", + "@npm//normalize-path", + "//packages/kbn-import-resolver:npm_module_types", + "//packages/kbn-utils:npm_module_types", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + declaration_map = True, + emit_declaration_only = True, + out_dir = "target_types", + root_dir = "src", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +pkg_npm_types( + name = "npm_module_types", + srcs = SRCS, + deps = [":tsc_types"], + package_name = PKG_REQUIRE_NAME, + tsconfig = ":tsconfig", + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-repo-source-classifier/README.mdx b/packages/kbn-repo-source-classifier/README.mdx new file mode 100644 index 00000000000000..2f197aa83a0c86 --- /dev/null +++ b/packages/kbn-repo-source-classifier/README.mdx @@ -0,0 +1,31 @@ +--- +id: kibDevDocsOpsRepoSourceClassifier +slug: /kibana-dev-docs/ops/repo-source-classifier +title: "@kbn/repo-source-classifier" +description: 'The tool which classifies source files into categories' +date: 2022-07-25 +tags: ['kibana', 'dev', 'contributor', 'operations', 'packages', 'scripts'] +--- + +This package exposes a class which can be used to efficiently classify all of the files in the repository into one of the following groups: + +- `server package`: plugin code in the root `server/` directory, eventually this will include packages of type `server-plugin` or `server-shared` + - `browser package`: plugin code in the root `public/` directory (and a few others in specific plugins), eventually this will include packages of type `browser-plugin` or `browser-shared` + - `common packages`: includes any existing package, plugin code in root `common/` directories, (and a few others in specific plugins), Eventually this will include `common-shared` packages + - `tests or mocks`: code that is loaded by jest/storybook, and mocks/helpers intended for use by that code. These files usually live along side package code but will have a separate dependency tree and are pieces of code which should never end up in the product. + - `static`: static files, currently any .json file or things loaded via `raw-loader` in browser code + - `tooling`: scripts, config files for tools like eslint, webpack, etc. + - `non-package`: code that lives outside of packages/plugins or doesn't fit into other more specific categories. Once the package project is complete this category should be limited to just `@kbn/pm` + +This is a map of types to the types they are allowed to import: + - `non-package`: `non-package`, `server package`, `browser package`, `common package` or `static` + - `server package`: `common package`, `server package`, or `static` + - `browser package`: `common package`, `browser package`, or `static` + - `common package`: `common package` or`static` + - `static`: static files are not allowed to have dependencies + - `tests or mocks`: anything + - `tooling`: anything + +The `RepoSourceClassifier` class implements several caches to make these lookups as efficient as possible in ESLint when all imports across the entire repository are validated. This cache lasts for the lifetime of the class and to invalidate the cache the object should just be discarded and a new instance created. + +A CLI is provided for inspecting the results of the classifier, check out `node scripts/classify_source --help` for more information about usage. \ No newline at end of file diff --git a/packages/kbn-repo-source-classifier/jest.config.js b/packages/kbn-repo-source-classifier/jest.config.js new file mode 100644 index 00000000000000..f7c33b88868050 --- /dev/null +++ b/packages/kbn-repo-source-classifier/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-repo-source-classifier'], +}; diff --git a/packages/kbn-repo-source-classifier/package.json b/packages/kbn-repo-source-classifier/package.json new file mode 100644 index 00000000000000..fce2be1b056ede --- /dev/null +++ b/packages/kbn-repo-source-classifier/package.json @@ -0,0 +1,10 @@ +{ + "name": "@kbn/repo-source-classifier", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "license": "SSPL-1.0 OR Elastic License 2.0", + "kibana": { + "devOnly": true + } +} diff --git a/packages/kbn-repo-source-classifier/src/config.ts b/packages/kbn-repo-source-classifier/src/config.ts new file mode 100644 index 00000000000000..6f6b9251988c06 --- /dev/null +++ b/packages/kbn-repo-source-classifier/src/config.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +// file names which indicate that the file is a testing file +export const RANDOM_TEST_FILE_NAMES = new Set([ + 'jest_setup', + 'test_data', + 'test_helper', + 'test_helpers', + 'stubs', + 'test_utils', + 'test_utilities', + 'rtl_helpers', + 'enzyme_helpers', + 'fixtures', + 'testbed', +]); + +// tags are found in filenames after a `.`, like `name.tag.ts` +export const TEST_TAG = new Set([ + 'test', + 'mock', + 'mocks', + 'stories', + 'story', + 'stub', + 'fixture', + 'story_decorators', + 'test_helpers', +]); + +// directories where test specific files are assumed to live, any file in a directory with these names is assumed to be test related +export const TEST_DIR = new Set([ + 'cypress', + 'test', + 'tests', + 'testing', + 'mock', + 'mocks', + '__jest__', + '__mock__', + '__test__', + '__mocks__', + '__stories__', + '__fixtures__', + '__snapshots__', + 'stub', + 'e2e', + 'ftr_e2e', + 'storybook', + '.storybook', + 'integration_tests', + ...RANDOM_TEST_FILE_NAMES, +]); diff --git a/packages/kbn-repo-source-classifier/src/index.ts b/packages/kbn-repo-source-classifier/src/index.ts new file mode 100644 index 00000000000000..fe684102083887 --- /dev/null +++ b/packages/kbn-repo-source-classifier/src/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export type { ModuleType } from './module_type'; +export { RepoSourceClassifier } from './repo_source_classifier'; diff --git a/packages/kbn-repo-source-classifier/src/module_id.ts b/packages/kbn-repo-source-classifier/src/module_id.ts new file mode 100644 index 00000000000000..ed79978a77994a --- /dev/null +++ b/packages/kbn-repo-source-classifier/src/module_id.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ModuleType } from './module_type'; +import { PkgInfo } from './pkg_info'; + +export interface ModuleId { + /** Type of the module */ + type: ModuleType; + /** repo relative path to the module's source file */ + repoRel: string; + /** info about the package the source file is within, in the case the file is found within a package */ + pkgInfo?: PkgInfo; +} diff --git a/packages/kbn-repo-source-classifier/src/module_type.ts b/packages/kbn-repo-source-classifier/src/module_type.ts new file mode 100644 index 00000000000000..f614e4180b8b42 --- /dev/null +++ b/packages/kbn-repo-source-classifier/src/module_type.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export type ModuleType = + | 'non-package' + | 'tests or mocks' + | 'static' + | 'tooling' + | 'server package' + | 'browser package' + | 'common package'; diff --git a/packages/kbn-repo-source-classifier/src/pkg_info.ts b/packages/kbn-repo-source-classifier/src/pkg_info.ts new file mode 100644 index 00000000000000..89d66092737b40 --- /dev/null +++ b/packages/kbn-repo-source-classifier/src/pkg_info.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export interface PkgInfo { + /** id of the package this file is from */ + pkgId: string; + /** Relative path to a file within a package directory */ + rel: string; + /** Absolute path to the package directory */ + pkgDir: string; +} diff --git a/packages/kbn-repo-source-classifier/src/repo_path.ts b/packages/kbn-repo-source-classifier/src/repo_path.ts new file mode 100644 index 00000000000000..05eef7105a7170 --- /dev/null +++ b/packages/kbn-repo-source-classifier/src/repo_path.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import { REPO_ROOT } from '@kbn/utils'; +import { ImportResolver } from '@kbn/import-resolver'; +import normalizePath from 'normalize-path'; + +import { PkgInfo } from './pkg_info'; + +const getNormal = Path.sep === '/' ? (path: string) => path : normalizePath; + +/** + * A wraper around an absolute path in the repository. We create this object and bind it + * to a specific ImportResolver primarily so that we can use it as a key in WeakMap instances + * and make analysis as efficient as possible. + * + * Instead of managing many caches in a stateful class somewhere we instead write memoized + * functions which use weakmaps that are key'd off of a RepoPath instance. RepoPath instances + * are then cached by the overall classifier, and that instance can then be cached by + * the tool running analysis on the repository + */ +export class RepoPath { + constructor(public readonly absolute: string, public readonly resolver: ImportResolver) {} + + private extname: string | undefined; + /** Get the extention for this path */ + getExtname() { + if (this.extname === undefined) { + this.extname = Path.extname(this.absolute); + } + + return this.extname; + } + + private filename: string | undefined; + /** Get the filename, without the extension, for this path */ + getFilename() { + if (this.filename === undefined) { + this.filename = Path.basename(this.absolute, this.getExtname()); + } + + return this.filename; + } + + private repoRel: string | undefined; + /** get and cache the repo-relative version of the path */ + getRepoRel() { + if (this.repoRel === undefined) { + this.repoRel = getNormal(Path.relative(REPO_ROOT, this.absolute)); + } + + return this.repoRel; + } + + private segs: string[] | undefined; + /** get and cache the path segments from the repo-realtive versions of this path */ + getSegs() { + if (this.segs === undefined) { + this.segs = Path.dirname(this.getRepoRel()).split('/'); + } + + return this.segs; + } + + private pkgInfo: PkgInfo | null | undefined; + /** get and cache the package info for a path */ + getPkgInfo() { + if (this.pkgInfo === undefined) { + const pkgId = this.resolver.getPackageIdForPath(this.absolute); + if (!pkgId) { + this.pkgInfo = null; + } else { + const pkgDir = this.resolver.getAbsolutePackageDir(pkgId); + if (!pkgDir) { + throw new Error(`unable to get package directory for package [${pkgId}]`); + } + + const rel = getNormal(Path.relative(pkgDir, this.absolute)); + if (rel.startsWith(`../`)) { + throw new Error( + `path [${this.getRepoRel()}] does not apear to be within package [${pkgId}]` + ); + } + + this.pkgInfo = { + pkgDir, + pkgId, + rel, + }; + } + } + + return this.pkgInfo; + } +} diff --git a/packages/kbn-repo-source-classifier/src/repo_source_classifier.ts b/packages/kbn-repo-source-classifier/src/repo_source_classifier.ts new file mode 100644 index 00000000000000..745ca523851661 --- /dev/null +++ b/packages/kbn-repo-source-classifier/src/repo_source_classifier.ts @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ImportResolver } from '@kbn/import-resolver'; +import { ModuleId } from './module_id'; +import { ModuleType } from './module_type'; +import { RANDOM_TEST_FILE_NAMES, TEST_DIR, TEST_TAG } from './config'; +import { RepoPath } from './repo_path'; + +export class RepoSourceClassifier { + constructor(private readonly resolver: ImportResolver) {} + + private repoPaths = new Map(); + private ids = new Map(); + + /** + * Get the cached repo path instance + */ + private getRepoPath(path: string) { + const cached = this.repoPaths.get(path); + + if (cached !== undefined) { + return cached; + } + + const rp = new RepoPath(path, this.resolver); + this.repoPaths.set(path, rp); + return rp; + } + + /** + * Is this a "test" file? + */ + private isTestFile(path: RepoPath) { + const name = path.getFilename(); + + if (name.startsWith('mock_') || RANDOM_TEST_FILE_NAMES.has(name)) { + return true; + } + + if (name.startsWith('_')) { + for (const tag of TEST_TAG) { + if (name.includes(tag)) { + return true; + } + } + } + + const tag = name.split('.').at(-1); + if (tag && TEST_TAG.has(tag)) { + return true; + } + + for (const seg of path.getSegs()) { + if (TEST_DIR.has(seg)) { + return true; + } + } + + return false; + } + + /** + * Is this a tooling file? + */ + private isToolingFile(path: RepoPath) { + const segs = path.getSegs(); + if ( + segs.includes('scripts') && + !path.getRepoRel().startsWith('src/plugins/data/server/scripts/') + ) { + return true; + } + + if (path.getFilename() === 'webpack.config' && path.getPkgInfo()?.pkgId !== '@kbn/optimizer') { + return true; + } + + return false; + } + + /** + * Apply canvas specific rules + * @param root the root dir within the canvas plugin + * @param dirs the directories after the root dir + * @returns a type, or undefined if the file should be classified as a standard file + */ + private classifyCanvas(root: string, dirs: string[]): ModuleType | undefined { + if (root === 'canvas_plugin_src') { + if (dirs[0] === 'expression_types') { + return 'common package'; + } + + const subRoot = dirs.slice(0, 2).join('/'); + if (subRoot === 'functions/external' || subRoot === 'functions/server') { + return 'server package'; + } + + return 'browser package'; + } + + if (root === 'i18n') { + return 'common package'; + } + + if (root === 'shareable_runtime') { + return 'non-package'; + } + + if (root === 'tasks') { + return 'tests or mocks'; + } + } + + /** + * Determine the "type" of a file + */ + private getType(path: RepoPath): ModuleType { + if (path.getExtname() === '.json') { + return 'static'; + } + + if (this.isTestFile(path)) { + return 'tests or mocks'; + } + + if (this.isToolingFile(path)) { + return 'tooling'; + } + + const pkgInfo = path.getPkgInfo(); + if (!pkgInfo) { + return 'non-package'; + } + + const { pkgId, rel } = pkgInfo; + const pkgIdWords = new Set(pkgId.split(/\W+/)); + // treat any package with "mocks" or "storybook" in the ID as a test-specific package + if (pkgIdWords.has('mocks') || pkgIdWords.has('storybook') || pkgIdWords.has('test')) { + return 'tests or mocks'; + } + + if (path.resolver.isBazelPackage(pkgId)) { + return 'common package'; + } + + const [root, ...dirs] = rel.split('/'); + + if (pkgId === '@kbn/core' && root === 'types') { + return 'common package'; + } + + if (pkgId === '@kbn/canvas-plugin') { + const type = this.classifyCanvas(root, dirs); + if (type) { + return type; + } + } + + if (root === 'public' || root === 'static') { + return 'browser package'; + } + + if (root === 'server') { + return 'server package'; + } + + return 'common package'; + } + + classify(absolute: string) { + const path = this.getRepoPath(absolute); + const cached = this.ids.get(path); + + if (cached) { + return cached; + } + + const id: ModuleId = { + type: this.getType(path), + repoRel: path.getRepoRel(), + pkgInfo: path.getPkgInfo() ?? undefined, + }; + this.ids.set(path, id); + return id; + } +} diff --git a/packages/kbn-repo-source-classifier/tsconfig.json b/packages/kbn-repo-source-classifier/tsconfig.json new file mode 100644 index 00000000000000..789c6b3111115d --- /dev/null +++ b/packages/kbn-repo-source-classifier/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "rootDir": "src", + "stripInternal": false, + "types": [ + "jest", + "node" + ] + }, + "include": [ + "src/**/*" + ] +} diff --git a/packages/kbn-test/BUILD.bazel b/packages/kbn-test/BUILD.bazel index b854379eb536bf..26cf9174d5936f 100644 --- a/packages/kbn-test/BUILD.bazel +++ b/packages/kbn-test/BUILD.bazel @@ -87,6 +87,7 @@ TYPES_DEPS = [ "//packages/kbn-utils:npm_module_types", "//packages/kbn-tooling-log:npm_module_types", "//packages/kbn-bazel-packages:npm_module_types", + "//packages/kbn-get-repo-files:npm_module_types", "@npm//@elastic/elasticsearch", "@npm//@jest/console", "@npm//@jest/reporters", diff --git a/packages/kbn-test/src/jest/configs/get_all_jest_paths.ts b/packages/kbn-test/src/jest/configs/get_all_jest_paths.ts index 63a829225ca609..336e28bd16fd5d 100644 --- a/packages/kbn-test/src/jest/configs/get_all_jest_paths.ts +++ b/packages/kbn-test/src/jest/configs/get_all_jest_paths.ts @@ -6,12 +6,10 @@ * Side Public License, v 1. */ -import Fs from 'fs'; import Path from 'path'; -import execa from 'execa'; import minimatch from 'minimatch'; -import { REPO_ROOT } from '@kbn/utils'; +import { getRepoFiles } from '@kbn/get-repo-files'; // @ts-expect-error jest-preset is necessarily a JS file import { testMatch } from '../../../jest-preset'; @@ -19,51 +17,31 @@ import { testMatch } from '../../../jest-preset'; const UNIT_CONFIG_NAME = 'jest.config.js'; const INTEGRATION_CONFIG_NAME = 'jest.integration.config.js'; -export async function getAllJestPaths() { - const proc = await execa('git', ['ls-files', '-comt', '--exclude-standard'], { - cwd: REPO_ROOT, - stdio: ['ignore', 'pipe', 'pipe'], - buffer: true, - }); +const testsRe = (testMatch as string[]).map((p) => minimatch.makeRe(p)); - const testsRe = (testMatch as string[]).map((p) => minimatch.makeRe(p)); - const classify = (rel: string) => { - if (testsRe.some((re) => re.test(rel))) { - return 'test' as const; - } +const classify = (rel: string) => { + if (testsRe.some((re) => re.test(rel))) { + return 'test' as const; + } - const basename = Path.basename(rel); - return basename === UNIT_CONFIG_NAME || basename === INTEGRATION_CONFIG_NAME - ? ('config' as const) - : undefined; - }; + const basename = Path.basename(rel); + return basename === UNIT_CONFIG_NAME || basename === INTEGRATION_CONFIG_NAME + ? ('config' as const) + : undefined; +}; +export async function getAllJestPaths() { const tests = new Set(); const configs = new Set(); - for (const line of proc.stdout.split('\n').map((l) => l.trim())) { - if (!line) { - continue; - } - - const rel = line.slice(2); // trim the single char status from the line - const type = classify(rel); - - if (!type) { - continue; - } - - const set = type === 'test' ? tests : configs; - const abs = Path.resolve(REPO_ROOT, rel); - - if (line.startsWith('C ')) { - // this line indicates that the previous path is changed in the working tree, so we need to determine if - // it was deleted, and if so, remove it from the set we added it to - if (!Fs.existsSync(abs)) { - set.delete(abs); - } - } else { - set.add(abs); + for (const { repoRel, abs } of await getRepoFiles()) { + switch (classify(repoRel)) { + case 'test': + tests.add(abs); + break; + case 'config': + configs.add(abs); + break; } } diff --git a/scripts/classify_source.js b/scripts/classify_source.js new file mode 100644 index 00000000000000..c73f339a1786a5 --- /dev/null +++ b/scripts/classify_source.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +require('../src/setup_node_env/ensure_node_preserve_symlinks'); +require('source-map-support/register'); +require('@kbn/repo-source-classifier-cli'); diff --git a/yarn.lock b/yarn.lock index 1b36723ba0e2a2..a650b44a1cc8a4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3463,6 +3463,10 @@ version "0.0.0" uid "" +"@kbn/get-repo-files@link:bazel-bin/packages/kbn-get-repo-files": + version "0.0.0" + uid "" + "@kbn/handlebars@link:bazel-bin/packages/kbn-handlebars": version "0.0.0" uid "" @@ -3559,6 +3563,14 @@ version "0.0.0" uid "" +"@kbn/repo-source-classifier-cli@link:bazel-bin/packages/kbn-repo-source-classifier-cli": + version "0.0.0" + uid "" + +"@kbn/repo-source-classifier@link:bazel-bin/packages/kbn-repo-source-classifier": + version "0.0.0" + uid "" + "@kbn/rule-data-utils@link:bazel-bin/packages/kbn-rule-data-utils": version "0.0.0" uid "" @@ -7191,6 +7203,10 @@ version "0.0.0" uid "" +"@types/kbn__get-repo-files@link:bazel-bin/packages/kbn-get-repo-files/npm_module_types": + version "0.0.0" + uid "" + "@types/kbn__handlebars@link:bazel-bin/packages/kbn-handlebars/npm_module_types": version "0.0.0" uid "" @@ -7291,6 +7307,14 @@ version "0.0.0" uid "" +"@types/kbn__repo-source-classifier-cli@link:bazel-bin/packages/kbn-repo-source-classifier-cli/npm_module_types": + version "0.0.0" + uid "" + +"@types/kbn__repo-source-classifier@link:bazel-bin/packages/kbn-repo-source-classifier/npm_module_types": + version "0.0.0" + uid "" + "@types/kbn__rule-data-utils@link:bazel-bin/packages/kbn-rule-data-utils/npm_module_types": version "0.0.0" uid "" From 37831e4e1978fc115e7830cd45e3c41026c56644 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 26 Jul 2022 00:51:41 +0100 Subject: [PATCH 3/4] skip flaky suite (#137124) --- .../api_integration/apis/uptime/rest/add_monitor_project.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/uptime/rest/add_monitor_project.ts b/x-pack/test/api_integration/apis/uptime/rest/add_monitor_project.ts index b1df6330f3b336..8549749e962154 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/add_monitor_project.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/add_monitor_project.ts @@ -17,7 +17,8 @@ import { PrivateLocationTestService } from './services/private_location_test_ser import { comparePolicies, getTestProjectSyntheticsPolicy } from './sample_data/test_policy'; export default function ({ getService }: FtrProviderContext) { - describe('[PUT] /api/uptime/service/monitors', function () { + // FLAKY: https://github.com/elastic/kibana/issues/137124 + describe.skip('[PUT] /api/uptime/service/monitors', function () { this.tags('skipCloud'); const supertest = getService('supertest'); From bf12ac2e6cbab88b6394364758dfdd38a811e290 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 26 Jul 2022 00:52:57 +0100 Subject: [PATCH 4/4] skip flaky suite (#135341) --- x-pack/test/accessibility/apps/spaces.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/accessibility/apps/spaces.ts b/x-pack/test/accessibility/apps/spaces.ts index 20ce3062c3e027..a898741d8636d3 100644 --- a/x-pack/test/accessibility/apps/spaces.ts +++ b/x-pack/test/accessibility/apps/spaces.ts @@ -85,7 +85,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); // creating space b and making it the current space so space selector page gets displayed when space b gets deleted - it('a11y test for delete space button', async () => { + // FLAKY: https://github.com/elastic/kibana/issues/135341 + it.skip('a11y test for delete space button', async () => { await PageObjects.spaceSelector.clickCreateSpace(); await PageObjects.spaceSelector.clickEnterSpaceName(); await PageObjects.spaceSelector.addSpaceName('space_b');