[Security Solution] Detection rules bootstrap endpoint (#189518)

**Resolves: #187647

## Summary

Added a new API endpoint `POST
/internal/detection_engine/prebuilt_rules/_bootstrap`. This endpoint is
responsible for installing the necessary packages for prebuilt detection
rules to function properly. This allows us to avoid calling Fleet
directly from FE and helps encapsulate complex logic of the package
version selection in a single place on the backend.

Currently, it installs or upgrades (if already installed) two packages:
`endpoint` and `security_detection_engine`.

The response looks like this:

```json5
{
  packages: [
    {
      name: 'detection_engine',
      version: '1.0.0',
      status: 'installed',
    },
    {
      name: 'endpoint',
      version: '1.0.0',
      status: 'already_installed',
    },
  ],
}
```

We call this endpoint from Kibana every time a user lands on any
security solution page. The endpoint checks if the required packages are
missing or if a newer version is available. If so, the newer version is
installed, and the package status will be `installed` in the response.
If all packages are up-to-date, the package status will be
`already_installed` in the response. This allows us to invalidate
prebuilt rule endpoints more efficiently and avoid sending extra
requests from Kibana:

```ts
if (
  response?.packages.find((pkg) => pkg.name === PREBUILT_RULES_PACKAGE_NAME)?.status === 'installed'
) {
  // Invalidate other pre-packaged rules related queries. We need to do
  // that only if the prebuilt rules package was installed, indicating
  // that there might be new rules to install.
  invalidatePrePackagedRulesStatus();
  invalidatePrebuiltRulesInstallReview();
  invalidatePrebuiltRulesUpdateReview();
}
```

The performance gain is that we do not invalidate prebuilt rules when
the package is already installed.

Previously:
`Fetch rules initially -> Upgrade rules package -(always)-> Re-fetch
rules`

Now:
`Fetch rules initially -> Upgrade rules package -(only if there's a new
package version)-> Re-fetch rules`

This will result in fewer redundant API requests from Kibana.

x-pack/plugins/fleet/server/services/epm/package_service.ts

@@ -40,7 +40,7 @@ import type { InstallResult } from '../../../common';
 
 import { appContextService } from '..';
 
-import type { CustomPackageDatasetConfiguration } from './packages/install';
+import type { CustomPackageDatasetConfiguration, EnsurePackageResult } from './packages/install';
 
 import type { FetchFindLatestPackageOptions } from './registry';
 import { getPackageFieldsMetadata } from './registry';
@@ -73,7 +73,7 @@ export interface PackageClient {
     pkgVersion?: string;
     spaceId?: string;
     force?: boolean;
-  }): Promise<Installation>;
+  }): Promise<EnsurePackageResult>;
 
   installPackage(options: {
     pkgName: string;
@@ -201,7 +201,7 @@ class PackageClientImpl implements PackageClient {
     pkgVersion?: string;
     spaceId?: string;
     force?: boolean;
-  }): Promise<Installation> {
+  }): Promise<EnsurePackageResult> {
     await this.#runPreflight(INSTALL_PACKAGES_AUTHZ);
 
     return ensureInstalledPackage({

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -158,6 +158,11 @@ export async function isPackageVersionOrLaterInstalled(options: {
   });
 }
 
+export interface EnsurePackageResult {
+  status: InstallResultStatus;
+  package: Installation;
+}
+
 export async function ensureInstalledPackage(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;
@@ -166,7 +171,7 @@ export async function ensureInstalledPackage(options: {
   spaceId?: string;
   force?: boolean;
   authorizationHeader?: HTTPAuthorizationHeader | null;
-}): Promise<Installation> {
+}): Promise<EnsurePackageResult> {
   const {
     savedObjectsClient,
     pkgName,
@@ -189,7 +194,10 @@ export async function ensureInstalledPackage(options: {
   });
 
   if (installedPackageResult) {
-    return installedPackageResult.package;
+    return {
+      status: 'already_installed',
+      package: installedPackageResult.package,
+    };
   }
   const pkgkey = Registry.pkgToPkgKey(pkgKeyProps);
   const installResult = await installPackage({
@@ -226,7 +234,10 @@ export async function ensureInstalledPackage(options: {
 
   const installation = await getInstallation({ savedObjectsClient, pkgName });
   if (!installation) throw new FleetError(`Could not get installation for ${pkgName}`);
-  return installation;
+  return {
+    status: 'installed',
+    package: installation,
+  };
 }
 
 export async function handleInstallPackageFailure({

x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts

@@ -383,7 +383,8 @@ async function ensureInstalledIntegrations(
       const { pkgName, installSource } = integration;
 
       if (installSource === 'registry') {
-        const pkg = await packageClient.ensureInstalledPackage({ pkgName });
+        const installation = await packageClient.ensureInstalledPackage({ pkgName });
+        const pkg = installation.package;
         const inputs = await packageClient.getAgentPolicyInputs(pkg.name, pkg.version);
         const { packageInfo } = await packageClient.getPackage(pkg.name, pkg.version);
 

x-pack/plugins/observability_solution/synthetics/server/routes/synthetics_service/install_index_templates.ts

@@ -27,9 +27,9 @@ export async function installSyntheticsIndexTemplates(server: SyntheticsServerSe
     pkgName: 'synthetics',
   });
 
-  if (!installation) {
+  if (!installation.package) {
     return Promise.reject('No package installation found.');
   }
 
-  return installation;
+  return installation.package;
 }

x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules.gen.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Bootstrap Prebuilt Rules API endpoint
+ *   version: 1
+ */
+
+import { z } from 'zod';
+
+export type PackageInstallStatus = z.infer<typeof PackageInstallStatus>;
+export const PackageInstallStatus = z.object({
+  /**
+   * The name of the package
+   */
+  name: z.string(),
+  /**
+   * The version of the package
+   */
+  version: z.string(),
+  /**
+   * The status of the package installation
+   */
+  status: z.enum(['installed', 'already_installed']),
+});
+
+export type BootstrapPrebuiltRulesResponse = z.infer<typeof BootstrapPrebuiltRulesResponse>;
+export const BootstrapPrebuiltRulesResponse = z.object({
+  /**
+   * The list of packages that were installed or upgraded
+   */
+  packages: z.array(PackageInstallStatus),
+});

x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules.schema.yaml

@@ -0,0 +1,51 @@
+openapi: 3.0.0
+info:
+  title: Bootstrap Prebuilt Rules API endpoint
+  version: '1'
+paths:
+  /internal/detection_engine/prebuilt_rules/_bootstrap:
+    post:
+      x-labels: [ess, serverless]
+      x-codegen-enabled: true
+      operationId: BootstrapPrebuiltRules
+      summary: Bootstrap Prebuilt Rules
+      description: Ensures that the packages needed for prebuilt detection rules to work are installed and up to date
+      tags:
+        - Prebuilt Rules API
+      responses:
+        200:
+          description: Indicates a successful call
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  packages:
+                    type: array
+                    description: The list of packages that were installed or upgraded
+                    items:
+                      $ref: '#/components/schemas/PackageInstallStatus'
+                required:
+                  - packages
+
+components:
+  schemas:
+    PackageInstallStatus:
+      type: object
+      properties:
+        name:
+          type: string
+          description: The name of the package
+        version:
+          type: string
+          description: The version of the package
+        status:
+          type: string
+          description: The status of the package installation
+          enum:
+            - installed
+            - already_installed
+      required:
+        - name
+        - version
+        - status

x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/urls.ts

@@ -10,14 +10,15 @@ import {
   INTERNAL_DETECTION_ENGINE_URL as INTERNAL,
 } from '../../../constants';
 
-const OLD_BASE_URL = `${RULES}/prepackaged` as const;
-const NEW_BASE_URL = `${INTERNAL}/prebuilt_rules` as const;
+const LEGACY_BASE_URL = `${RULES}/prepackaged` as const;
+const BASE_URL = `${INTERNAL}/prebuilt_rules` as const;
 
-export const PREBUILT_RULES_URL = OLD_BASE_URL;
-export const PREBUILT_RULES_STATUS_URL = `${OLD_BASE_URL}/_status` as const;
+export const PREBUILT_RULES_URL = LEGACY_BASE_URL;
+export const PREBUILT_RULES_STATUS_URL = `${LEGACY_BASE_URL}/_status` as const;
 
-export const GET_PREBUILT_RULES_STATUS_URL = `${NEW_BASE_URL}/status` as const;
-export const REVIEW_RULE_UPGRADE_URL = `${NEW_BASE_URL}/upgrade/_review` as const;
-export const PERFORM_RULE_UPGRADE_URL = `${NEW_BASE_URL}/upgrade/_perform` as const;
-export const REVIEW_RULE_INSTALLATION_URL = `${NEW_BASE_URL}/installation/_review` as const;
-export const PERFORM_RULE_INSTALLATION_URL = `${NEW_BASE_URL}/installation/_perform` as const;
+export const GET_PREBUILT_RULES_STATUS_URL = `${BASE_URL}/status` as const;
+export const BOOTSTRAP_PREBUILT_RULES_URL = `${BASE_URL}/_bootstrap` as const;
+export const REVIEW_RULE_UPGRADE_URL = `${BASE_URL}/upgrade/_review` as const;
+export const PERFORM_RULE_UPGRADE_URL = `${BASE_URL}/upgrade/_perform` as const;
+export const REVIEW_RULE_INSTALLATION_URL = `${BASE_URL}/installation/_review` as const;
+export const PERFORM_RULE_INSTALLATION_URL = `${BASE_URL}/installation/_perform` as const;

x-pack/plugins/security_solution/common/detection_engine/constants.ts

@@ -29,6 +29,7 @@ export enum RULE_PREVIEW_FROM {
 }
 
 export const PREBUILT_RULES_PACKAGE_NAME = 'security_detection_engine';
+export const ENDPOINT_PACKAGE_NAME = 'endpoint';
 
 /**
  * Rule signature id (`rule.rule_id`) of the prebuilt "Endpoint Security" rule.

x-pack/plugins/security_solution/public/detection_engine/rule_management/api/api.ts

@@ -13,9 +13,6 @@ import { INTERNAL_ALERTING_API_FIND_RULES_PATH } from '@kbn/alerting-plugin/comm
 import { BASE_ACTION_API_PATH } from '@kbn/actions-plugin/common';
 import type { ActionType, AsApiContract } from '@kbn/actions-plugin/common';
 import type { ActionResult } from '@kbn/actions-plugin/server';
-import type { BulkInstallPackagesResponse } from '@kbn/fleet-plugin/common';
-import { epmRouteService } from '@kbn/fleet-plugin/common';
-import type { InstallPackageResponse } from '@kbn/fleet-plugin/common/types';
 import { convertRulesFilterToKQL } from '../../../../common/detection_engine/rule_management/rule_filtering';
 import type {
   UpgradeSpecificRulesRequest,
@@ -48,6 +45,7 @@ import {
 } from '../../../../common/constants';
 
 import {
+  BOOTSTRAP_PREBUILT_RULES_URL,
   GET_PREBUILT_RULES_STATUS_URL,
   PERFORM_RULE_INSTALLATION_URL,
   PERFORM_RULE_UPGRADE_URL,
@@ -81,6 +79,7 @@ import type {
   RulesSnoozeSettingsMap,
   UpdateRulesProps,
 } from '../logic/types';
+import type { BootstrapPrebuiltRulesResponse } from '../../../../common/api/detection_engine/prebuilt_rules/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules.gen';
 
 /**
  * Create provided Rule
@@ -594,66 +593,6 @@ export const addRuleExceptions = async ({
     }
   );
 
-export interface InstallFleetPackageProps {
-  packageName: string;
-  packageVersion: string;
-  prerelease?: boolean;
-  force?: boolean;
-}
-
-/**
- * Install a Fleet package from the registry
- *
- * @param packageName Name of the package to install
- * @param packageVersion Version of the package to install
- * @param prerelease Whether to install a prerelease version of the package
- * @param force Whether to force install the package. If false, the package will only be installed if it is not already installed
- *
- * @returns The response from the Fleet API
- */
-export const installFleetPackage = ({
-  packageName,
-  packageVersion,
-  prerelease = false,
-  force = true,
-}: InstallFleetPackageProps): Promise<InstallPackageResponse> => {
-  return KibanaServices.get().http.post<InstallPackageResponse>(
-    epmRouteService.getInstallPath(packageName, packageVersion),
-    {
-      query: { prerelease },
-      version: '2023-10-31',
-      body: JSON.stringify({ force }),
-    }
-  );
-};
-
-export interface BulkInstallFleetPackagesProps {
-  packages: string[];
-  prerelease?: boolean;
-}
-
-/**
- * Install multiple Fleet packages from the registry
- *
- * @param packages Array of package names to install
- * @param prerelease Whether to install prerelease versions of the packages
- *
- * @returns The response from the Fleet API
- */
-export const bulkInstallFleetPackages = ({
-  packages,
-  prerelease = false,
-}: BulkInstallFleetPackagesProps): Promise<BulkInstallPackagesResponse> => {
-  return KibanaServices.get().http.post<BulkInstallPackagesResponse>(
-    epmRouteService.getBulkInstallPath(),
-    {
-      query: { prerelease },
-      version: '2023-10-31',
-      body: JSON.stringify({ packages }),
-    }
-  );
-};
-
 /**
  * NEW PREBUILT RULES ROUTES START HERE! 👋
  * USE THESE ONES! THEY'RE THE NICE ONES, PROMISE!
@@ -759,3 +698,9 @@ export const performUpgradeSpecificRules = async (
       pick_version: 'TARGET', // Setting fixed 'TARGET' temporarily for Milestone 2
     }),
   });
+
+export const bootstrapPrebuiltRules = async (): Promise<BootstrapPrebuiltRulesResponse> =>
+  KibanaServices.get().http.fetch(BOOTSTRAP_PREBUILT_RULES_URL, {
+    method: 'POST',
+    version: '1',
+  });

x-pack/plugins/security_solution/public/detection_engine/rule_management/api/hooks/use_bulk_install_fleet_packages_mutation.ts → x-pack/plugins/security_solution/public/detection_engine/rule_management/api/hooks/use_bootstrap_prebuilt_rules.ts

@@ -4,39 +4,37 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { EPM_API_ROUTES } from '@kbn/fleet-plugin/common';
-import type { BulkInstallPackagesResponse } from '@kbn/fleet-plugin/common/types';
 import type { UseMutationOptions } from '@tanstack/react-query';
 import { useMutation } from '@tanstack/react-query';
+import { BOOTSTRAP_PREBUILT_RULES_URL } from '../../../../../common/api/detection_engine';
+import type { BootstrapPrebuiltRulesResponse } from '../../../../../common/api/detection_engine/prebuilt_rules/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules.gen';
 import { PREBUILT_RULES_PACKAGE_NAME } from '../../../../../common/detection_engine/constants';
-import type { BulkInstallFleetPackagesProps } from '../api';
-import { bulkInstallFleetPackages } from '../api';
+import { bootstrapPrebuiltRules } from '../api';
 import { useInvalidateFetchPrebuiltRulesInstallReviewQuery } from './prebuilt_rules/use_fetch_prebuilt_rules_install_review_query';
 import { useInvalidateFetchPrebuiltRulesStatusQuery } from './prebuilt_rules/use_fetch_prebuilt_rules_status_query';
 import { useInvalidateFetchPrebuiltRulesUpgradeReviewQuery } from './prebuilt_rules/use_fetch_prebuilt_rules_upgrade_review_query';
 
-export const BULK_INSTALL_FLEET_PACKAGES_MUTATION_KEY = [
-  'POST',
-  EPM_API_ROUTES.BULK_INSTALL_PATTERN,
-];
+export const BOOTSTRAP_PREBUILT_RULES_KEY = ['POST', BOOTSTRAP_PREBUILT_RULES_URL];
 
-export const useBulkInstallFleetPackagesMutation = (
-  options?: UseMutationOptions<BulkInstallPackagesResponse, Error, BulkInstallFleetPackagesProps>
+export const useBootstrapPrebuiltRulesMutation = (
+  options?: UseMutationOptions<BootstrapPrebuiltRulesResponse, Error>
 ) => {
   const invalidatePrePackagedRulesStatus = useInvalidateFetchPrebuiltRulesStatusQuery();
   const invalidatePrebuiltRulesInstallReview = useInvalidateFetchPrebuiltRulesInstallReviewQuery();
   const invalidatePrebuiltRulesUpdateReview = useInvalidateFetchPrebuiltRulesUpgradeReviewQuery();
 
-  return useMutation((props: BulkInstallFleetPackagesProps) => bulkInstallFleetPackages(props), {
+  return useMutation(() => bootstrapPrebuiltRules(), {
     ...options,
-    mutationKey: BULK_INSTALL_FLEET_PACKAGES_MUTATION_KEY,
+    mutationKey: BOOTSTRAP_PREBUILT_RULES_KEY,
     onSettled: (...args) => {
       const response = args[0];
-      const rulesPackage = response?.items.find(
-        (item) => item.name === PREBUILT_RULES_PACKAGE_NAME
-      );
-      if (rulesPackage && 'result' in rulesPackage && rulesPackage.result.status === 'installed') {
-        // The rules package was installed/updated, so invalidate the pre-packaged rules status query
+      if (
+        response?.packages.find((pkg) => pkg.name === PREBUILT_RULES_PACKAGE_NAME)?.status ===
+        'installed'
+      ) {
+        // Invalidate other pre-packaged rules related queries. We need to do
+        // that only in case the prebuilt rules package was installed indicating
+        // that there might be new rules to install.
         invalidatePrePackagedRulesStatus();
         invalidatePrebuiltRulesInstallReview();
         invalidatePrebuiltRulesUpdateReview();