diff --git a/x-pack/plugins/observability_solution/observability_onboarding/kibana.jsonc b/x-pack/plugins/observability_solution/observability_onboarding/kibana.jsonc index 35d206123943bd..79eb387c2486c0 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_onboarding/kibana.jsonc @@ -1,22 +1,37 @@ { "type": "plugin", "id": "@kbn/observability-onboarding-plugin", - "owner": ["@elastic/obs-ux-logs-team", "@elastic/obs-ux-onboarding-team"], + "owner": [ + "@elastic/obs-ux-logs-team", + "@elastic/obs-ux-onboarding-team" + ], "plugin": { "id": "observabilityOnboarding", "server": true, "browser": true, - "configPath": ["xpack", "observability_onboarding"], + "configPath": [ + "xpack", + "observability_onboarding" + ], "requiredPlugins": [ "data", "observability", "observabilityShared", "discover", "share", - "fleet" + "fleet", + "security" + ], + "optionalPlugins": [ + "cloud", + "cloudExperiments", + "usageCollection" + ], + "requiredBundles": [ + "kibanaReact" ], - "optionalPlugins": ["cloud", "cloudExperiments", "usageCollection"], - "requiredBundles": ["kibanaReact"], - "extraPublicDirs": ["common"] + "extraPublicDirs": [ + "common" + ] } -} +} \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/app.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/app.tsx index 2b928b37f471b9..835233b424c6fd 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/app.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/app.tsx @@ -22,6 +22,7 @@ import { ObservabilityOnboardingHeaderActionMenu } from './shared/header_action_ import { ObservabilityOnboardingPluginSetupDeps, ObservabilityOnboardingPluginStartDeps, + ObservabilityOnboardingContextValue, } from '../plugin'; import { ObservabilityOnboardingFlow } from './observability_onboarding_flow'; @@ -40,14 +41,13 @@ export const breadcrumbsApp = { export function ObservabilityOnboardingAppRoot({ appMountParameters, core, - deps, - corePlugins: { observability, data }, + corePlugins, config, }: { appMountParameters: AppMountParameters; } & RenderAppProps) { const { history, setHeaderActionMenu, theme$ } = appMountParameters; - const plugins = { ...deps }; + const services: ObservabilityOnboardingContextValue = { ...core, ...corePlugins, config }; const renderFeedbackLinkAsPortal = !config.serverless.enabled; @@ -63,15 +63,7 @@ export function ObservabilityOnboardingAppRoot({ application: core.application, }} > - + + + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/auto_detect_panel.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/auto_detect_panel.tsx new file mode 100644 index 00000000000000..f9d1c7c4df6081 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/auto_detect_panel.tsx @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FunctionComponent } from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiPanel, + EuiSteps, + EuiCodeBlock, + EuiSpacer, + EuiSkeletonText, + EuiBadge, + EuiFlexGroup, + EuiFlexItem, + EuiText, + EuiImage, + EuiSkeletonRectangle, + useGeneratedHtmlId, +} from '@elastic/eui'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { + type SingleDatasetLocatorParams, + SINGLE_DATASET_LOCATOR_ID, +} from '@kbn/deeplinks-observability/locators'; +import { type DashboardLocatorParams } from '@kbn/dashboard-plugin/public'; +import { DASHBOARD_APP_LOCATOR } from '@kbn/deeplinks-analytics'; +import { getAutoDetectCommand } from './get_auto_detect_command'; +import { useOnboardingFlow } from './use_onboarding_flow'; +import { ProgressIndicator } from '../shared/progress_indicator'; +import { AccordionWithIcon } from '../shared/accordion_with_icon'; +import { type ObservabilityOnboardingContextValue } from '../../../plugin'; +import { EmptyPrompt } from '../shared/empty_prompt'; +import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button'; +import { LocatorButtonEmpty } from '../shared/locator_button_empty'; + +export const AutoDetectPanel: FunctionComponent = () => { + const { + services: { http }, + } = useKibana(); + const { status, data, error, refetch, installedIntegrations } = useOnboardingFlow(); + const command = data ? getAutoDetectCommand(data) : undefined; + const accordionId = useGeneratedHtmlId({ prefix: 'accordion' }); + + if (error) { + return ; + } + + const registryIntegrations = installedIntegrations.filter( + (integration) => integration.installSource === 'registry' + ); + const customIntegrations = installedIntegrations.filter( + (integration) => integration.installSource === 'custom' + ); + + return ( + + + +

+ {i18n.translate( + 'xpack.observability_onboarding.autoDetectPanel.p.wellScanYourHostLabel', + { + defaultMessage: "We'll scan your host for logs and metrics, including:", + } + )} +

+ + + + {['Apache', 'Docker', 'Nginx', 'System', 'Custom .log files'].map((item) => ( + + {item} + + ))} + + + {/* Bash syntax highlighting only highlights a few random numbers (badly) so it looks less messy to go with plain text */} + + {command} + + + + + ) : ( + + ), + }, + { + title: i18n.translate( + 'xpack.observability_onboarding.autoDetectPanel.visualizeYourDataLabel', + { defaultMessage: 'Visualize your data' } + ), + status: + status === 'dataReceived' + ? 'complete' + : status === 'awaitingData' || status === 'inProgress' + ? 'current' + : 'incomplete', + children: ( + <> + {status === 'dataReceived' ? ( + + ) : status === 'awaitingData' ? ( + + ) : status === 'inProgress' ? ( + + ) : null} + {(status === 'awaitingData' || status === 'dataReceived') && + installedIntegrations.length > 0 ? ( + <> + + {registryIntegrations.map((integration) => ( + + + + {status === 'dataReceived' ? ( + + ) : ( + + )} + + +
    + {integration.kibanaAssets + .filter((asset) => asset.type === 'dashboard') + .map((dashboard) => ( +
  • + + locator={DASHBOARD_APP_LOCATOR} + params={{ dashboardId: dashboard.id }} + target="_blank" + iconType="dashboardApp" + isDisabled={status !== 'dataReceived'} + flush="left" + size="s" + > + {dashboard.attributes.title} + +
    • + ))} +
+ + + + ))} + {customIntegrations.length > 0 && ( + +
    + {customIntegrations.map((integration) => + integration.dataStreams.map((datastream) => ( +
  • + + locator={SINGLE_DATASET_LOCATOR_ID} + params={{ + integration: integration.pkgName, + dataset: datastream.dataset, + }} + target="_blank" + iconType="document" + isDisabled={status !== 'dataReceived'} + flush="left" + size="s" + > + {integration.pkgName} + +
    • + )) + )} +
+ + )} + + ) : null} + + ), + }, + ]} + /> + + ); +}; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/get_auto_detect_command.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/get_auto_detect_command.tsx new file mode 100644 index 00000000000000..d9e695b8c10f04 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/get_auto_detect_command.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { flatten, zip } from 'lodash'; +import { useOnboardingFlow } from './use_onboarding_flow'; + +export function getAutoDetectCommand( + options: NonNullable['data']> +) { + const scriptName = 'auto_detect.sh'; + return oneLine` + curl ${options.scriptDownloadUrl} -so ${scriptName} && + sudo bash ${scriptName} + --id=${options.onboardingFlow.id} + --kibana-url=${options.kibanaUrl} + --install-key=${options.installApiKey} + --ingest-key=${options.ingestApiKey} + --ea-version=${options.elasticAgentVersion} + `; +} +function oneLine(parts: TemplateStringsArray, ...args: string[]) { + const str = flatten(zip(parts, args)).join(''); + return str.replace(/\s+/g, ' ').trim(); +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/get_installed_integrations.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/get_installed_integrations.tsx new file mode 100644 index 00000000000000..aa2ab268f33ad7 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/get_installed_integrations.tsx @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InstallIntegrationsStepPayload } from '../../../../server/routes/types'; +import type { ObservabilityOnboardingFlow } from '../../../../server/saved_objects/observability_onboarding_status'; +import type { InstalledIntegration } from '../../../../server/routes/types'; + +export function getInstalledIntegrations( + data: Pick | undefined +): InstalledIntegration[] { + return (data?.progress['install-integrations']?.payload as InstallIntegrationsStepPayload) ?? []; +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/get_onboarding_status.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/get_onboarding_status.tsx new file mode 100644 index 00000000000000..bb6e58143e7ddb --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/get_onboarding_status.tsx @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ObservabilityOnboardingFlow } from '../../../../server/saved_objects/observability_onboarding_status'; + +export type ObservabilityOnboardingFlowStatus = + | 'notStarted' + | 'inProgress' + | 'awaitingData' + | 'dataReceived'; + +/** + * Returns the current status of the onboarding flow: + * + * - `notStarted`: No progress has been made. + * - `inProgress`: The user is running the installation command on the host. + * - `awaitingData`: The installation has completed and we are waiting for data to be ingested. + * - `dataReceived`: Data has been ingested - The Agent is up and running. + */ +export function getOnboardingStatus( + data: Pick | undefined +): ObservabilityOnboardingFlowStatus { + if (!data) { + return 'notStarted'; + } + return data.progress['logs-ingest']?.status === 'complete' + ? 'dataReceived' + : data.progress['logs-ingest']?.status === 'loading' + ? 'awaitingData' + : Object.values(data.progress).some((step) => step.status !== 'incomplete') + ? 'inProgress' + : 'notStarted'; +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/index.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/index.tsx new file mode 100644 index 00000000000000..16dab0dabdfeae --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/index.tsx @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { AutoDetectPanel } from './auto_detect_panel'; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/use_onboarding_flow.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/use_onboarding_flow.tsx new file mode 100644 index 00000000000000..50f5636dd84b51 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/use_onboarding_flow.tsx @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import useInterval from 'react-use/lib/useInterval'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import useAsync from 'react-use/lib/useAsync'; +import { type AssetSOObject, type GetBulkAssetsResponse } from '@kbn/fleet-plugin/common'; +import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; +import { getOnboardingStatus } from './get_onboarding_status'; +import { getInstalledIntegrations } from './get_installed_integrations'; +import { type ObservabilityOnboardingContextValue } from '../../../plugin'; + +export function useOnboardingFlow() { + const { + services: { fleet }, + } = useKibana(); + + // Create onboarding session + const { data, error, refetch } = useFetcher( + (callApi) => + callApi('POST /internal/observability_onboarding/flow', { + params: { + body: { + name: 'auto-detect', + }, + }, + }), + [], + { showToastOnError: false } + ); + + const onboardingId = data?.onboardingFlow.id; + + // Fetch onboarding progress + const { + data: progressData, + status: progressStatus, + refetch: refetchProgress, + } = useFetcher( + (callApi) => { + if (onboardingId) { + return callApi('GET /internal/observability_onboarding/flow/{onboardingId}/progress', { + params: { path: { onboardingId } }, + }); + } + }, + [onboardingId] + ); + + const status = getOnboardingStatus(progressData); + const installedIntegrations = getInstalledIntegrations(progressData); + + // Fetch metadata for installed Kibana assets + const assetsState = useAsync(async () => { + if (installedIntegrations.length === 0) { + return []; + } + const assetsMetadata = await fleet.hooks.epm.getBulkAssets({ + assetIds: installedIntegrations + .map((integration) => integration.kibanaAssets) + .flat() as AssetSOObject[], + }); + return installedIntegrations.map((integration) => { + return { + ...integration, + // Enrich installed Kibana assets with metadata from Fleet API (e.g. title, description, etc.) + kibanaAssets: integration.kibanaAssets.reduce( + (acc, asset) => { + const assetWithMetadata = assetsMetadata.data?.items.find(({ id }) => id === asset.id); + if (assetWithMetadata) { + acc.push(assetWithMetadata); + } + return acc; + }, + [] + ), + }; + }); + }, [installedIntegrations.length]); // eslint-disable-line react-hooks/exhaustive-deps + + useInterval( + refetchProgress, + progressStatus === FETCH_STATUS.SUCCESS && status !== 'dataReceived' ? 3000 : null + ); + + return { + data, + error, + refetch, + status, + installedIntegrations: assetsState.value ?? [], + }; +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/accordion_with_icon.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/accordion_with_icon.tsx new file mode 100644 index 00000000000000..9301ddb9a78a4b --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/accordion_with_icon.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FunctionComponent } from 'react'; +import { + EuiAccordion, + EuiIcon, + EuiTitle, + EuiFlexGroup, + EuiFlexItem, + type EuiAccordionProps, +} from '@elastic/eui'; + +interface AccordionWithIconProps + extends Omit { + title: string; + iconType: string; +} +export const AccordionWithIcon: FunctionComponent = ({ + title, + iconType, + children, + ...rest +}) => { + return ( + + + + + + +

{title}

+ + + + } + buttonProps={{ paddingSize: 'l' }} + borders="horizontal" + paddingSize="none" + > +
{children}
+ + ); +}; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/copy_to_clipboard_button.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/copy_to_clipboard_button.tsx new file mode 100644 index 00000000000000..770efa96b0fcca --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/copy_to_clipboard_button.tsx @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FunctionComponent } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiCopy, EuiButton, type EuiButtonProps } from '@elastic/eui'; + +interface CopyToClipboardButtonProps extends Omit { + textToCopy: string; +} + +export const CopyToClipboardButton: FunctionComponent = ({ + textToCopy, + children, + ...rest +}) => { + return ( + + {(copyToClipboard) => ( + + {children ?? + i18n.translate( + 'xpack.observability_onboarding.copyToClipboardButton.copyToClipboardButtonLabel', + { defaultMessage: 'Copy to clipboard' } + )} + + )} + + ); +}; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/empty_prompt.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/empty_prompt.tsx new file mode 100644 index 00000000000000..75d72e1ff0e5d7 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/empty_prompt.tsx @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FunctionComponent } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiButton, EuiEmptyPrompt } from '@elastic/eui'; +import type { IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; + +interface EmptyPromptProps { + error: IHttpFetchError; + onRetryClick(): void; +} +export const EmptyPrompt: FunctionComponent = ({ error, onRetryClick }) => { + if (error.response?.status === 403) { + return ( + + {i18n.translate( + 'xpack.observability_onboarding.autoDetectPanel.h2.contactYourAdministratorForLabel', + { defaultMessage: 'Contact your administrator for access' } + )} + + } + body={ +

+ {i18n.translate( + 'xpack.observability_onboarding.autoDetectPanel.p.toInstallIntegrationsAndLabel', + { + defaultMessage: + 'To install integrations and ingest data, you need additional privileges.', + } + )} +

+ } + /> + ); + } + + return ( + + {i18n.translate( + 'xpack.observability_onboarding.autoDetectPanel.h2.unableToInitiateDataLabel', + { defaultMessage: 'Unable to load content' } + )} + + } + body={ +

+ {i18n.translate( + 'xpack.observability_onboarding.autoDetectPanel.p.thereWasAProblemLabel', + { + defaultMessage: + 'There was a problem loading the application. Retry or contact your administrator for help.', + } + )} +

+ } + actions={ + + {i18n.translate( + 'xpack.observability_onboarding.autoDetectPanel.backToSelectionButtonLabel', + { defaultMessage: 'Retry' } + )} + + } + /> + ); +}; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/locator_button_empty.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/locator_button_empty.tsx new file mode 100644 index 00000000000000..73fe406c46e6e8 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/locator_button_empty.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type AnchorHTMLAttributes } from 'react'; +import { EuiButtonEmpty, type EuiButtonEmptyProps } from '@elastic/eui'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import type { SerializableRecord } from '@kbn/utility-types'; +import { type LocatorPublic } from '@kbn/share-plugin/common'; +import { type ObservabilityOnboardingContextValue } from '../../../plugin'; + +type EuiButtonEmptyPropsForAnchor = Extract< + EuiButtonEmptyProps, + AnchorHTMLAttributes +>; + +export interface LocatorButtonEmptyProps + extends Omit { + locator: string | LocatorPublic; + params: Params; +} + +/** + * Same as `EuiButtonEmpty` but uses locators to navigate instead of URLs. + * + * Accepts the following props instead of an `href`: + * - `locator`: Either the URL locator public contract or the ID of the locator if previously registered. + * - `params`: The params to pass to the locator. + * + * Get type safety for `params` by passing the correct type to the generic component. + * + * Example 1: + * + * ```ts + * + * View dashboard + * + * ``` + * + * Example 2: + * + * ```ts + * import { type SingleDatasetLocatorParams, SINGLE_DATASET_LOCATOR_ID } from '@kbn/deeplinks-observability/locators'; + * + * + * locator={SINGLE_DATASET_LOCATOR_ID} + * params={{ + * integration: 'system', + * dataset: 'system.syslog', + * }} + * > + * View in Logs Explorer + * + * ``` + */ +export const LocatorButtonEmpty = ({ + locator, + params, + ...rest +}: LocatorButtonEmptyProps) => { + const { + services: { share }, + } = useKibana(); + + const locatorObj = + typeof locator === 'string' ? share.url.locators.get(locator) : locator; + + return ( + + ); +}; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/progress_indicator.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/progress_indicator.tsx new file mode 100644 index 00000000000000..337ab8172e9713 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/progress_indicator.tsx @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FunctionComponent } from 'react'; +import { + EuiIcon, + EuiFlexGroup, + EuiFlexItem, + type EuiCallOutProps, + EuiCallOut, + EuiLoadingSpinner, +} from '@elastic/eui'; + +interface ProgressIndicatorProps extends EuiCallOutProps { + iconType?: string; + isLoading?: boolean; +} +export const ProgressIndicator: FunctionComponent = ({ + iconType, + isLoading = true, + title, + color = isLoading ? 'primary' : 'success', + ...rest +}) => { + return ( + + {isLoading ? ( + + + + ) : iconType ? ( + + + + ) : null} + {title} + + } + color={color} + {...rest} + /> + ); +}; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh index 718107050335f8..e946fb66535425 100755 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh @@ -87,9 +87,9 @@ selected_unknown_log_file_pattern_array=() excluded_options_string="" selected_unknown_log_file_pattern_tsv_string="" custom_log_file_path_list_tsv_string="" -install_integrations_api_body_string="" elastic_agent_artifact_name="" elastic_agent_config_path="/opt/Elastic/Agent/elastic-agent.yml" +elastic_agent_tmp_config_path="/tmp/elastic-agent-config-template.yml" OS="$(uname)" ARCH="$(uname -m)" @@ -130,14 +130,16 @@ update_step_progress() { --header "x-elastic-internal-origin: Kibana" \ --data "$data" \ --output /dev/null \ - --no-progress-meter + --no-progress-meter \ + --fail } download_elastic_agent() { local download_url="https://artifacts.elastic.co/downloads/beats/elastic-agent/${elastic_agent_artifact_name}.tar.gz" - curl -L -O $download_url --fail + curl -L -O $download_url --silent --fail if [ "$?" -eq 0 ]; then + printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz" update_step_progress "ea-download" "complete" else update_step_progress "ea-download" "danger" "Failed to download Elastic Agent, see script output for error." @@ -149,6 +151,7 @@ extract_elastic_agent() { tar -xzf "${elastic_agent_artifact_name}.tar.gz" if [ "$?" -eq 0 ]; then + printf "\e[1;32m✓\e[0m %s\n" "Archive extracted" update_step_progress "ea-extract" "complete" else update_step_progress "ea-extract" "danger" "Failed to extract Elastic Agent, see script output for error." @@ -157,9 +160,10 @@ extract_elastic_agent() { } install_elastic_agent() { - "./${elastic_agent_artifact_name}/elastic-agent" install -f + "./${elastic_agent_artifact_name}/elastic-agent" install -f -n > /dev/null if [ "$?" -eq 0 ]; then + printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent installed to $(dirname $elastic_agent_config_path)" update_step_progress "ea-install" "complete" else update_step_progress "ea-install" "danger" "Failed to install Elastic Agent, see script output for error." @@ -170,17 +174,14 @@ install_elastic_agent() { wait_for_elastic_agent_status() { local MAX_RETRIES=10 local i=0 - echo -n "." elastic-agent status > /dev/null 2>&1 local ELASTIC_AGENT_STATUS_EXIT_CODE="$?" while [ "$ELASTIC_AGENT_STATUS_EXIT_CODE" -ne 0 ] && [ $i -le $MAX_RETRIES ]; do sleep 1 - echo -n "." elastic-agent status > /dev/null 2>&1 ELASTIC_AGENT_STATUS_EXIT_CODE="$?" ((i++)) done - echo "" if [ "$ELASTIC_AGENT_STATUS_EXIT_CODE" -ne 0 ]; then update_step_progress "ea-status" "warning" "Unable to determine agent status" @@ -214,11 +215,11 @@ backup_elastic_agent_config() { confirmation_reply="${confirmation_reply:-Y}" if [[ "$confirmation_reply" =~ ^[Yy](es)?$ ]]; then - local backup_path="${elastic_agent_config_path%.yml}.$(date +%s).yml" # e.g. /opt/Elastic/Agent/elastic-agent.1712267614.yml + local backup_path="$(pwd)/$(basename "${elastic_agent_config_path%.yml}.$(date +%s).yml")" # e.g. /opt/Elastic/Agent/elastic-agent.1712267614.yml cp $elastic_agent_config_path $backup_path if [ "$?" -eq 0 ]; then - printf "\n\e[1;32m✓\e[0m \e[1m%s\e[0m\n" "Backup saved to $backup_path" + printf "\n\e[1;32m✓\e[0m %s\n" "Backup saved to $backup_path" else update_step_progress "ea-config" "warning" "Failed to backup existing configuration" fail "Failed to backup existing config file - Try manually creating a backup or delete your existing config file before re-running this script" @@ -229,31 +230,56 @@ backup_elastic_agent_config() { fi } -download_elastic_agent_config() { - local decoded_ingest_api_key=$(echo "$ingest_api_key_encoded" | base64 -d) - local tmp_path="/tmp/elastic-agent-config-template.yml" +install_integrations() { + local install_integrations_api_body_string="" + + for item in "${selected_known_integrations_array[@]}"; do + install_integrations_api_body_string+="$item\tregistry\n" + done + + for item in "${selected_unknown_log_file_pattern_array[@]}" "${custom_log_file_path_list_array[@]}"; do + local integration_name=$(generate_custom_integration_name "$item") - update_step_progress "ea-config" "loading" + install_integrations_api_body_string+="$integration_name\tcustom\t$item\n" + done curl --request POST \ - -o $tmp_path \ + -o $elastic_agent_tmp_config_path \ --url "$kibana_api_endpoint/internal/observability_onboarding/flow/$onboarding_flow_id/integrations/install" \ --header "Authorization: ApiKey $install_api_key_encoded" \ --header "Content-Type: text/tab-separated-values" \ + --header "kbn-xsrf: true" \ + --header "x-elastic-internal-origin: Kibana" \ --data "$(echo -e "$install_integrations_api_body_string")" \ - --no-progress-meter + --no-progress-meter \ + --fail - if [ "$?" -ne 0 ]; then - update_step_progress "ea-config" "warning" "Failed to install integrations." - fail "Failed to install integrations." + if [ "$?" -eq 0 ]; then + printf "\n\e[1;32m✓\e[0m %s\n" "Integrations installed" + else + update_step_progress "ea-config" "warning" "Failed to install integrations" + fail "Failed to install integrations" fi +} - sed "s/'\${API_KEY}'/$decoded_ingest_api_key/g" $tmp_path > $elastic_agent_config_path +apply_elastic_agent_config() { + local decoded_ingest_api_key=$(echo "$ingest_api_key_encoded" | base64 -d) + + sed "s/'\${API_KEY}'/$decoded_ingest_api_key/g" $elastic_agent_tmp_config_path > $elastic_agent_config_path + if [ "$?" -eq 0 ]; then + printf "\e[1;32m✓\e[0m %s\n" "Config written to $elastic_agent_config_path" + update_step_progress "ea-config" "complete" + else + update_step_progress "ea-config" "warning" "Failed to configure Elastic Agent" + fail "Failed to configure Elastic Agent" + fi } read_open_log_file_list() { local exclude_patterns=( "^\/Users\/.+?\/Library\/Application Support" + "^\/Users\/.+?\/Library\/Group Containers" + "^\/Users\/.+?\/Library\/Containers" "^\/Users\/.+?\/Library\/Caches" "^\/private" # Excluding all patterns that correspond to known integrations @@ -269,7 +295,7 @@ read_open_log_file_list() { "^\/var\/log\/secure" ) - local list=$(lsof -Fn | grep "\.log$" | awk '/^n/ {print substr($0, 2)}' | sort | uniq) + local list=$(lsof -Fn / | grep "^n.*\.log$" | cut -c2- | sort -u) # Filtering by the exclude patterns while IFS= read -r line; do @@ -385,12 +411,12 @@ function select_list() { fi done - printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Ingest all detected logs?" "[Y/n] (default: Yes): " + printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Continue installation with detected logs?" "[Y/n] (default: Yes): " read confirmation_reply confirmation_reply="${confirmation_reply:-Y}" if [[ ! "$confirmation_reply" =~ ^[Yy](es)?$ ]]; then - printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Exclude logs by listing their index numbers" "(e.g. 1, 2, 3): " + printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m\n" "Exclude logs by listing their index numbers" "(e.g. 1, 2, 3). Press Enter to skip." read exclude_index_list_string IFS=', ' read -r -a exclude_index_list_array <<< "$exclude_index_list_string" @@ -417,6 +443,33 @@ function select_list() { fi fi done + + if [[ -n "$excluded_options_string" ]]; then + echo -e "\nThese logs will not be ingested:" + echo -e "$excluded_options_string" + fi + + printf "\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m\n" "List any additional logs you'd like to ingest" "(e.g. /path1/*.log, /path2/*.log). Press Enter to skip." + read custom_log_file_path_list_string + + IFS=', ' read -r -a custom_log_file_path_list_array <<< "$custom_log_file_path_list_string" + + echo -e "\nYou've selected these logs for ingestion:" + for item in "${selected_known_integrations_array[@]}"; do + printf "\e[32m•\e[0m %s\n" "$(known_integration_title "${item}")" + done + for item in "${selected_unknown_log_file_pattern_array[@]}" "${custom_log_file_path_list_array[@]}"; do + printf "\e[32m•\e[0m %s\n" "$item" + done + + printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Continue installation with selected logs?" "[Y/n] (default: Yes): " + read confirmation_reply + confirmation_reply="${confirmation_reply:-Y}" + + if [[ ! "$confirmation_reply" =~ ^[Yy](es)?$ ]]; then + echo -e "Rerun the script again to select different logs." + exit 1 + fi else selected_known_integrations_array=("${known_integrations_options[@]}") selected_unknown_log_file_pattern_array=("${unknown_logs_options[@]}") @@ -450,70 +503,26 @@ generate_custom_integration_name() { echo "$name" } -build_install_integrations_api_body_string() { - for item in "${selected_known_integrations_array[@]}"; do - install_integrations_api_body_string+="$item\tregistry\n" - done - - for item in "${selected_unknown_log_file_pattern_array[@]}" "${custom_log_file_path_list_array[@]}"; do - local integration_name=$(generate_custom_integration_name "$item") - - install_integrations_api_body_string+="$integration_name\tcustom\t$item\n" - done -} - -echo "Looking for log files..." +printf "\e[1m%s\e[0m\n" "Looking for log files..." +update_step_progress "logs-detect" "loading" detect_known_integrations read_open_log_file_list build_unknown_log_file_patterns - +update_step_progress "logs-detect" "complete" echo -e "\nWe found these logs on your system:" select_list -if [[ -n "$excluded_options_string" ]]; then - echo -e "\nThese logs will not be ingested:" - echo -e "$excluded_options_string" -fi - - -printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m\n" "Add paths to any custom logs we've missed" "(e.g. /path1/*.log, /path2/*.log). Press Enter to skip." -read custom_log_file_path_list_string - -IFS=', ' read -r -a custom_log_file_path_list_array <<< "$custom_log_file_path_list_string" - -echo -e "\nYou've selected these logs to ingest:" -for item in "${selected_known_integrations_array[@]}"; do - printf "• %s\n" "$(known_integration_title "${item}")" -done -for item in "${selected_unknown_log_file_pattern_array[@]}" "${custom_log_file_path_list_array[@]}"; do - printf "• %s\n" "$item" -done - - -printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Continue installation with selected logs?" "[Y/n] (default: Yes): " -read confirmation_reply -confirmation_reply="${confirmation_reply:-Y}" - -if [[ ! "$confirmation_reply" =~ ^[Yy](es)?$ ]]; then - echo -e "Rerun the script again to select different logs." - exit 1 -fi - -build_install_integrations_api_body_string - backup_elastic_agent_config -echo -e "\nDownloading Elastic Agent...\n" +printf "\n\e[1m%s\e[0m\n" "Installing Elastic Agent..." +install_integrations download_elastic_agent extract_elastic_agent - -echo -e "\nInstalling Elastic Agent...\n" install_elastic_agent +apply_elastic_agent_config + +printf "\n\e[1m%s\e[0m\n" "Waiting for healthy status..." wait_for_elastic_agent_status ensure_elastic_agent_healthy -echo -e "\nInstalling integrations...\n" -download_elastic_agent_config - -update_step_progress "ea-config" "complete" printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running. You can now go back to Kibana and check for incoming logs." diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/charts_screen.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/charts_screen.svg new file mode 100644 index 00000000000000..925d71c1740616 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/charts_screen.svg @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/waterfall_screen.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/waterfall_screen.svg new file mode 100644 index 00000000000000..7501d44620a944 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/waterfall_screen.svg @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/plugin.ts b/x-pack/plugins/observability_solution/observability_onboarding/public/plugin.ts index 8afe3b29c30e06..be73b77bd336e5 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/plugin.ts @@ -10,7 +10,10 @@ import { ObservabilityPublicStart, } from '@kbn/observability-plugin/public'; import { - HttpStart, + ObservabilitySharedPluginSetup, + ObservabilitySharedPluginStart, +} from '@kbn/observability-shared-plugin/public'; +import { AppMountParameters, CoreSetup, CoreStart, @@ -20,7 +23,12 @@ import { } from '@kbn/core/public'; import type { CloudExperimentsPluginStart } from '@kbn/cloud-experiments-plugin/common'; import { DataPublicPluginSetup, DataPublicPluginStart } from '@kbn/data-plugin/public'; -import { SharePluginSetup } from '@kbn/share-plugin/public'; +import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/public'; +import { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public'; +import { DiscoverSetup, DiscoverStart } from '@kbn/discover-plugin/public'; +import { FleetSetup, FleetStart } from '@kbn/fleet-plugin/public'; +import { CloudSetup, CloudStart } from '@kbn/cloud-plugin/public'; +import { UsageCollectionSetup, UsageCollectionStart } from '@kbn/usage-collection-plugin/public'; import type { ObservabilityOnboardingConfig } from '../server'; import { PLUGIN_ID } from '../common'; import { ObservabilityOnboardingLocatorDefinition } from './locators/onboarding_locator/locator_definition'; @@ -34,23 +42,30 @@ export type ObservabilityOnboardingPluginStart = void; export interface ObservabilityOnboardingPluginSetupDeps { data: DataPublicPluginSetup; observability: ObservabilityPublicSetup; + observabilityShared: ObservabilitySharedPluginSetup; + discover: DiscoverSetup; share: SharePluginSetup; + fleet: FleetSetup; + security: SecurityPluginSetup; + cloud?: CloudSetup; + usageCollection?: UsageCollectionSetup; } export interface ObservabilityOnboardingPluginStartDeps { - cloudExperiments?: CloudExperimentsPluginStart; - http: HttpStart; data: DataPublicPluginStart; observability: ObservabilityPublicStart; + observabilityShared: ObservabilitySharedPluginStart; + discover: DiscoverStart; + share: SharePluginStart; + fleet: FleetStart; + security: SecurityPluginStart; + cloud?: CloudStart; + usageCollection?: UsageCollectionStart; + cloudExperiments?: CloudExperimentsPluginStart; } -export interface ObservabilityOnboardingPluginContextValue { - core: CoreStart; - plugins: ObservabilityOnboardingPluginSetupDeps; - data: DataPublicPluginStart; - observability: ObservabilityPublicStart; - config: ConfigSchema; -} +export type ObservabilityOnboardingContextValue = CoreStart & + ObservabilityOnboardingPluginStartDeps & { config: ConfigSchema }; export class ObservabilityOnboardingPlugin implements Plugin diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/lib/get_agent_version.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/lib/get_agent_version.ts new file mode 100644 index 00000000000000..c9dd959e6bb75c --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/lib/get_agent_version.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FleetStartContract } from '@kbn/fleet-plugin/server'; + +export function getAgentVersion(fleetStart: FleetStartContract, kibanaVersion: string) { + // If undefined, we will follow fleet's strategy to select latest available version: + // for serverless we will use the latest published version, for statefull we will use + // current Kibana version. If false, irrespective of fleet flags and logic, we are + // explicitly deciding to not append the current version. + const includeCurrentVersion = kibanaVersion.endsWith('-SNAPSHOT') ? false : undefined; + + const agentClient = fleetStart.agentService.asInternalUser; + return agentClient.getLatestAgentAvailableVersion(includeCurrentVersion); +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/lib/get_fallback_urls.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/lib/get_fallback_urls.ts index fc79d7e37cebbc..15185521563a18 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/lib/get_fallback_urls.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/lib/get_fallback_urls.ts @@ -5,10 +5,19 @@ * 2.0. */ -import { CoreStart } from '@kbn/core/server'; +import { CoreSetup } from '@kbn/core/server'; +import { CloudSetup } from '@kbn/cloud-plugin/server'; import { EsLegacyConfigService } from '../services/es_legacy_config_service'; -export function getFallbackKibanaUrl({ http }: CoreStart) { +export function getKibanaUrl(coreSetup: CoreSetup, cloudSetup?: CloudSetup) { + return ( + coreSetup.http.basePath.publicBaseUrl ?? // priority given to server.publicBaseUrl + cloudSetup?.kibanaUrl ?? // then cloud id + getFallbackKibanaUrl(coreSetup) // falls back to local network binding + ); +} + +export function getFallbackKibanaUrl({ http }: CoreSetup) { const basePath = http.basePath; const { protocol, hostname, port } = http.getServerInfo(); return `${protocol}://${hostname}:${port}${basePath diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/get_has_logs.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/get_has_logs.ts index bedd1de0a80da7..7816843bca3dc9 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/get_has_logs.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/get_has_logs.ts @@ -7,52 +7,24 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { termQuery } from '@kbn/observability-plugin/server'; +import type { estypes } from '@elastic/elasticsearch'; import { AGENT_ID } from '../../../common/es_fields'; -import { - LogFilesState, - ObservabilityOnboardingType, - SystemLogsState, -} from '../../saved_objects/observability_onboarding_status'; -import { ElasticAgentStepPayload } from '../types'; - -export async function getHasLogs({ - type, - state, - esClient, - payload, -}: { - type: ObservabilityOnboardingType; - state?: LogFilesState | SystemLogsState; - esClient: ElasticsearchClient; - payload?: ElasticAgentStepPayload; -}) { - if (!state) { - return false; - } +export async function getHasLogs(esClient: ElasticsearchClient, agentId: string) { try { - const { namespace } = state; - const index = - type === 'logFiles' - ? `logs-${(state as LogFilesState).datasetName}-${namespace}` - : [`logs-system.syslog-${namespace}`, `logs-system.auth-${namespace}`]; - - const agentId = payload?.agentId; - - const { hits } = await esClient.search({ - index, + const result = await esClient.search({ + index: ['logs-*', 'metrics-*'], ignore_unavailable: true, + size: 0, terminate_after: 1, - body: { - query: { - bool: { - filter: [...termQuery(AGENT_ID, agentId)], - }, + query: { + bool: { + filter: termQuery(AGENT_ID, agentId), }, }, }); - const total = hits.total as { value: number }; - return total.value > 0; + const { value } = result.hits.total as estypes.SearchTotalHits; + return value > 0; } catch (error) { if (error.statusCode === 404) { return false; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts index b43edf76ce0a5d..c58a5ef257fbb2 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts @@ -12,15 +12,20 @@ import { FleetUnauthorizedError, type PackageClient, } from '@kbn/fleet-plugin/server'; -import type { TemplateAgentPolicyInput } from '@kbn/fleet-plugin/common'; import { dump } from 'js-yaml'; +import { PackageDataStreamTypes } from '@kbn/fleet-plugin/common/types'; import { getObservabilityOnboardingFlow, saveObservabilityOnboardingFlow } from '../../lib/state'; +import type { SavedObservabilityOnboardingFlow } from '../../saved_objects/observability_onboarding_status'; import { ObservabilityOnboardingFlow } from '../../saved_objects/observability_onboarding_status'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { getHasLogs } from './get_has_logs'; - +import { getKibanaUrl } from '../../lib/get_fallback_urls'; +import { hasLogMonitoringPrivileges } from '../logs/api_key/has_log_monitoring_privileges'; +import { createShipperApiKey } from '../logs/api_key/create_shipper_api_key'; +import { createInstallApiKey } from '../logs/api_key/create_install_api_key'; +import { getAgentVersion } from '../../lib/get_agent_version'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; -import { ElasticAgentStepPayload, Integration, StepProgressPayloadRT } from '../types'; +import { ElasticAgentStepPayload, InstalledIntegration, StepProgressPayloadRT } from '../types'; const updateOnboardingFlowRoute = createObservabilityOnboardingServerRoute({ endpoint: 'PUT /internal/observability_onboarding/flow/{onboardingId}', @@ -129,9 +134,7 @@ const getProgressRoute = createObservabilityOnboardingServerRoute({ onboardingId: t.string, }), }), - async handler(resources): Promise<{ - progress: Record; - }> { + async handler(resources): Promise> { const { params: { path: { onboardingId }, @@ -154,21 +157,11 @@ const getProgressRoute = createObservabilityOnboardingServerRoute({ const esClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; - const type = savedObservabilityOnboardingState.type; - if (progress['ea-status']?.status === 'complete') { + const { agentId } = progress['ea-status']?.payload as ElasticAgentStepPayload; try { - const hasLogs = await getHasLogs({ - type, - state: savedObservabilityOnboardingState.state, - esClient, - payload: progress['ea-status']?.payload as ElasticAgentStepPayload, - }); - if (hasLogs) { - progress['logs-ingest'] = { status: 'complete' }; - } else { - progress['logs-ingest'] = { status: 'loading' }; - } + const hasLogs = await getHasLogs(esClient, agentId); + progress['logs-ingest'] = { status: hasLogs ? 'complete' : 'loading' }; } catch (error) { progress['logs-ingest'] = { status: 'warning', message: error.message }; } @@ -180,6 +173,88 @@ const getProgressRoute = createObservabilityOnboardingServerRoute({ }, }); +/** + * This endpoint starts a new onboarding flow and creates two API keys: + * 1. A short-lived API key with privileges to install integrations. + * 2. An API key with privileges to ingest log and metric data used to configure Elastic Agent. + * + * It also returns all required information to download the onboarding script and install the + * Elastic agent. + * + * If the user does not have all necessary privileges a 403 Forbidden response is returned. + * + * This endpoint differs from the existing `POST /internal/observability_onboarding/logs/flow` + * endpoint in that it caters for the auto-detect flow where integrations are detected and installed + * on the host system, rather than in the Kiabana UI. + */ +const createFlowRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'POST /internal/observability_onboarding/flow', + options: { tags: [] }, + params: t.type({ + body: t.type({ + name: t.string, + }), + }), + async handler(resources) { + const { + context, + params: { + body: { name }, + }, + core, + request, + plugins, + kibanaVersion, + } = resources; + const coreStart = await core.start(); + const { + elasticsearch: { client }, + } = await context.core; + const savedObjectsClient = coreStart.savedObjects.getScopedClient(request); + + const hasPrivileges = await hasLogMonitoringPrivileges(client.asCurrentUser); + if (!hasPrivileges) { + throw Boom.forbidden('Unauthorized to create log indices'); + } + + const fleetPluginStart = await plugins.fleet.start(); + const securityPluginStart = await plugins.security.start(); + + const [onboardingFlow, ingestApiKey, installApiKey, elasticAgentVersion] = await Promise.all([ + saveObservabilityOnboardingFlow({ + savedObjectsClient, + observabilityOnboardingState: { + type: 'autoDetect', + state: undefined, + progress: {}, + }, + }), + createShipperApiKey(client.asCurrentUser, name), + securityPluginStart.authc.apiKeys.create(request, createInstallApiKey(name)), + getAgentVersion(fleetPluginStart, kibanaVersion), + ]); + + if (!installApiKey) { + throw Boom.notFound('License does not allow API key creation.'); + } + + const kibanaUrl = getKibanaUrl(core.setup, plugins.cloud?.setup); + const scriptDownloadUrl = new URL( + core.setup.http.staticAssets.getPluginAssetHref('auto_detect.sh'), + kibanaUrl + ).toString(); + + return { + onboardingFlow, + ingestApiKey: ingestApiKey.encoded, + installApiKey: installApiKey.encoded, + elasticAgentVersion, + kibanaUrl, + scriptDownloadUrl, + }; + }, +}); + /** * This endpoints installs the requested integrations and returns the corresponding config file for Elastic Agent. * @@ -239,9 +314,12 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({ }); } - let agentPolicyInputs: TemplateAgentPolicyInput[] = []; + let installedIntegrations: InstalledIntegration[] = []; try { - agentPolicyInputs = await ensureInstalledIntegrations(integrationsToInstall, packageClient); + installedIntegrations = await ensureInstalledIntegrations( + integrationsToInstall, + packageClient + ); } catch (error) { if (error instanceof FleetUnauthorizedError) { return response.forbidden({ @@ -262,10 +340,10 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({ ...savedObservabilityOnboardingState.progress, 'install-integrations': { status: 'complete', - payload: integrationsToInstall, + payload: installedIntegrations, }, }, - } as ObservabilityOnboardingFlow, + }, }); const elasticsearchUrl = plugins.cloud?.setup?.elasticsearchUrl @@ -278,55 +356,89 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({ }, body: generateAgentConfig({ esHost: elasticsearchUrl, - inputs: agentPolicyInputs, + inputs: installedIntegrations.map(({ inputs }) => inputs).flat(), }), }); }, }); +export interface RegistryIntegrationToInstall { + pkgName: string; + installSource: 'registry'; +} +export interface CustomIntegrationToInstall { + pkgName: string; + installSource: 'custom'; + logFilePaths: string[]; +} +export type IntegrationToInstall = RegistryIntegrationToInstall | CustomIntegrationToInstall; + async function ensureInstalledIntegrations( - integrationsToInstall: Integration[], + integrationsToInstall: IntegrationToInstall[], packageClient: PackageClient -) { - const agentPolicyInputs: TemplateAgentPolicyInput[] = []; - for (const integration of integrationsToInstall) { - const { pkgName, installSource } = integration; - if (installSource === 'registry') { - const pkg = await packageClient.ensureInstalledPackage({ pkgName }); - const inputs = await packageClient.getAgentPolicyInputs(pkg.name, pkg.version); - agentPolicyInputs.push(...inputs.filter((input) => input.type !== 'httpjson')); - } else if (installSource === 'custom') { - const input: TemplateAgentPolicyInput = { - id: `filestream-${pkgName}`, - type: 'filestream', - streams: [ +): Promise { + return Promise.all( + integrationsToInstall.map(async (integration) => { + const { pkgName, installSource } = integration; + + if (installSource === 'registry') { + const pkg = await packageClient.ensureInstalledPackage({ pkgName }); + const inputs = await packageClient.getAgentPolicyInputs(pkg.name, pkg.version); + const { packageInfo } = await packageClient.getPackage(pkg.name, pkg.version); + + return { + installSource, + pkgName: pkg.name, + pkgVersion: pkg.version, + title: packageInfo.title, + inputs: inputs.filter((input) => input.type !== 'httpjson'), + dataStreams: + packageInfo.data_streams?.map(({ type, dataset }) => ({ type, dataset })) ?? [], + kibanaAssets: pkg.installed_kibana, + }; + } + + const dataStream = { + type: 'logs', + dataset: pkgName, + }; + const installed: InstalledIntegration = { + installSource, + pkgName, + pkgVersion: '1.0.0', // Custom integrations are always installed as version `1.0.0` + title: pkgName, + inputs: [ { id: `filestream-${pkgName}`, - data_stream: { - type: 'logs', - dataset: pkgName, - }, - paths: integration.logFilePaths, + type: 'filestream', + streams: [ + { + id: `filestream-${pkgName}`, + data_stream: dataStream, + paths: integration.logFilePaths, + }, + ], }, ], + dataStreams: [dataStream], + kibanaAssets: [], }; try { await packageClient.installCustomIntegration({ pkgName, - datasets: [{ name: pkgName, type: 'logs' }], + datasets: [{ name: dataStream.dataset, type: dataStream.type as PackageDataStreamTypes }], }); - agentPolicyInputs.push(input); + return installed; } catch (error) { // If the error is a naming collision, we can assume the integration is already installed and treat this step as successful if (error instanceof NamingCollisionError) { - agentPolicyInputs.push(input); + return installed; } else { throw error; } } - } - } - return agentPolicyInputs; + }) + ); } /** @@ -347,48 +459,46 @@ async function ensureInstalledIntegrations( function parseIntegrationsTSV(tsv: string) { return Object.values( tsv + .trim() .split('\n') .map((line) => line.split('\t', 3)) - .reduce>((acc, [pkgName, installSource, logFilePath]) => { - const key = `${pkgName}-${installSource}`; - if (installSource === 'registry') { - if (logFilePath) { - throw new Error(`Integration '${pkgName}' does not support a file path`); - } - acc[key] = { - pkgName, - installSource, - }; - return acc; - } else if (installSource === 'custom') { - if (!logFilePath) { - throw new Error(`Missing file path for integration: ${pkgName}`); - } - // Append file path if integration is already in the list - const existing = acc[key]; - if (existing && existing.installSource === 'custom') { - existing.logFilePaths.push(logFilePath); + .reduce>( + (acc, [pkgName, installSource, logFilePath]) => { + const key = `${pkgName}-${installSource}`; + if (installSource === 'registry') { + if (logFilePath) { + throw new Error(`Integration '${pkgName}' does not support a file path`); + } + acc[key] = { + pkgName, + installSource, + }; + return acc; + } else if (installSource === 'custom') { + if (!logFilePath) { + throw new Error(`Missing file path for integration: ${pkgName}`); + } + // Append file path if integration is already in the list + const existing = acc[key]; + if (existing && existing.installSource === 'custom') { + existing.logFilePaths.push(logFilePath); + return acc; + } + acc[key] = { + pkgName, + installSource, + logFilePaths: [logFilePath], + }; return acc; } - acc[key] = { - pkgName, - installSource, - logFilePaths: [logFilePath], - }; - return acc; - } - throw new Error(`Invalid install source: ${installSource}`); - }, {}) + throw new Error(`Invalid install source: ${installSource}`); + }, + {} + ) ); } -const generateAgentConfig = ({ - esHost, - inputs = [], -}: { - esHost: string[]; - inputs: TemplateAgentPolicyInput[]; -}) => { +const generateAgentConfig = ({ esHost, inputs = [] }: { esHost: string[]; inputs: unknown[] }) => { return dump({ outputs: { default: { @@ -402,6 +512,7 @@ const generateAgentConfig = ({ }; export const flowRouteRepository = { + ...createFlowRoute, ...updateOnboardingFlowRoute, ...stepProgressUpdateRoute, ...getProgressRoute, diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/api_key/create_install_api_key.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/api_key/create_install_api_key.ts new file mode 100644 index 00000000000000..d97dd6ac6580ca --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/api_key/create_install_api_key.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ALL_SPACES_ID } from '@kbn/spaces-plugin/common/constants'; +import type { CreateAPIKeyParams } from '@kbn/security-plugin/server'; + +/** + * Creates a short lived API key with the necessary permissions to install integrations + */ +export function createInstallApiKey(name: string): CreateAPIKeyParams { + return { + name: `onboarding_install_${name}`, + expiration: '1h', // This API key is only used for initial setup and should be short lived + metadata: { + managed: true, + application: 'logs', + }, + kibana_role_descriptors: { + can_install_integrations: { + elasticsearch: { + cluster: [], + indices: [], + }, + kibana: [ + { + feature: { + fleet: ['all'], + fleetv2: ['all'], // TODO: Remove this once #183020 is resolved + }, + spaces: [ALL_SPACES_ID], + }, + ], + }, + }, + }; +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts index 80814aa308abc5..70a3bf344fee69 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts @@ -13,7 +13,10 @@ export function createShipperApiKey(esClient: ElasticsearchClient, name: string) return esClient.security.createApiKey({ body: { name: `standalone_agent_logs_onboarding_${name}`, - metadata: { application: 'logs' }, + metadata: { + managed: true, + application: 'logs', + }, role_descriptors: { standalone_agent: { cluster, diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/route.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/route.ts index b46b1508ed21b2..4f7c1360dc082d 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/route.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/route.ts @@ -7,7 +7,8 @@ import * as t from 'io-ts'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; -import { getFallbackKibanaUrl } from '../../lib/get_fallback_urls'; +import { getKibanaUrl } from '../../lib/get_fallback_urls'; +import { getAgentVersion } from '../../lib/get_agent_version'; import { hasLogMonitoringPrivileges } from './api_key/has_log_monitoring_privileges'; import { saveObservabilityOnboardingFlow } from '../../lib/state'; import { createShipperApiKey } from './api_key/create_shipper_api_key'; @@ -39,27 +40,12 @@ const installShipperSetupRoute = createObservabilityOnboardingServerRoute({ elasticAgentVersion: string; }> { const { core, plugins, kibanaVersion } = resources; - const coreStart = await core.start(); const fleetPluginStart = await plugins.fleet.start(); - const agentClient = fleetPluginStart.agentService.asInternalUser; - - // If undefined, we will follow fleet's strategy to select latest available version: - // for serverless we will use the latest published version, for statefull we will use - // current Kibana version. If false, irrespective of fleet flags and logic, we are - // explicitly deciding to not append the current version. - const includeCurrentVersion = kibanaVersion.endsWith('-SNAPSHOT') ? false : undefined; - - const elasticAgentVersion = await agentClient.getLatestAgentAvailableVersion( - includeCurrentVersion - ); - - const kibanaUrl = - core.setup.http.basePath.publicBaseUrl ?? // priority given to server.publicBaseUrl - plugins.cloud?.setup?.kibanaUrl ?? // then cloud id - getFallbackKibanaUrl(coreStart); // falls back to local network binding + const elasticAgentVersion = await getAgentVersion(fleetPluginStart, kibanaVersion); + const kibanaUrl = getKibanaUrl(core.setup, plugins.cloud?.setup); const scriptDownloadUrl = new URL( - coreStart.http.staticAssets.getPluginAssetHref('standalone_agent_setup.sh'), + core.setup.http.staticAssets.getPluginAssetHref('standalone_agent_setup.sh'), kibanaUrl ).toString(); diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts index e9ab6b14dab544..de2e7ce65fd2d3 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts @@ -52,19 +52,27 @@ export interface ObservabilityOnboardingRouteCreateOptions { }; } -export const IntegrationRT = t.union([ - t.type({ - pkgName: t.string, - installSource: t.literal('registry'), - }), - t.type({ - pkgName: t.string, - installSource: t.literal('custom'), - logFilePaths: t.array(t.string), - }), -]); +export const IntegrationRT = t.type({ + installSource: t.union([t.literal('registry'), t.literal('custom')]), + pkgName: t.string, + pkgVersion: t.string, + title: t.string, + inputs: t.array(t.unknown), + dataStreams: t.array( + t.type({ + type: t.string, + dataset: t.string, + }) + ), + kibanaAssets: t.array( + t.type({ + type: t.string, + id: t.string, + }) + ), +}); -export type Integration = t.TypeOf; +export type InstalledIntegration = t.TypeOf; export const ElasticAgentStepPayloadRT = t.type({ agentId: t.string, diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/saved_objects/observability_onboarding_status.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/saved_objects/observability_onboarding_status.ts index 297f7f33a9d64a..a7ef942d7ea0a5 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/saved_objects/observability_onboarding_status.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/saved_objects/observability_onboarding_status.ts @@ -23,7 +23,7 @@ export interface SystemLogsState { namespace: string; } -export type ObservabilityOnboardingType = 'logFiles' | 'systemLogs'; +export type ObservabilityOnboardingType = 'logFiles' | 'systemLogs' | 'autoDetect'; type ObservabilityOnboardingFlowState = LogFilesState | SystemLogsState | undefined; @@ -64,8 +64,21 @@ const ElasticAgentStepPayloadSchema = schema.object({ export const InstallIntegrationsStepPayloadSchema = schema.arrayOf( schema.object({ pkgName: schema.string(), - installSource: schema.string(), - logFilePaths: schema.maybe(schema.arrayOf(schema.string())), + pkgVersion: schema.string(), + installSource: schema.oneOf([schema.literal('registry'), schema.literal('custom')]), + inputs: schema.arrayOf(schema.any()), + dataStreams: schema.arrayOf( + schema.object({ + type: schema.string(), + dataset: schema.string(), + }) + ), + kibanaAssets: schema.arrayOf( + schema.object({ + type: schema.string(), + id: schema.string(), + }) + ), }) ); diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/types.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/types.ts index 1c3cbbf26937cc..8eee0943d3590f 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/types.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/types.ts @@ -12,6 +12,7 @@ import { PluginStart as DataPluginStart, } from '@kbn/data-plugin/server'; import { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server'; +import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; @@ -21,6 +22,7 @@ export interface ObservabilityOnboardingPluginSetupDependencies { cloud: CloudSetup; usageCollection: UsageCollectionSetup; fleet: FleetSetupContract; + security: SecurityPluginSetup; } export interface ObservabilityOnboardingPluginStartDependencies { @@ -29,6 +31,7 @@ export interface ObservabilityOnboardingPluginStartDependencies { cloud: CloudStart; usageCollection: undefined; fleet: FleetStartContract; + security: SecurityPluginStart; } // eslint-disable-next-line @typescript-eslint/no-empty-interface diff --git a/x-pack/plugins/observability_solution/observability_onboarding/tsconfig.json b/x-pack/plugins/observability_solution/observability_onboarding/tsconfig.json index 947a1230afd164..5833bba22a6e4d 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_onboarding/tsconfig.json @@ -38,7 +38,14 @@ "@kbn/home-sample-data-tab", "@kbn/react-kibana-context-render", "@kbn/react-kibana-context-theme", - "@kbn/ebt" + "@kbn/discover-plugin", + "@kbn/utility-types", + "@kbn/spaces-plugin", + "@kbn/ebt", + "@kbn/dashboard-plugin", + "@kbn/deeplinks-analytics" ], - "exclude": ["target/**/*"] + "exclude": [ + "target/**/*" + ] }