[Logs+] End onboarding wizard in default discover (#163218)

closes [#163080](#163080)


## 📝  Summary

This PR navigates to default discover at the end of the onboarding flow
for both custom and system workflows.
It also adds `logs-*` as the default dataview along with a preset filter
for the intended dataset during the onboarding flow.

## ✅  Testing

1. Navigate to the onboarding flow `/app/observabilityOnboarding/`
2. Choose either System logs or Stream log files
3. Go through the onboarding wizard
4. Click the Explore logs button at the end
5. Observe the DataView and Preset Filter after being navigated to
Discover

## 🎥 Demo


https://github.com/elastic/kibana/assets/11225826/5eff74e4-c12a-46e7-968a-6efa34a6a7a9

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/observability_onboarding/common/elastic_agent_logs/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './custom_logs/generate_custom_logs_yml';
+export * from './system_logs/generate_system_logs_yml';

x-pack/plugins/observability_onboarding/common/elastic_agent_logs/system_logs/generate_system_logs_yml.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { dump } from 'js-yaml';
+
+interface SystemLogsStream {
+  id: string;
+  data_stream: {
+    dataset: string;
+    type: string;
+  };
+  paths: string[];
+  exclude_files: string[];
+  multiline: {
+    pattern: string;
+    match: string;
+  };
+  tags?: string[];
+  processors: Array<{
+    add_locale: string | null;
+  }>;
+}
+
+export const generateSystemLogsYml = ({
+  namespace = 'default',
+  apiKey,
+  esHost,
+  uuid,
+}: {
+  namespace?: string;
+  apiKey: string;
+  esHost: string[];
+  uuid: string;
+}) => {
+  return dump({
+    outputs: {
+      default: {
+        type: 'elasticsearch',
+        hosts: esHost,
+        api_key: apiKey,
+      },
+    },
+    inputs: [
+      {
+        id: `system-logs-${uuid}`,
+        type: 'logfile',
+        data_stream: {
+          namespace,
+        },
+        streams: getSystemLogsDataStreams(uuid),
+      },
+    ],
+  });
+};
+
+/*
+ * Utils
+ */
+export const getSystemLogsDataStreams = (
+  uuid: string = ''
+): SystemLogsStream[] => [
+  {
+    id: `logfile-system.auth-${uuid}`,
+    data_stream: {
+      dataset: 'system.auth',
+      type: 'logs',
+    },
+    paths: ['/var/log/auth.log*', '/var/log/secure*'],
+    exclude_files: ['.gz$'],
+    multiline: {
+      pattern: '^s',
+      match: 'after',
+    },
+    tags: ['system-auth'],
+    processors: [
+      {
+        add_locale: null,
+      },
+    ],
+  },
+  {
+    id: `logfile-system.syslog-${uuid}`,
+    data_stream: {
+      dataset: 'system.syslog',
+      type: 'logs',
+    },
+    paths: ['/var/log/messages*', '/var/log/syslog*', '/var/log/system*'],
+    exclude_files: ['.gz$'],
+    multiline: {
+      pattern: '^s',
+      match: 'after',
+    },
+    processors: [
+      {
+        add_locale: null,
+      },
+    ],
+  },
+];

x-pack/plugins/observability_onboarding/kibana.jsonc

@@ -7,7 +7,7 @@
     "server": true,
     "browser": true,
     "configPath": ["xpack", "observability_onboarding"],
-    "requiredPlugins": ["data", "observability", "observabilityShared"],
+    "requiredPlugins": ["data", "observability", "observabilityShared", "discover"],
     "optionalPlugins": ["cloud", "usageCollection"],
     "requiredBundles": ["kibanaReact"],
     "extraPublicDirs": ["common"]

x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx

@@ -14,10 +14,11 @@ import {
   EuiText,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
+import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { default as React, useCallback, useEffect, useState } from 'react';
+import { ObservabilityOnboardingPluginSetupDeps } from '../../../../plugin';
 import { useWizard } from '.';
 import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
-import { useKibanaNavigation } from '../../../../hooks/use_kibana_navigation';
 import {
   ElasticAgentPlatform,
   getElasticAgentSetupCommand,
@@ -34,9 +35,14 @@ import {
 } from '../../../shared/step_panel';
 import { ApiKeyBanner } from './api_key_banner';
 import { BackButton } from './back_button';
+import { getDiscoverNavigationParams } from '../../utils';
 
 export function InstallElasticAgent() {
-  const { navigateToKibanaUrl } = useKibanaNavigation();
+  const {
+    services: {
+      discover: { locator },
+    },
+  } = useKibana<ObservabilityOnboardingPluginSetupDeps>();
   const { goBack, goToStep, getState, setState } = useWizard();
   const wizardState = getState();
   const [elasticAgentPlatform, setElasticAgentPlatform] =
@@ -45,8 +51,10 @@ export function InstallElasticAgent() {
   function onInspect() {
     goToStep('inspect');
   }
-  function onContinue() {
-    navigateToKibanaUrl('/app/logs/stream');
+  async function onContinue() {
+    await locator?.navigate(
+      getDiscoverNavigationParams([wizardState.datasetName])
+    );
   }
 
   function onAutoDownloadConfig() {

x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx

@@ -13,7 +13,10 @@ import {
   EuiText,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
+import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { default as React, useCallback, useEffect, useState } from 'react';
+import { getSystemLogsDataStreams } from '../../../../common/elastic_agent_logs';
+import { ObservabilityOnboardingPluginSetupDeps } from '../../../plugin';
 import { useWizard } from '.';
 import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
 import { useKibanaNavigation } from '../../../hooks/use_kibana_navigation';
@@ -32,8 +35,15 @@ import {
   StepPanelFooter,
 } from '../../shared/step_panel';
 import { ApiKeyBanner } from '../custom_logs/wizard/api_key_banner';
+import { getDiscoverNavigationParams } from '../utils';
 
 export function InstallElasticAgent() {
+  const {
+    services: {
+      discover: { locator },
+    },
+  } = useKibana<ObservabilityOnboardingPluginSetupDeps>();
+
   const { navigateToKibanaUrl } = useKibanaNavigation();
   const { getState, setState } = useWizard();
   const wizardState = getState();
@@ -45,8 +55,12 @@ export function InstallElasticAgent() {
   function onBack() {
     navigateToKibanaUrl('/app/observabilityOnboarding');
   }
-  function onContinue() {
-    navigateToKibanaUrl('/app/logs/stream');
+  async function onContinue() {
+    const dataStreams = getSystemLogsDataStreams();
+    const dataSets = dataStreams.map(
+      (dataSream) => dataSream.data_stream.dataset
+    );
+    await locator?.navigate(getDiscoverNavigationParams(dataSets));
   }
 
   function onAutoDownloadConfig() {

x-pack/plugins/observability_onboarding/public/components/app/utils.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DataViewSpec } from '@kbn/data-views-plugin/common';
+import { DiscoverAppLocatorParams } from '@kbn/discover-plugin/common';
+import { Filter, FilterStateStore } from '@kbn/es-query';
+
+type DiscoverPropertiesToPick = 'dataViewId' | 'dataViewSpec' | 'filters';
+
+type DiscoverNavigationParams = Pick<
+  DiscoverAppLocatorParams,
+  DiscoverPropertiesToPick
+>;
+
+const defaultFilterKey = 'data_stream.dataset';
+const defaultLogsDataViewId = 'logs-*';
+const defaultLogsDataView: DataViewSpec = {
+  id: defaultLogsDataViewId,
+  title: defaultLogsDataViewId,
+};
+
+const getDefaultDatasetFilter = (datasets: string[]): Filter[] => [
+  {
+    meta: {
+      index: defaultLogsDataViewId,
+      key: defaultFilterKey,
+      params: datasets,
+      type: 'phrases',
+    },
+    query: {
+      bool: {
+        minimum_should_match: 1,
+        should: datasets.map((dataset) => ({
+          match_phrase: {
+            [defaultFilterKey]: dataset,
+          },
+        })),
+      },
+    },
+    $state: {
+      store: FilterStateStore.APP_STATE,
+    },
+  },
+];
+
+export const getDiscoverNavigationParams = (
+  datasets: string[]
+): DiscoverNavigationParams => ({
+  dataViewId: defaultLogsDataViewId,
+  dataViewSpec: defaultLogsDataView,
+  filters: getDefaultDatasetFilter(datasets),
+});

x-pack/plugins/observability_onboarding/public/plugin.ts

@@ -23,6 +23,7 @@ import {
   DataPublicPluginSetup,
   DataPublicPluginStart,
 } from '@kbn/data-plugin/public';
+import type { DiscoverSetup } from '@kbn/discover-plugin/public';
 import type { ObservabilityOnboardingConfig } from '../server';
 
 export type ObservabilityOnboardingPluginSetup = void;
@@ -31,6 +32,7 @@ export type ObservabilityOnboardingPluginStart = void;
 export interface ObservabilityOnboardingPluginSetupDeps {
   data: DataPublicPluginSetup;
   observability: ObservabilityPublicSetup;
+  discover: DiscoverSetup;
 }
 
 export interface ObservabilityOnboardingPluginStartDeps {

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts

@@ -7,12 +7,14 @@
 
 import * as t from 'io-ts';
 import { v4 as uuidv4 } from 'uuid';
+import {
+  generateSystemLogsYml,
+  generateCustomLogsYml,
+} from '../../../common/elastic_agent_logs';
 import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key';
 import { getFallbackESUrl } from '../../lib/get_fallback_urls';
 import { getObservabilityOnboardingFlow } from '../../lib/state';
 import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
-import { generateCustomLogsYml } from './custom_logs/generate_custom_logs_yml';
-import { generateSystemLogsYml } from './system_logs/generate_system_logs_yml';
 
 const generateConfig = createObservabilityOnboardingServerRoute({
   endpoint: 'GET /internal/observability_onboarding/elastic_agent/config',

x-pack/plugins/observability_onboarding/tsconfig.json

@@ -14,6 +14,7 @@
   "kbn_references": [
     "@kbn/core",
     "@kbn/data-plugin",
+    "@kbn/discover-plugin",
     "@kbn/kibana-react-plugin",
     "@kbn/observability-plugin",
     "@kbn/i18n",
@@ -29,6 +30,8 @@
     "@kbn/core-http-server",
     "@kbn/security-plugin",
     "@kbn/std",
+    "@kbn/data-views-plugin",
+    "@kbn/es-query",
   ],
   "exclude": [
     "target/**/*",