From d01c1c2dea9b78851b767bbae240a82b5d4cdb3a Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Fri, 27 Jan 2023 11:14:07 +0100 Subject: [PATCH] [Fleet] enabling diagnostics feature flag and changed query for files to use upload_id (#149575) ## Summary Closes https://github.com/elastic/kibana/issues/141074 Enabled feature flag and tweaked implementation to find file by `upload_id` rather than doc id. How to test: - Start local kibana, start Fleet Server, enroll Elastic Agent from local (pull [these changes](https://github.com/elastic/elastic-agent/pull/1703) ) - Click on Request Diagnostics action on the Agent - The diagnostics file should appear on Agent Details / Diagnostics tab. - The action should be completed on Agent activity image image The file metadata and binary can be queried from these indices: ``` GET .fleet-files-agent/_search GET .fleet-file-data-agent/_search ``` Tweaked the implementation so that the pending actions are showing up as soon as the `.fleet-actions` record is created (it can take several minutes until the action result is ready) Plus added a tooltip for error status image ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../fleet/common/experimental_features.ts | 2 +- .../fleet/common/types/models/agent.ts | 2 +- .../components/agent_diagnostics/index.tsx | 6 ++- .../fleet/server/services/agents/uploads.ts | 49 ++++++++++++++----- .../apis/agents/uploads.ts | 3 +- 5 files changed, 45 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts index 7a9ede69c86055..bb28867d3bf05f 100644 --- a/x-pack/plugins/fleet/common/experimental_features.ts +++ b/x-pack/plugins/fleet/common/experimental_features.ts @@ -15,7 +15,7 @@ export const allowedExperimentalValues = Object.freeze({ createPackagePolicyMultiPageLayout: true, packageVerification: true, showDevtoolsRequest: true, - diagnosticFileUploadEnabled: false, + diagnosticFileUploadEnabled: true, experimentalDataStreamSettings: false, displayAgentMetrics: true, showIntegrationsSubcategories: false, diff --git a/x-pack/plugins/fleet/common/types/models/agent.ts b/x-pack/plugins/fleet/common/types/models/agent.ts index b3beb3d6cdec7f..fb7c5137627849 100644 --- a/x-pack/plugins/fleet/common/types/models/agent.ts +++ b/x-pack/plugins/fleet/common/types/models/agent.ts @@ -161,7 +161,7 @@ export interface AgentDiagnostics { name: string; createTime: string; filePath: string; - status: 'READY' | 'AWAITING_UPLOAD' | 'DELETED' | 'IN_PROGRESS'; + status: 'READY' | 'AWAITING_UPLOAD' | 'DELETED' | 'IN_PROGRESS' | 'FAILED'; actionId: string; } diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_diagnostics/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_diagnostics/index.tsx index 7f9bc76799ed30..b0a588a1d12c4f 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_diagnostics/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_diagnostics/index.tsx @@ -6,6 +6,7 @@ */ import type { EuiTableFieldDataColumnType } from '@elastic/eui'; +import { EuiToolTip } from '@elastic/eui'; import { EuiBasicTable, EuiButton, @@ -131,7 +132,10 @@ export const AgentDiagnosticsTab: React.FunctionComponent ) : ( -   + + + +   {currentItem?.name} ); diff --git a/x-pack/plugins/fleet/server/services/agents/uploads.ts b/x-pack/plugins/fleet/server/services/agents/uploads.ts index 7596f9cef7a643..7402eedc840edf 100644 --- a/x-pack/plugins/fleet/server/services/agents/uploads.ts +++ b/x-pack/plugins/fleet/server/services/agents/uploads.ts @@ -34,13 +34,23 @@ export async function getAgentUploads( const getFile = async (fileId: string) => { if (!fileId) return; try { - const file = await esClient.get({ + const fileResponse = await esClient.search({ index: FILE_STORAGE_METADATA_AGENT_INDEX, - id: fileId, + query: { + bool: { + filter: { + term: { upload_id: fileId }, + }, + }, + }, }); + if (fileResponse.hits.total === 0) { + appContextService.getLogger().debug(`No matches for upload_id ${fileId}`); + return; + } return { - id: file._id, - ...(file._source as any)?.file, + id: fileResponse.hits.hits[0]._id, + ...(fileResponse.hits.hits[0]._source as any)?.file, }; } catch (err) { if (err.statusCode === 404) { @@ -56,13 +66,13 @@ export async function getAgentUploads( const results = []; for (const action of actions) { - const file = await getFile(action.fileId); + const file = action.fileId ? await getFile(action.fileId) : undefined; const fileName = file?.name ?? `${moment(action.timestamp!).format('YYYY-MM-DD HH:mm:ss')}.zip`; const filePath = file ? agentRouteService.getAgentFileDownloadLink(file.id, file.name) : ''; const result = { actionId: action.actionId, id: file?.id ?? action.actionId, - status: file?.Status ?? 'IN_PROGRESS', + status: file?.Status ?? (action.error ? 'FAILED' : 'IN_PROGRESS'), name: fileName, createTime: action.timestamp!, filePath, @@ -76,7 +86,7 @@ export async function getAgentUploads( async function _getRequestDiagnosticsActions( esClient: ElasticsearchClient, agentId: string -): Promise> { +): Promise> { const agentActionRes = await esClient.search({ index: AGENT_ACTIONS_INDEX, ignore_unavailable: true, @@ -99,14 +109,17 @@ async function _getRequestDiagnosticsActions( }, }); - const agentActionIds = agentActionRes.hits.hits.map((hit) => hit._source?.action_id as string); + const agentActions = agentActionRes.hits.hits.map((hit) => ({ + actionId: hit._source?.action_id as string, + timestamp: hit._source?.['@timestamp'], + })); - if (agentActionIds.length === 0) { + if (agentActions.length === 0) { return []; } try { - const actionResults = await esClient.search({ + const actionResultsRes = await esClient.search({ index: AGENT_ACTIONS_RESULTS_INDEX, ignore_unavailable: true, size: SO_SEARCH_LIMIT, @@ -115,7 +128,7 @@ async function _getRequestDiagnosticsActions( must: [ { terms: { - action_id: agentActionIds, + action_id: agentActions.map((action) => action.actionId), }, }, { @@ -127,11 +140,21 @@ async function _getRequestDiagnosticsActions( }, }, }); - return actionResults.hits.hits.map((hit) => ({ + const actionResults = actionResultsRes.hits.hits.map((hit) => ({ actionId: hit._source?.action_id as string, timestamp: hit._source?.['@timestamp'], - fileId: hit._source?.data?.file_id as string, + fileId: hit._source?.data?.upload_id as string, + error: hit._source?.error, })); + return agentActions.map((action) => { + const actionResult = actionResults.find((result) => result.actionId === action.actionId); + return { + actionId: action.actionId, + timestamp: actionResult?.timestamp ?? action.timestamp, + fileId: actionResult?.fileId, + error: actionResult?.error, + }; + }); } catch (err) { if (err.statusCode === 404) { // .fleet-actions-results does not yet exist diff --git a/x-pack/test/fleet_api_integration/apis/agents/uploads.ts b/x-pack/test/fleet_api_integration/apis/agents/uploads.ts index 07e44bfa2a641c..c0f32104d24f93 100644 --- a/x-pack/test/fleet_api_integration/apis/agents/uploads.ts +++ b/x-pack/test/fleet_api_integration/apis/agents/uploads.ts @@ -53,7 +53,7 @@ export default function (providerContext: FtrProviderContext) { agent_id: 'agent1', '@timestamp': '2022-10-07T12:00:00.000Z', data: { - file_id: 'file1', + upload_id: 'file1', }, }, }, @@ -67,6 +67,7 @@ export default function (providerContext: FtrProviderContext) { body: { doc_as_upsert: true, doc: { + upload_id: 'file1', file: { ChunkSize: 4194304, extension: 'zip',