From d01c1c2dea9b78851b767bbae240a82b5d4cdb3a Mon Sep 17 00:00:00 2001
From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com>
Date: Fri, 27 Jan 2023 11:14:07 +0100
Subject: [PATCH] [Fleet] enabling diagnostics feature flag and changed query
for files to use upload_id (#149575)
## Summary
Closes https://github.com/elastic/kibana/issues/141074
Enabled feature flag and tweaked implementation to find file by
`upload_id` rather than doc id.
How to test:
- Start local kibana, start Fleet Server, enroll Elastic Agent from
local (pull [these
changes](https://github.com/elastic/elastic-agent/pull/1703) )
- Click on Request Diagnostics action on the Agent
- The diagnostics file should appear on Agent Details / Diagnostics tab.
- The action should be completed on Agent activity
The file metadata and binary can be queried from these indices:
```
GET .fleet-files-agent/_search
GET .fleet-file-data-agent/_search
```
Tweaked the implementation so that the pending actions are showing up as
soon as the `.fleet-actions` record is created (it can take several
minutes until the action result is ready)
Plus added a tooltip for error status
### Checklist
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
---
.../fleet/common/experimental_features.ts | 2 +-
.../fleet/common/types/models/agent.ts | 2 +-
.../components/agent_diagnostics/index.tsx | 6 ++-
.../fleet/server/services/agents/uploads.ts | 49 ++++++++++++++-----
.../apis/agents/uploads.ts | 3 +-
5 files changed, 45 insertions(+), 17 deletions(-)
diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts
index 7a9ede69c86055..bb28867d3bf05f 100644
--- a/x-pack/plugins/fleet/common/experimental_features.ts
+++ b/x-pack/plugins/fleet/common/experimental_features.ts
@@ -15,7 +15,7 @@ export const allowedExperimentalValues = Object.freeze({
createPackagePolicyMultiPageLayout: true,
packageVerification: true,
showDevtoolsRequest: true,
- diagnosticFileUploadEnabled: false,
+ diagnosticFileUploadEnabled: true,
experimentalDataStreamSettings: false,
displayAgentMetrics: true,
showIntegrationsSubcategories: false,
diff --git a/x-pack/plugins/fleet/common/types/models/agent.ts b/x-pack/plugins/fleet/common/types/models/agent.ts
index b3beb3d6cdec7f..fb7c5137627849 100644
--- a/x-pack/plugins/fleet/common/types/models/agent.ts
+++ b/x-pack/plugins/fleet/common/types/models/agent.ts
@@ -161,7 +161,7 @@ export interface AgentDiagnostics {
name: string;
createTime: string;
filePath: string;
- status: 'READY' | 'AWAITING_UPLOAD' | 'DELETED' | 'IN_PROGRESS';
+ status: 'READY' | 'AWAITING_UPLOAD' | 'DELETED' | 'IN_PROGRESS' | 'FAILED';
actionId: string;
}
diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_diagnostics/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_diagnostics/index.tsx
index 7f9bc76799ed30..b0a588a1d12c4f 100644
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_diagnostics/index.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_diagnostics/index.tsx
@@ -6,6 +6,7 @@
*/
import type { EuiTableFieldDataColumnType } from '@elastic/eui';
+import { EuiToolTip } from '@elastic/eui';
import {
EuiBasicTable,
EuiButton,
@@ -131,7 +132,10 @@ export const AgentDiagnosticsTab: React.FunctionComponent
) : (
-
+
+
+
+
{currentItem?.name}
);
diff --git a/x-pack/plugins/fleet/server/services/agents/uploads.ts b/x-pack/plugins/fleet/server/services/agents/uploads.ts
index 7596f9cef7a643..7402eedc840edf 100644
--- a/x-pack/plugins/fleet/server/services/agents/uploads.ts
+++ b/x-pack/plugins/fleet/server/services/agents/uploads.ts
@@ -34,13 +34,23 @@ export async function getAgentUploads(
const getFile = async (fileId: string) => {
if (!fileId) return;
try {
- const file = await esClient.get({
+ const fileResponse = await esClient.search({
index: FILE_STORAGE_METADATA_AGENT_INDEX,
- id: fileId,
+ query: {
+ bool: {
+ filter: {
+ term: { upload_id: fileId },
+ },
+ },
+ },
});
+ if (fileResponse.hits.total === 0) {
+ appContextService.getLogger().debug(`No matches for upload_id ${fileId}`);
+ return;
+ }
return {
- id: file._id,
- ...(file._source as any)?.file,
+ id: fileResponse.hits.hits[0]._id,
+ ...(fileResponse.hits.hits[0]._source as any)?.file,
};
} catch (err) {
if (err.statusCode === 404) {
@@ -56,13 +66,13 @@ export async function getAgentUploads(
const results = [];
for (const action of actions) {
- const file = await getFile(action.fileId);
+ const file = action.fileId ? await getFile(action.fileId) : undefined;
const fileName = file?.name ?? `${moment(action.timestamp!).format('YYYY-MM-DD HH:mm:ss')}.zip`;
const filePath = file ? agentRouteService.getAgentFileDownloadLink(file.id, file.name) : '';
const result = {
actionId: action.actionId,
id: file?.id ?? action.actionId,
- status: file?.Status ?? 'IN_PROGRESS',
+ status: file?.Status ?? (action.error ? 'FAILED' : 'IN_PROGRESS'),
name: fileName,
createTime: action.timestamp!,
filePath,
@@ -76,7 +86,7 @@ export async function getAgentUploads(
async function _getRequestDiagnosticsActions(
esClient: ElasticsearchClient,
agentId: string
-): Promise> {
+): Promise> {
const agentActionRes = await esClient.search({
index: AGENT_ACTIONS_INDEX,
ignore_unavailable: true,
@@ -99,14 +109,17 @@ async function _getRequestDiagnosticsActions(
},
});
- const agentActionIds = agentActionRes.hits.hits.map((hit) => hit._source?.action_id as string);
+ const agentActions = agentActionRes.hits.hits.map((hit) => ({
+ actionId: hit._source?.action_id as string,
+ timestamp: hit._source?.['@timestamp'],
+ }));
- if (agentActionIds.length === 0) {
+ if (agentActions.length === 0) {
return [];
}
try {
- const actionResults = await esClient.search({
+ const actionResultsRes = await esClient.search({
index: AGENT_ACTIONS_RESULTS_INDEX,
ignore_unavailable: true,
size: SO_SEARCH_LIMIT,
@@ -115,7 +128,7 @@ async function _getRequestDiagnosticsActions(
must: [
{
terms: {
- action_id: agentActionIds,
+ action_id: agentActions.map((action) => action.actionId),
},
},
{
@@ -127,11 +140,21 @@ async function _getRequestDiagnosticsActions(
},
},
});
- return actionResults.hits.hits.map((hit) => ({
+ const actionResults = actionResultsRes.hits.hits.map((hit) => ({
actionId: hit._source?.action_id as string,
timestamp: hit._source?.['@timestamp'],
- fileId: hit._source?.data?.file_id as string,
+ fileId: hit._source?.data?.upload_id as string,
+ error: hit._source?.error,
}));
+ return agentActions.map((action) => {
+ const actionResult = actionResults.find((result) => result.actionId === action.actionId);
+ return {
+ actionId: action.actionId,
+ timestamp: actionResult?.timestamp ?? action.timestamp,
+ fileId: actionResult?.fileId,
+ error: actionResult?.error,
+ };
+ });
} catch (err) {
if (err.statusCode === 404) {
// .fleet-actions-results does not yet exist
diff --git a/x-pack/test/fleet_api_integration/apis/agents/uploads.ts b/x-pack/test/fleet_api_integration/apis/agents/uploads.ts
index 07e44bfa2a641c..c0f32104d24f93 100644
--- a/x-pack/test/fleet_api_integration/apis/agents/uploads.ts
+++ b/x-pack/test/fleet_api_integration/apis/agents/uploads.ts
@@ -53,7 +53,7 @@ export default function (providerContext: FtrProviderContext) {
agent_id: 'agent1',
'@timestamp': '2022-10-07T12:00:00.000Z',
data: {
- file_id: 'file1',
+ upload_id: 'file1',
},
},
},
@@ -67,6 +67,7 @@ export default function (providerContext: FtrProviderContext) {
body: {
doc_as_upsert: true,
doc: {
+ upload_id: 'file1',
file: {
ChunkSize: 4194304,
extension: 'zip',