[Security Solution][Endpoint] Update to Endpoint List and associated …

…supporting API to be space aware (elastic#194312)

## Summary

PR makes the following changes in support of Endpoint management support
for Spaces:

### Fleet

- Adds new method - `getByIds()` - to the server-side Agent service
- Updates some mocks

### Security Solution

- Updates the Endpoint host metadata API for both the list and single
endpoint to be space aware
- Updates the agent status API for Endpoint to be space aware
- Updates endpoint policy response API to be space aware
- New FTR API integration test suite was created
- included changes to some service utilities to enable loading test data
per-space

(cherry picked from commit 2a786b8)

.buildkite/ftr_security_serverless_configs.yml

@@ -95,6 +95,7 @@ disabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   # serverless config files that run deployment-agnostic tests

.buildkite/ftr_security_stateful_configs.yml

@@ -85,6 +85,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/integrations.config.ts
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts

x-pack/plugins/fleet/common/errors.ts

@@ -8,9 +8,9 @@
 
 import type { FleetErrorType } from './types';
 
-export class FleetError extends Error {
+export class FleetError<TMeta = unknown> extends Error {
   attributes?: { type: FleetErrorType };
-  constructor(message?: string, public readonly meta?: unknown) {
+  constructor(message?: string, public readonly meta?: TMeta) {
     super(message);
     this.name = this.constructor.name; // for stack traces
   }

x-pack/plugins/fleet/server/errors/index.ts

@@ -95,7 +95,7 @@ export class FleetEncryptedSavedObjectEncryptionKeyRequired extends FleetError {
 export class FleetSetupError extends FleetError {}
 export class GenerateServiceTokenError extends FleetError {}
 export class FleetUnauthorizedError extends FleetError {}
-export class FleetNotFoundError extends FleetError {}
+export class FleetNotFoundError<TMeta = unknown> extends FleetError<TMeta> {}
 export class FleetTooManyRequestsError extends FleetError {}
 
 export class OutputUnauthorizedError extends FleetError {}
@@ -105,7 +105,7 @@ export class DownloadSourceError extends FleetError {}
 export class DeleteUnenrolledAgentsPreconfiguredError extends FleetError {}
 
 // Not found errors
-export class AgentNotFoundError extends FleetNotFoundError {}
+export class AgentNotFoundError extends FleetNotFoundError<{ agentId: string }> {}
 export class AgentPolicyNotFoundError extends FleetNotFoundError {}
 export class AgentActionNotFoundError extends FleetNotFoundError {}
 export class DownloadSourceNotFound extends FleetNotFoundError {}
@@ -115,7 +115,10 @@ export class SigningServiceNotFoundError extends FleetNotFoundError {}
 export class InputNotFoundError extends FleetNotFoundError {}
 export class OutputNotFoundError extends FleetNotFoundError {}
 export class PackageNotFoundError extends FleetNotFoundError {}
-export class PackagePolicyNotFoundError extends FleetNotFoundError {}
+export class PackagePolicyNotFoundError extends FleetNotFoundError<{
+  /** The package policy ID that was not found */
+  packagePolicyId: string;
+}> {}
 export class StreamNotFoundError extends FleetNotFoundError {}
 
 export class FleetServerHostUnauthorizedError extends FleetUnauthorizedError {}

x-pack/plugins/fleet/server/services/agents/agent_service.mock.ts

@@ -15,10 +15,12 @@ const createClientMock = (): jest.Mocked<AgentClient> => ({
   getAgentStatusForAgentPolicy: jest.fn(),
   listAgents: jest.fn(),
   getLatestAgentAvailableVersion: jest.fn(),
+  getByIds: jest.fn(async (..._) => []),
 });
 
 const createServiceMock = (): DeeplyMockedKeys<AgentService> => ({
   asInternalUser: createClientMock(),
+  asInternalScopedUser: jest.fn().mockReturnValue(createClientMock()),
   asScoped: jest.fn().mockReturnValue(createClientMock()),
 });
 

x-pack/plugins/fleet/server/services/agents/agent_service.test.ts

@@ -185,6 +185,27 @@ describe('AgentService', () => {
       () => new AgentServiceImpl(mockEsClient, mockSoClient).asInternalUser
     );
   });
+
+  describe('asInternalScopedUser', () => {
+    it('should throw error if no space id is passed', () => {
+      const agentService = new AgentServiceImpl(
+        elasticsearchServiceMock.createElasticsearchClient(),
+        savedObjectsClientMock.create()
+      );
+
+      expect(() => agentService.asInternalScopedUser('')).toThrowError(TypeError);
+    });
+
+    {
+      const mockEsClient = elasticsearchServiceMock.createElasticsearchClient();
+      const mockSoClient = savedObjectsClientMock.create();
+      expectApisToCallServicesSuccessfully(
+        mockEsClient,
+        () => mockSoClient,
+        () => new AgentServiceImpl(mockEsClient, mockSoClient).asInternalUser
+      );
+    }
+  });
 });
 
 function expectApisToCallServicesSuccessfully(

x-pack/plugins/fleet/server/services/agents/agent_service.ts

@@ -27,7 +27,7 @@ import { FleetUnauthorizedError } from '../../errors';
 
 import { getCurrentNamespace } from '../spaces/get_current_namespace';
 
-import { getAgentsByKuery, getAgentById } from './crud';
+import { getAgentsByKuery, getAgentById, getByIds } from './crud';
 import { getAgentStatusById, getAgentStatusForAgentPolicy } from './status';
 import { getLatestAvailableAgentVersion } from './versions';
 
@@ -42,6 +42,11 @@ export interface AgentService {
    */
   asScoped(req: KibanaRequest): AgentClient;
 
+  /**
+   * Scoped services to a given space
+   */
+  asInternalScopedUser(spaceId: string): AgentClient;
+
   /**
    * Only use for server-side usages (eg. telemetry), should not be used for end users unless an explicit authz check is
    * done.
@@ -60,6 +65,12 @@ export interface AgentClient {
    */
   getAgent(agentId: string): Promise<Agent>;
 
+  /**
+   * Get multiple agents by id
+   * @param agentIds
+   */
+  getByIds(agentIds: string[], options?: { ignoreMissing?: boolean }): Promise<Agent[]>;
+
   /**
    * Return the status by the Agent's id
    */
@@ -128,6 +139,14 @@ class AgentClientImpl implements AgentClient {
     return getAgentById(this.internalEsClient, this.soClient, agentId);
   }
 
+  public async getByIds(
+    agentIds: string[],
+    options?: Partial<{ ignoreMissing: boolean }>
+  ): Promise<Agent[]> {
+    await this.#runPreflight();
+    return getByIds(this.internalEsClient, this.soClient, agentIds, options);
+  }
+
   public async getAgentStatusById(agentId: string) {
     await this.#runPreflight();
     return getAgentStatusById(this.internalEsClient, this.soClient, agentId);
@@ -187,6 +206,21 @@ export class AgentServiceImpl implements AgentService {
     );
   }
 
+  public asInternalScopedUser(spaceId: string): AgentClient {
+    if (!spaceId) {
+      throw new TypeError(`spaceId argument is required!`);
+    }
+
+    const soClient = appContextService.getInternalUserSOClientForSpaceId(spaceId);
+
+    return new AgentClientImpl(
+      this.internalEsClient,
+      soClient,
+      undefined,
+      getCurrentNamespace(soClient)
+    );
+  }
+
   public get asInternalUser() {
     return new AgentClientImpl(this.internalEsClient, this.soClient);
   }

x-pack/plugins/fleet/server/services/agents/crud.test.ts

@@ -9,6 +9,10 @@ import type { ElasticsearchClient } from '@kbn/core/server';
 import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks';
 import { toElasticsearchQuery } from '@kbn/es-query';
 
+import { isSpaceAwarenessEnabled as _isSpaceAwarenessEnabled } from '../spaces/helpers';
+
+import { AgentNotFoundError } from '../..';
+
 import { AGENTS_INDEX } from '../../constants';
 import { createAppContextStartContractMock } from '../../mocks';
 import type { Agent } from '../../types';
@@ -24,6 +28,7 @@ import {
   openPointInTime,
   updateAgent,
   _joinFilters,
+  getByIds,
 } from './crud';
 
 jest.mock('../audit_logging');
@@ -41,6 +46,7 @@ jest.mock('./versions', () => {
 jest.mock('../spaces/helpers');
 
 const mockedAuditLoggingService = auditLoggingService as jest.Mocked<typeof auditLoggingService>;
+const isSpaceAwarenessEnabledMock = _isSpaceAwarenessEnabled as jest.Mock;
 
 describe('Agents CRUD test', () => {
   const soClientMock = savedObjectsClientMock.create();
@@ -63,13 +69,22 @@ describe('Agents CRUD test', () => {
     appContextService.start(mockContract);
   });
 
-  function getEsResponse(ids: string[], total: number, status: AgentStatus) {
+  afterEach(() => {
+    isSpaceAwarenessEnabledMock.mockReset();
+  });
+
+  function getEsResponse(
+    ids: string[],
+    total: number,
+    status: AgentStatus,
+    generateSource: (id: string) => Partial<Agent> = () => ({})
+  ) {
     return {
       hits: {
         total,
         hits: ids.map((id: string) => ({
           _id: id,
-          _source: {},
+          _source: generateSource(id),
           fields: {
             status: [status],
           },
@@ -513,4 +528,48 @@ describe('Agents CRUD test', () => {
       });
     });
   });
+
+  describe(`getByIds()`, () => {
+    let searchResponse: ReturnType<typeof getEsResponse>;
+
+    beforeEach(() => {
+      searchResponse = getEsResponse(['1', '2'], 2, 'online', (id) => {
+        return { id, namespaces: ['foo'] };
+      });
+      (soClientMock.getCurrentNamespace as jest.Mock).mockReturnValue('foo');
+      searchMock.mockImplementation(async () => searchResponse);
+    });
+
+    it('should return a list of agents', async () => {
+      await expect(getByIds(esClientMock, soClientMock, ['1', '2'])).resolves.toEqual([
+        expect.objectContaining({ id: '1' }),
+        expect.objectContaining({ id: '2' }),
+      ]);
+    });
+
+    it('should omit agents that are not found if `ignoreMissing` is true', async () => {
+      searchResponse.hits.hits = [searchResponse.hits.hits[0]];
+
+      await expect(
+        getByIds(esClientMock, soClientMock, ['1', '2'], { ignoreMissing: true })
+      ).resolves.toEqual([expect.objectContaining({ id: '1' })]);
+    });
+
+    it('should error if agent is not found and `ignoreMissing` is false', async () => {
+      searchResponse.hits.hits = [searchResponse.hits.hits[0]];
+
+      await expect(getByIds(esClientMock, soClientMock, ['1', '2'])).rejects.toThrow(
+        AgentNotFoundError
+      );
+    });
+
+    it('should error if agent is not part of current space', async () => {
+      searchResponse.hits.hits[0]._source.namespaces = ['bar'];
+      isSpaceAwarenessEnabledMock.mockResolvedValue(true);
+
+      await expect(getByIds(esClientMock, soClientMock, ['1', '2'])).rejects.toThrow(
+        AgentNotFoundError
+      );
+    });
+  });
 });

x-pack/plugins/fleet/server/services/agents/crud.ts

@@ -415,6 +415,46 @@ export async function getAgentById(
   return agentHit;
 }
 
+/**
+ * Get list of agents by `id`. service method performs space awareness checks.
+ * @param esClient
+ * @param soClient
+ * @param agentIds
+ * @param options
+ *
+ * @throws AgentNotFoundError
+ */
+export const getByIds = async (
+  esClient: ElasticsearchClient,
+  soClient: SavedObjectsClientContract,
+  agentIds: string[],
+  options?: Partial<{ ignoreMissing: boolean }>
+): Promise<Agent[]> => {
+  const agentsHits = await getAgentsById(esClient, soClient, agentIds);
+  const currentNamespace = getCurrentNamespace(soClient);
+  const response: Agent[] = [];
+
+  for (const agentHit of agentsHits) {
+    let throwError = false;
+
+    if ('notFound' in agentHit && !options?.ignoreMissing) {
+      throwError = true;
+    } else if ((await isAgentInNamespace(agentHit as Agent, currentNamespace)) !== true) {
+      throwError = true;
+    }
+
+    if (throwError) {
+      throw new AgentNotFoundError(`Agent ${agentHit.id} not found`, { agentId: agentHit.id });
+    }
+
+    if (!(`notFound` in agentHit)) {
+      response.push(agentHit);
+    }
+  }
+
+  return response;
+};
+
 async function _filterAgents(
   esClient: ElasticsearchClient,
   soClient: SavedObjectsClientContract,

x-pack/plugins/fleet/server/services/package_policy.ts

@@ -771,7 +771,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
           if (options.ignoreMissing && so.error.statusCode === 404) {
             return null;
           } else if (so.error.statusCode === 404) {
-            throw new PackagePolicyNotFoundError(`Package policy ${so.id} not found`);
+            throw new PackagePolicyNotFoundError(`Package policy ${so.id} not found`, {
+              packagePolicyId: so.id,
+            });
           } else {
             throw new FleetError(so.error.message);
           }

x-pack/plugins/security_solution/common/endpoint/data_generators/endpoint_metadata_generator.ts

@@ -11,9 +11,15 @@ import type { DeepPartial } from 'utility-types';
 import { merge } from 'lodash';
 import { set } from '@kbn/safer-lodash-set';
 import { gte } from 'semver';
+import type { Agent } from '@kbn/fleet-plugin/common';
 import type { EndpointCapabilities } from '../service/response_actions/constants';
 import { BaseDataGenerator } from './base_data_generator';
-import type { HostMetadataInterface, OSFields, HostInfoInterface } from '../types';
+import type {
+  HostMetadataInterface,
+  OSFields,
+  HostInfoInterface,
+  UnitedAgentMetadataPersistedData,
+} from '../types';
 import { EndpointStatus, HostPolicyResponseActionStatus, HostStatus } from '../types';
 
 export interface GetCustomEndpointMetadataGeneratorOptions {
@@ -226,6 +232,30 @@ export class EndpointMetadataGenerator extends BaseDataGenerator {
     return merge(hostInfo, overrides);
   }
 
+  generateUnitedAgentMetadata(
+    overrides: DeepPartial<UnitedAgentMetadataPersistedData> = {}
+  ): UnitedAgentMetadataPersistedData {
+    const endpointMetadata = this.generate();
+
+    return merge(
+      {
+        agent: {
+          id: endpointMetadata.agent.id,
+        },
+        united: {
+          endpoint: endpointMetadata,
+          agent: {
+            agent: {
+              id: endpointMetadata.agent.id,
+            },
+            policy_id: this.seededUUIDv4(),
+          } as Agent,
+        },
+      } as UnitedAgentMetadataPersistedData,
+      overrides
+    );
+  }
+
   protected randomOsFields(): OSFields {
     return this.randomChoice([
       EndpointMetadataGenerator.windowsOSFields,