Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Telemetry] Remove from and to timestamps from usage stats APIs #81579

Merged
merged 12 commits into from
Nov 2, 2020
30 changes: 9 additions & 21 deletions src/plugins/telemetry/public/services/telemetry_service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,42 +18,30 @@
*/

/* eslint-disable dot-notation */
import { mockTelemetryService } from '../mocks';

const mockSubtract = jest.fn().mockImplementation(() => {
return {
toISOString: jest.fn(),
};
});

const mockClone = jest.fn().mockImplementation(() => {
return {
clone: mockClone,
subtract: mockSubtract,
toISOString: jest.fn(),
};
});
const mockMomentValueOf = jest.fn();

jest.mock('moment', () => {
return jest.fn().mockImplementation(() => {
return {
clone: mockClone,
subtract: mockSubtract,
toISOString: jest.fn(),
valueOf: mockMomentValueOf,
};
});
});

import { mockTelemetryService } from '../mocks';

describe('TelemetryService', () => {
describe('fetchTelemetry', () => {
it('calls expected URL with 20 minutes - now', async () => {
const timestamp = Date.now();
mockMomentValueOf.mockReturnValueOnce(timestamp);
const telemetryService = mockTelemetryService();

await telemetryService.fetchTelemetry();
expect(telemetryService['http'].post).toBeCalledWith('/api/telemetry/v2/clusters/_stats', {
body: JSON.stringify({ unencrypted: false, timeRange: {} }),
body: JSON.stringify({ unencrypted: false, timestamp }),
});
expect(mockClone).toBeCalled();
expect(mockSubtract).toBeCalledWith(20, 'minutes');
expect(mockMomentValueOf).toBeCalled();
});
});

Expand Down
9 changes: 1 addition & 8 deletions src/plugins/telemetry/public/services/telemetry_service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,10 @@ export class TelemetryService {
};

public fetchTelemetry = async ({ unencrypted = false } = {}) => {
const now = moment();
return this.http.post('/api/telemetry/v2/clusters/_stats', {
body: JSON.stringify({
unencrypted,
timeRange: {
min: now
.clone() // Need to clone it to avoid mutation (and max being the same value)
.subtract(20, 'minutes')
.toISOString(),
max: now.toISOString(),
},
timestamp: moment().valueOf(),
}),
});
};
Expand Down
3 changes: 1 addition & 2 deletions src/plugins/telemetry/server/fetcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -213,8 +213,7 @@ export class FetcherTask {
private async fetchTelemetry() {
return await this.telemetryCollectionManager!.getStats({
unencrypted: false,
start: moment().subtract(20, 'minutes').toISOString(),
end: moment().toISOString(),
timestamp: moment().valueOf(),
});
}

Expand Down
3 changes: 1 addition & 2 deletions src/plugins/telemetry/server/routes/telemetry_opt_in.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,7 @@ export function registerTelemetryOptInRoutes({
}

const statsGetterConfig: StatsGetterConfig = {
start: moment().subtract(20, 'minutes').toISOString(),
end: moment().toISOString(),
timestamp: moment().valueOf(),
unencrypted: false,
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,7 @@ export function registerTelemetryOptInStatsRoutes(
const unencrypted = req.body.unencrypted;

const statsGetterConfig: StatsGetterConfig = {
start: moment().subtract(20, 'minutes').toISOString(),
end: moment().toISOString(),
timestamp: moment().valueOf(),
unencrypted,
request: req,
};
Expand Down
17 changes: 5 additions & 12 deletions src/plugins/telemetry/server/routes/telemetry_usage_stats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ const validate: TypeOptions<string | number>['validate'] = (value) => {
}
};

const dateSchema = schema.oneOf([schema.string({ validate }), schema.number({ validate })]);

export function registerTelemetryUsageStatsRoutes(
router: IRouter,
telemetryCollectionManager: TelemetryCollectionManagerPluginSetup,
Expand All @@ -45,25 +43,20 @@ export function registerTelemetryUsageStatsRoutes(
validate: {
body: schema.object({
unencrypted: schema.boolean({ defaultValue: false }),
timeRange: schema.object({
min: dateSchema,
max: dateSchema,
}),
timestamp: schema.oneOf([schema.string({ validate }), schema.number({ validate })]),
}),
},
},
async (context, req, res) => {
const start = moment(req.body.timeRange.min).toISOString();
const end = moment(req.body.timeRange.max).toISOString();
const unencrypted = req.body.unencrypted;
const { unencrypted, timestamp } = req.body;

try {
const statsConfig: StatsGetterConfig = {
unencrypted,
start,
end,
timestamp: moment(timestamp).valueOf(),
request: req,
unencrypted,
};

const stats = await telemetryCollectionManager.getStats(statsConfig);
return res.ok({ body: stats });
} catch (err) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,7 @@ function mockStatsCollectionConfig(clusterInfo: any, clusterStats: any, kibana:
...createCollectorFetchContextMock(),
esClient: mockGetLocalStats(clusterInfo, clusterStats),
usageCollection: mockUsageCollection(kibana),
start: '',
end: '',
timestamp: Date.now(),
};
}

Expand Down
4 changes: 2 additions & 2 deletions src/plugins/telemetry_collection_manager/server/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ export class TelemetryCollectionManagerPlugin
collectionSoService: SavedObjectsServiceStart,
usageCollection: UsageCollectionSetup
): StatsCollectionConfig {
const { start, end, request } = config;
const { timestamp, request } = config;

const callCluster = config.unencrypted
? collection.esCluster.asScoped(request).callAsCurrentUser
Expand All @@ -157,7 +157,7 @@ export class TelemetryCollectionManagerPlugin
const soClient = config.unencrypted
? collectionSoService.getScopedClient(config.request)
: collectionSoService.createInternalRepository();
return { callCluster, start, end, usageCollection, esClient, soClient };
return { callCluster, timestamp, usageCollection, esClient, soClient };
}

private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) {
Expand Down
6 changes: 2 additions & 4 deletions src/plugins/telemetry_collection_manager/server/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,7 @@ export interface TelemetryOptInStats {

export interface BaseStatsGetterConfig {
unencrypted: boolean;
start: string;
end: string;
timestamp: number;
request?: KibanaRequest;
}

Expand All @@ -77,8 +76,7 @@ export interface ClusterDetails {
export interface StatsCollectionConfig {
usageCollection: UsageCollectionSetup;
callCluster: LegacyAPICaller;
start: string | number;
end: string | number;
timestamp: number;
esClient: ElasticsearchClient;
soClient: SavedObjectsClientContract | ISavedObjectsRepository;
}
Expand Down
23 changes: 7 additions & 16 deletions test/api_integration/apis/telemetry/telemetry_local.js
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,12 @@ export default function ({ getService }) {
});

it('should pull local stats and validate data types', async () => {
const timeRange = {
min: '2018-07-23T22:07:00Z',
max: '2018-07-23T22:13:00Z',
};
const timestamp = '2018-07-23T22:13:00Z';

const { body } = await supertest
.post('/api/telemetry/v2/clusters/_stats')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, unencrypted: true })
.send({ timestamp, unencrypted: true })
.expect(200);

expect(body.length).to.be(1);
Expand Down Expand Up @@ -98,15 +95,12 @@ export default function ({ getService }) {
});

it('should pull local stats and validate fields', async () => {
const timeRange = {
min: '2018-07-23T22:07:00Z',
max: '2018-07-23T22:13:00Z',
};
const timestamp = '2018-07-23T22:13:00Z';

const { body } = await supertest
.post('/api/telemetry/v2/clusters/_stats')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, unencrypted: true })
.send({ timestamp, unencrypted: true })
.expect(200);

const stats = body[0];
Expand Down Expand Up @@ -156,10 +150,7 @@ export default function ({ getService }) {
});

describe('application usage limits', () => {
const timeRange = {
min: '2018-07-23T22:07:00Z',
max: '2018-07-23T22:13:00Z',
};
const timestamp = '2018-07-23T22:13:00Z';

function createSavedObject() {
return supertest
Expand Down Expand Up @@ -191,7 +182,7 @@ export default function ({ getService }) {
const { body } = await supertest
.post('/api/telemetry/v2/clusters/_stats')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, unencrypted: true })
.send({ timestamp, unencrypted: true })
.expect(200);

expect(body.length).to.be(1);
Expand Down Expand Up @@ -242,7 +233,7 @@ export default function ({ getService }) {
const { body } = await supertest
.post('/api/telemetry/v2/clusters/_stats')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, unencrypted: true })
.send({ timestamp, unencrypted: true })
.expect(200);

expect(body.length).to.be(1);
Expand Down
14 changes: 2 additions & 12 deletions x-pack/dev-tools/api_debug/apis/telemetry/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,8 @@
import moment from 'moment';

export const name = 'telemetry';
export const description = 'Get the clusters stats for the last 1 hour from the Kibana server';
export const description = 'Get the clusters stats from the Kibana server';
export const method = 'POST';
export const path = '/api/telemetry/v2/clusters/_stats';

// Get an object with start and end times for the last 1 hour, ISO format, in UTC
function getTimeRange() {
const end = moment();
const start = moment(end).subtract(1, 'hour');
return {
min: moment.utc(start).format(),
max: moment.utc(end).format(),
};
}

export const body = { timeRange: getTimeRange(), unencrypted: true };
export const body = { timeRange: moment().valueOf(), unencrypted: true };
16 changes: 16 additions & 0 deletions x-pack/plugins/monitoring/common/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,22 @@ export const REPORTING_SYSTEM_ID = 'reporting';
*/
export const TELEMETRY_COLLECTION_INTERVAL = 86400000;

/**
* The amount of time, in milliseconds, to fetch the cluster uuids from es.
*
* Currently 3 hours.
* @type {Number}
*/
export const CLUSTER_DETAILS_FETCH_INTERVAL = 10800000;

/**
* The amount of time, in milliseconds, to fetch the usage data from es.
*
* Currently 20 minutes.
* @type {Number}
*/
export const USAGE_FETCH_INTERVAL = 1200000;

/**
* The prefix for all alert types used by monitoring
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ import { ClustersHighLevelStats } from './get_high_level_stats';
import { coreMock } from 'src/core/server/mocks';

describe('get_all_stats', () => {
const start = 0;
const end = 1;
const timestamp = Date.now();
const callCluster = sinon.stub();
const esClient = sinon.stub();
const soClient = sinon.stub();
Expand Down Expand Up @@ -181,8 +180,7 @@ describe('get_all_stats', () => {
esClient: esClient as any,
soClient: soClient as any,
usageCollection: {} as any,
start,
end,
timestamp,
},
{
logger: coreMock.createPluginInitializerContext().logger.get('test'),
Expand All @@ -208,8 +206,7 @@ describe('get_all_stats', () => {
esClient: esClient as any,
soClient: soClient as any,
usageCollection: {} as any,
start,
end,
timestamp,
},
{
logger: coreMock.createPluginInitializerContext().logger.get('test'),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,17 @@ import { set } from '@elastic/safer-lodash-set';
import { get, merge } from 'lodash';

import { StatsGetter } from 'src/plugins/telemetry_collection_manager/server';
import { LOGSTASH_SYSTEM_ID, KIBANA_SYSTEM_ID, BEATS_SYSTEM_ID } from '../../common/constants';
import moment from 'moment';
import {
LOGSTASH_SYSTEM_ID,
KIBANA_SYSTEM_ID,
BEATS_SYSTEM_ID,
USAGE_FETCH_INTERVAL,
} from '../../common/constants';
import { getElasticsearchStats, ESClusterStats } from './get_es_stats';
import { getKibanaStats, KibanaStats } from './get_kibana_stats';
import { getBeatsStats } from './get_beats_stats';
import { getHighLevelStats } from './get_high_level_stats';

type PromiseReturnType<T extends (...args: any[]) => any> = ReturnType<T> extends Promise<infer R>
? R
: T;
import { getBeatsStats, BeatsStatsByClusterUuid } from './get_beats_stats';
import { getHighLevelStats, ClustersHighLevelStats } from './get_high_level_stats';

export interface CustomContext {
maxBucketSize: number;
Expand All @@ -28,9 +30,12 @@ export interface CustomContext {
*/
export const getAllStats: StatsGetter<CustomContext> = async (
clustersDetails,
{ callCluster, start, end, esClient, soClient },
{ callCluster, timestamp },
{ maxBucketSize }
) => {
const start = moment(timestamp).subtract(USAGE_FETCH_INTERVAL, 'ms').toISOString();
const end = moment(timestamp).toISOString();

const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid);

const [esClusters, kibana, logstash, beats] = await Promise.all([
Expand Down Expand Up @@ -61,8 +66,8 @@ export function handleAllStats(
beats,
}: {
kibana: KibanaStats;
logstash: PromiseReturnType<typeof getHighLevelStats>;
beats: PromiseReturnType<typeof getBeatsStats>;
logstash: ClustersHighLevelStats;
beats: BeatsStatsByClusterUuid;
}
) {
return clusters.map((cluster) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ const getBaseOptions = () => ({
describe('Get Beats Stats', () => {
describe('fetchBeatsStats', () => {
const clusterUuids = ['aCluster', 'bCluster', 'cCluster'];
const start = 100;
const end = 200;
const start = new Date().toISOString();
const end = new Date().toISOString();
let callCluster = sinon.stub();

beforeEach(() => {
Expand Down
Loading