Skip to content

Commit

Permalink
Merge branch 'main' into drilldown_theme
Browse files Browse the repository at this point in the history
  • Loading branch information
kibanamachine authored Dec 7, 2022
2 parents 597b8b3 + a3220fe commit 8ba0dba
Show file tree
Hide file tree
Showing 105 changed files with 5,677 additions and 4,694 deletions.
1 change: 1 addition & 0 deletions .buildkite/ftr_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ enabled:
- x-pack/test/alerting_api_integration/basic/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
- x-pack/test/alerting_api_integration/spaces_only/config.ts
- x-pack/test/api_integration_basic/config.ts
Expand Down
48 changes: 2 additions & 46 deletions .buildkite/scripts/steps/scalability/benchmarking.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,53 +74,9 @@ download_artifacts
echo "--- Clone kibana-load-testing repo and compile project"
checkout_and_compile_load_runner

echo "--- Run Scalability Tests with Elasticsearch started only once and Kibana restart before each journey"
echo "--- Run Scalability Tests"
cd "$KIBANA_DIR"
node scripts/es snapshot&

esPid=$!
# Set trap on EXIT to stop Elasticsearch process
trap "kill -9 $esPid" EXIT

# unset env vars defined in other parts of CI for automatic APM collection of
# Kibana. We manage APM config in our FTR config and performance service, and
# APM treats config in the ENV with a very high precedence.
unset ELASTIC_APM_ENVIRONMENT
unset ELASTIC_APM_TRANSACTION_SAMPLE_RATE
unset ELASTIC_APM_SERVER_URL
unset ELASTIC_APM_SECRET_TOKEN
unset ELASTIC_APM_ACTIVE
unset ELASTIC_APM_CONTEXT_PROPAGATION_ONLY
unset ELASTIC_APM_GLOBAL_LABELS
unset ELASTIC_APM_MAX_QUEUE_SIZE
unset ELASTIC_APM_METRICS_INTERVAL
unset ELASTIC_APM_CAPTURE_SPAN_STACK_TRACES
unset ELASTIC_APM_BREAKDOWN_METRICS


export TEST_ES_DISABLE_STARTUP=true
ES_HOST="localhost:9200"
export TEST_ES_URL="http://elastic:changeme@${ES_HOST}"
# Overriding Gatling default configuration
export ES_URL="http://${ES_HOST}"

# Pings the ES server every second for 2 mins until its status is green
curl --retry 120 \
--retry-delay 1 \
--retry-connrefused \
-I -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow"

export ELASTIC_APM_ACTIVE=true

for journey in scalability_traces/server/*; do
export SCALABILITY_JOURNEY_PATH="$KIBANA_DIR/$journey"
echo "--- Run scalability file: $SCALABILITY_JOURNEY_PATH"
node scripts/functional_tests \
--config x-pack/test/scalability/config.ts \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--logToFile \
--debug
done
node scripts/run_scalability --kibana-install-dir "$KIBANA_BUILD_LOCATION" --journey-config-path "scalability_traces/server"

echo "--- Upload test results"
upload_test_results
4 changes: 2 additions & 2 deletions docs/index-custom-title-page.html
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ <h2>Bring your data to life</h2>
</a>
</p>
<p>
<a class="inline-block mr-3" href="https://www.elastic.co/guide/en/kibana/current/whats-new.html">What's new</a>
<a class="inline-block mr-3" href="https://www.elastic.co/guide/en/kibana/8.5/release-notes-8.5.0.html">Release notes</a>
<a class="inline-block mr-3" href="https://www.elastic.co/guide/en/kibana/8.6/whats-new.html">What's new</a>
<a class="inline-block mr-3" href="https://www.elastic.co/guide/en/kibana/8.6/release-notes.html">Release notes</a>
<a class="inline-block mr-3" href="https://www.elastic.co/videos/training-how-to-series-stack">How-to videos</a>
</p>
</div>
Expand Down
14 changes: 7 additions & 7 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@
},
"dependencies": {
"@appland/sql-parser": "^1.5.1",
"@babel/runtime": "^7.20.1",
"@babel/runtime": "^7.20.6",
"@dnd-kit/core": "^3.1.1",
"@dnd-kit/sortable": "^4.0.0",
"@dnd-kit/utilities": "^2.0.0",
Expand Down Expand Up @@ -694,12 +694,12 @@
"devDependencies": {
"@apidevtools/swagger-parser": "^10.0.3",
"@babel/cli": "^7.19.3",
"@babel/core": "^7.20.2",
"@babel/core": "^7.20.5",
"@babel/eslint-parser": "^7.19.1",
"@babel/eslint-plugin": "^7.19.1",
"@babel/generator": "^7.20.4",
"@babel/generator": "^7.20.5",
"@babel/helper-plugin-utils": "^7.20.2",
"@babel/parser": "^7.20.3",
"@babel/parser": "^7.20.5",
"@babel/plugin-proposal-class-properties": "^7.18.6",
"@babel/plugin-proposal-export-namespace-from": "^7.18.9",
"@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6",
Expand All @@ -711,8 +711,8 @@
"@babel/preset-react": "^7.18.6",
"@babel/preset-typescript": "^7.18.6",
"@babel/register": "^7.18.9",
"@babel/traverse": "^7.20.1",
"@babel/types": "^7.20.2",
"@babel/traverse": "^7.20.5",
"@babel/types": "^7.20.5",
"@bazel/ibazel": "^0.16.2",
"@bazel/typescript": "4.6.2",
"@cypress/code-coverage": "^3.10.0",
Expand Down Expand Up @@ -981,7 +981,7 @@
"argsplit": "^1.0.5",
"autoprefixer": "^10.4.7",
"axe-core": "^4.0.2",
"babel-jest": "^29.2.2",
"babel-jest": "^29.3.1",
"babel-loader": "^8.2.5",
"babel-plugin-add-module-exports": "^1.0.4",
"babel-plugin-istanbul": "^6.1.1",
Expand Down
1 change: 0 additions & 1 deletion renovate.json
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,6 @@
"@jest/console",
"@jest/reporters",
"@jest/types",
"@types/jest",
"babel-jest",
"expect",
"jest",
Expand Down
10 changes: 10 additions & 0 deletions scripts/run_scalability.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

require('../src/setup_node_env');
require('../src/dev/performance/run_scalability_cli');
104 changes: 104 additions & 0 deletions src/dev/performance/run_scalability_cli.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { createFlagError } from '@kbn/dev-cli-errors';
import { run } from '@kbn/dev-cli-runner';
import { REPO_ROOT } from '@kbn/utils';
import fs from 'fs';
import path from 'path';

run(
async ({ log, flagsReader, procRunner }) => {
const kibanaInstallDir = flagsReader.path('kibana-install-dir');
const journeyConfigPath = flagsReader.requiredPath('journey-config-path');

if (kibanaInstallDir && !fs.existsSync(kibanaInstallDir)) {
throw createFlagError('--kibana-install-dir must be an existing directory');
}
if (
!fs.existsSync(journeyConfigPath) ||
(!fs.statSync(journeyConfigPath).isDirectory() && path.extname(journeyConfigPath) !== '.json')
) {
throw createFlagError(
'--journey-config-path must be an existing directory or scalability json path'
);
}

const journeys = fs.statSync(journeyConfigPath).isDirectory()
? fs
.readdirSync(journeyConfigPath)
.filter((fileName) => path.extname(fileName) === '.json')
.map((fileName) => path.resolve(journeyConfigPath, fileName))
: [journeyConfigPath];

log.info(`Found ${journeys.length} journeys to run:\n${JSON.stringify(journeys)}`);

const failedJourneys = [];

for (const journey of journeys) {
try {
process.stdout.write(`--- Running scalability journey: ${journey}\n`);
await runScalabilityJourney(journey, kibanaInstallDir);
} catch (e) {
log.error(e);
failedJourneys.push(journey);
}
}

if (failedJourneys.length > 0) {
throw new Error(`${failedJourneys.length} journeys failed: ${failedJourneys.join(',')}`);
}

async function runScalabilityJourney(filePath: string, kibanaDir?: string) {
// Pass in a clean APM environment, so that FTR can later
// set it's own values.
const cleanApmEnv = {
ELASTIC_APM_ACTIVE: undefined,
ELASTIC_APM_BREAKDOWN_METRICS: undefined,
ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: undefined,
ELASTIC_APM_CAPTURE_SPAN_STACK_TRACES: undefined,
ELASTIC_APM_ENVIRONMENT: undefined,
ELASTIC_APM_GLOBAL_LABELS: undefined,
ELASTIC_APM_MAX_QUEUE_SIZE: undefined,
ELASTIC_APM_METRICS_INTERVAL: undefined,
ELASTIC_APM_SERVER_URL: undefined,
ELASTIC_APM_SECRET_TOKEN: undefined,
ELASTIC_APM_TRANSACTION_SAMPLE_RATE: undefined,
};

await procRunner.run('scalability-tests', {
cmd: 'node',
args: [
'scripts/functional_tests',
['--config', 'x-pack/test/scalability/config.ts'],
kibanaDir ? ['--kibana-install-dir', kibanaDir] : [],
'--debug',
'--logToFile',
'--bail',
].flat(),
cwd: REPO_ROOT,
wait: true,
env: {
...cleanApmEnv,
SCALABILITY_JOURNEY_PATH: filePath, // journey json file for Gatling test runner
KIBANA_DIR: REPO_ROOT, // Gatling test runner use it to find kbn/es archives
},
});
}
},
{
flags: {
string: ['kibana-install-dir', 'journey-config-path'],
help: `
--kibana-install-dir Run Kibana from existing install directory instead of from source
--journey-config-path Define a scalability journey config or directory with multiple
configs that should be executed
`,
},
}
);
1 change: 0 additions & 1 deletion x-pack/plugins/alerting/server/rules_client.mock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ const createRulesClientMock = () => {
bulkDisableRules: jest.fn(),
snooze: jest.fn(),
unsnooze: jest.fn(),
calculateIsSnoozedUntil: jest.fn(),
clearExpiredSnoozes: jest.fn(),
runSoon: jest.fn(),
clone: jest.fn(),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { RawRule } from '../../types';
import { CreateAPIKeyResult } from '../types';

export function apiKeyAsAlertAttributes(
apiKey: CreateAPIKeyResult | null,
username: string | null
): Pick<RawRule, 'apiKey' | 'apiKeyOwner'> {
return apiKey && apiKey.apiKeysEnabled
? {
apiKeyOwner: username,
apiKey: Buffer.from(`${apiKey.result.id}:${apiKey.result.api_key}`).toString('base64'),
}
: {
apiKeyOwner: null,
apiKey: null,
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/
import { set, get } from 'lodash';
import type { BulkEditOperation, BulkEditFields } from '../rules_client';
import type { BulkEditOperation, BulkEditFields } from '../types';

// defining an union type that will passed directly to generic function as a workaround for the issue similar to
// https://github.com/microsoft/TypeScript/issues/29479
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { RuleSnooze } from '../../types';
import { getRuleSnoozeEndTime } from '../../lib';

export function calculateIsSnoozedUntil(rule: {
muteAll: boolean;
snoozeSchedule?: RuleSnooze;
}): string | null {
const isSnoozedUntil = getRuleSnoozeEndTime(rule);
return isSnoozedUntil ? isSnoozedUntil.toISOString() : null;
}
26 changes: 26 additions & 0 deletions x-pack/plugins/alerting/server/rules_client/common/constants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import {
AlertingAuthorizationFilterType,
AlertingAuthorizationFilterOpts,
} from '../../authorization';

// NOTE: Changing this prefix will require a migration to update the prefix in all existing `rule` saved objects
export const extractedSavedObjectParamReferenceNamePrefix = 'param:';

// NOTE: Changing this prefix will require a migration to update the prefix in all existing `rule` saved objects
export const preconfiguredConnectorActionRefPrefix = 'preconfigured:';

export const alertingAuthorizationFilterOpts: AlertingAuthorizationFilterOpts = {
type: AlertingAuthorizationFilterType.KQL,
fieldNames: { ruleTypeId: 'alert.attributes.alertTypeId', consumer: 'alert.attributes.consumer' },
};

export const MAX_RULES_NUMBER_FOR_BULK_OPERATION = 10000;
export const API_KEY_GENERATE_CONCURRENCY = 50;
export const RULE_TYPE_CHECKS_CONCURRENCY = 50;
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { truncate, trim } from 'lodash';

export function generateAPIKeyName(alertTypeId: string, alertName: string) {
return truncate(`Alerting: ${alertTypeId}/${trim(alertName)}`, { length: 256 });
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import Boom from '@hapi/boom';
import { BulkOptions, BulkOptionsFilter, BulkOptionsIds } from '../types';

export const getAndValidateCommonBulkOptions = (options: BulkOptions) => {
const filter = (options as BulkOptionsFilter).filter;
const ids = (options as BulkOptionsIds).ids;

if (!ids && !filter) {
throw Boom.badRequest(
"Either 'ids' or 'filter' property in method's arguments should be provided"
);
}

if (ids?.length === 0) {
throw Boom.badRequest("'ids' property should not be an empty array");
}

if (ids && filter) {
throw Boom.badRequest(
"Both 'filter' and 'ids' are supplied. Define either 'ids' or 'filter' properties in method's arguments"
);
}
return { ids, filter };
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { uniq } from 'lodash';

export function includeFieldsRequiredForAuthentication(fields: string[]): string[] {
return uniq([...fields, 'alertTypeId', 'consumer']);
}
Loading

0 comments on commit 8ba0dba

Please sign in to comment.