Skip to content

Commit

Permalink
Add the vcs info
Browse files Browse the repository at this point in the history
  • Loading branch information
@wayneseymour
wayneseymour committed Apr 1, 2020
1 parent f07b34f commit f229c24
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 51 deletions.
88 changes: 49 additions & 39 deletions .ci/Jenkinsfile_coverage
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,85 +9,96 @@ kibanaPipeline(timeoutMinutes: 180) {
'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc.
]) {

// workers.base(name: 'fail-fast-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) {
// maybeFailFast('secret/kibana-issues/prod/coverage/elasticsearch')
// }
workers.base(name: 'fail-fast-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) {
maybeFailFast('secret/kibana-issues/prod/coverage/elasticsearch')
}

}

// parallel([
// 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
// 'x-pack-intake-agent': {
// withEnv([
// 'NODE_ENV=test' // Needed for jest tests only
// ]) {
// workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh')()
// }
// },
// 'kibana-oss-agent' : workers.functional(
// 'kibana-oss-tests',
// { kibanaPipeline.buildOss() },
// ossProks()
// ),
// 'kibana-xpack-agent' : workers.functional(
// 'kibana-xpack-tests',
// { kibanaPipeline.buildXpack() },
// xpackProks()
// ),
// ])
parallel([
'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
'x-pack-intake-agent': {
withEnv([
'NODE_ENV=test' // Needed for jest tests only
]) {
workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh')()
}
},
'kibana-oss-agent' : workers.functional(
'kibana-oss-tests',
{ kibanaPipeline.buildOss() },
ossProks()
),
'kibana-xpack-agent' : workers.functional(
'kibana-xpack-tests',
{ kibanaPipeline.buildXpack() },
xpackProks()
),
])

workers.base(name: 'coverage-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) {
// kibanaPipeline.downloadCoverageArtifacts()
kibanaPipeline.downloadCoverageArtifacts()

def timestamp = new Date(currentBuild.startTimeInMillis).format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC"))
withEnv([
"TIME_STAMP=${timestamp}",
"QA_TEAM_ROCKS=true"
]) {
listAndCollectVcsEnvVars()
collectVcsInfo()

def vaultSecret = 'secret/kibana-issues/prod/coverage/elasticsearch'
withVaultSecret(secret: vaultSecret, secret_field: 'host', variable_name: 'HOST_FROM_VAULT') {
withVaultSecret(secret: vaultSecret, secret_field: 'username', variable_name: 'USER_FROM_VAULT') {
withVaultSecret(secret: vaultSecret, secret_field: 'password', variable_name: 'PASS_FROM_VAULT') {
kibanaPipeline.bash("""
source src/dev/ci_setup/setup_env.sh
# bootstrap from x-pack folder
cd x-pack
yarn kbn bootstrap --prefer-offline
# Return to project root
cd ..
. src/dev/code_coverage/shell_scripts/extract_archives.sh
. src/dev/code_coverage/shell_scripts/fix_html_reports_parallel.sh
echo "### Print vcs info file"
cat VCS_INFO.txt || echo "### VCS Info file NOT found"
. src/dev/code_coverage/shell_scripts/merge_jest_and_functional.sh
. src/dev/code_coverage/shell_scripts/copy_mocha_reports.sh
. src/dev/code_coverage/shell_scripts/ingest_coverage.sh ${BUILD_NUMBER} ${env.BUILD_URL}
""", "### Bootstrap shell and kibana env, merge and ingest code coverage")
}
}
}
// kibanaCoverage.uploadCoverageStaticSite(TIME_STAMP)
kibanaCoverage.uploadCoverageStaticSite(TIME_STAMP)
}

// sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*'
// kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz')
// sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*'
// kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz')
// sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*'
// kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz')
sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*'
kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz')
sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*'
kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz')
sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*'
kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz')

}

}

// kibanaPipeline.sendMail()
kibanaPipeline.sendMail()
}

def listAndCollectVcsEnvVars() {
def collectVcsInfo() {
kibanaPipeline.bash(
'''
predicate() {
x=$1
if [ -n "$x" ]; then
echo "### Defined: ${x}"
return
else
echo "### 1 or more variables that Code Coverage needs, are undefined"
exit 1
Expand All @@ -105,9 +116,8 @@ def listAndCollectVcsEnvVars() {
for X in "${!XS[@]}"; do
{
echo "### Checking: $X"
predicate "${XS[X]}"
echo " ${XS[X]}" >> VCS_INFO.txt
echo "${XS[X]}" >> VCS_INFO.txt
}
done
Expand Down
6 changes: 4 additions & 2 deletions src/dev/code_coverage/ingest_coverage/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,10 @@ export function runCoverageIngestionCli() {
if (flags.path === '') throw createFlagError('please provide a single --path flag');
if (flags.verbose) log.verbose(`Verbose logging enabled`);

const jsonSummaryPath = resolve(ROOT, flags.path);
parseAndProcess({ jsonSummaryPath }, log);
const resolveRoot = resolve.bind(null, ROOT);
const jsonSummaryPath = resolveRoot(flags.path);
const vcsInfoFilePath = resolveRoot('VCS_INFO.txt');
parseAndProcess({ jsonSummaryPath, vcsInfoFilePath }, log);
},
{
description: `
Expand Down
42 changes: 32 additions & 10 deletions src/dev/code_coverage/ingest_coverage/process.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@
* under the License.
*/

import { fromEventPattern, of } from 'rxjs';
import { concatMap, delay, map } from 'rxjs/operators';
import { fromEventPattern, of, fromEvent } from 'rxjs';
import { concatMap, delay, map, takeUntil } from 'rxjs/operators';
import jsonStream from './json_stream';
import { pipe, noop, green } from './utils';
import { pipe, noop, green, always } from './utils';
import { ingest } from './ingest';
import {
staticSite,
Expand All @@ -31,8 +31,11 @@ import {
buildId,
coveredFilePath,
ciRunUrl,
itemizeVcs,
} from './transforms';
import { resolve } from 'path';
import { createReadStream } from 'fs';
import readline from 'readline';

const KIBANA_ROOT_PATH = '../../../..';
const KIBANA_ROOT = resolve(__dirname, KIBANA_ROOT_PATH);
Expand All @@ -43,19 +46,15 @@ const addPrePopulatedTimeStamp = addTimeStamp(process.env.TIME_STAMP);
const prokStatsTimeStampBuildId = pipe(statsAndstaticSiteUrl, buildId, addPrePopulatedTimeStamp);
const addTestRunnerAndStaticSiteUrl = pipe(testRunner, staticSite(staticSiteUrlBase));

export default ({ jsonSummaryPath }, log) => {
log.debug(`### Code coverage ingestion set to delay for: ${green(ms)} ms`);
log.debug(`### KIBANA_ROOT: \n\t${green(KIBANA_ROOT)}`);
log.debug(`### Ingesting from summary json: \n\t[${green(jsonSummaryPath)}]`);

validateRoot(KIBANA_ROOT, log);

const execute = jsonSummaryPath => log => vcsInfo => {
const objStream = jsonStream(jsonSummaryPath).on('done', noop);
const itemizeVcsInfo = itemizeVcs(vcsInfo);

fromEventPattern(_ => objStream.on('node', '!.*', _))
.pipe(
map(prokStatsTimeStampBuildId),
map(coveredFilePath),
map(itemizeVcsInfo),
map(ciRunUrl),
map(addJsonSummaryPath(jsonSummaryPath)),
map(addTestRunnerAndStaticSiteUrl),
Expand All @@ -64,6 +63,29 @@ export default ({ jsonSummaryPath }, log) => {
.subscribe(ingest(log));
};

export default ({ jsonSummaryPath, vcsInfoFilePath }, log) => {
log.debug(`### Code coverage ingestion set to delay for: ${green(ms)} ms`);
log.debug(`### KIBANA_ROOT: \n\t${green(KIBANA_ROOT)}`);
log.debug(`### Ingesting from summary json: \n\t[${green(jsonSummaryPath)}]`);

validateRoot(KIBANA_ROOT, log);

const vcsInfo = [];
const vcsInfoLines$ = vcsInfoFilePath => {
const rl = readline.createInterface({ input: createReadStream(vcsInfoFilePath) });
return fromEvent(rl, 'line').pipe(takeUntil(fromEvent(rl, 'close')));
};

const executeWithPath = execute(jsonSummaryPath)(log);
const mutate = x => vcsInfo.push(x.trimStart().trimEnd());

vcsInfoLines$(vcsInfoFilePath).subscribe(
mutate,
err => console.log('Error: %s', err),
always(executeWithPath(vcsInfo))
);
};

function validateRoot(x, log) {
return /kibana$/.test(x) ? noop() : log.warning(`!!! 'kibana' NOT FOUND in ROOT: ${x}\n`);
}
8 changes: 8 additions & 0 deletions src/dev/code_coverage/ingest_coverage/transforms.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,14 @@ export const ciRunUrl = obj => {
};
};

export const itemizeVcs = vcsInfo => obj => {
const [branch, sha, author, commitMsg] = vcsInfo;
return {
...obj,
vcs: { branch, sha, author, commitMsg },
};
};

export const testRunner = obj => {
const { jsonSummaryPath } = obj;

Expand Down

0 comments on commit f229c24

Please sign in to comment.