feat: support configurable chunk size (#944)

* support configurable chunk size * update error * move error further up * add test adjust test confirm if pushed
elastic · Aug 2, 2024 · b7603d8 · b7603d8
1 parent a8167cd
commit b7603d8
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 2 deletions.
diff --git a/__tests__/push/index.test.ts b/__tests__/push/index.test.ts
@@ -200,8 +200,8 @@ heartbeat.monitors:
     `
     );
     const output = await runPush();
-    expect(output).toContain(`Aborted: Duplicate monitors found`);
     await rm(heartbeatYml, { force: true });
+    expect(output).toContain(`Aborted: Duplicate monitors found`);
   });
 
   it('format duplicate monitors', () => {
@@ -218,6 +218,38 @@ heartbeat.monitors:
     expect(formatDuplicateError(duplicates as Set<Monitor>)).toMatchSnapshot();
   });
 
+  it('error on invalid CHUNK SIZE', async () => {
+    await fakeProjectSetup(
+      { id: 'test-project', space: 'dummy', url: server.PREFIX },
+      { locations: ['test-loc'], schedule: 3 }
+    );
+    const output = await runPush(undefined, { CHUNK_SIZE: '251' });
+    expect(output).toContain(
+      'Invalid CHUNK_SIZE. CHUNK_SIZE must be less than or equal to 250'
+    );
+  });
+
+  it('respects valid CHUNK SIZE', async () => {
+    await fakeProjectSetup(
+      { id: 'test-project', space: 'dummy', url: server.PREFIX },
+      { locations: ['test-loc'], schedule: 3 }
+    );
+    const testJourney = join(PROJECT_DIR, 'chunk.journey.ts');
+    await writeFile(
+      testJourney,
+      `import {journey, monitor} from '../../../';
+    journey('a', () => monitor.use({ tags: ['chunk'] }));
+    journey('b', () => monitor.use({ tags: ['chunk'] }));`
+    );
+    const output = await runPush([...DEFAULT_ARGS, '--tags', 'chunk'], {
+      CHUNK_SIZE: '1',
+    });
+    await rm(testJourney, { force: true });
+    expect(output).toContain('Added(2)');
+    expect(output).toContain('creating or updating 1 monitors');
+    expect(output).toContain('✓ Pushed:');
+  });
+
   ['8.5.0', '8.6.0'].forEach(version => {
     describe('API: ' + version, () => {
       let server: Server;

diff --git a/src/push/index.ts b/src/push/index.ts
@@ -61,6 +61,11 @@ import {
 import { log } from '../core/logger';
 
 export async function push(monitors: Monitor[], options: PushOptions) {
+  if (parseInt(process.env.CHUNK_SIZE) > 250) {
+    throw error(
+      'Invalid CHUNK_SIZE. CHUNK_SIZE must be less than or equal to 250'
+    );
+  }
   const duplicates = trackDuplicates(monitors);
   if (duplicates.size > 0) {
     throw error(formatDuplicateError(duplicates));

diff --git a/src/push/kibana_api.ts b/src/push/kibana_api.ts
@@ -37,7 +37,7 @@ import {
 import { generateURL } from './utils';
 
 // Default chunk size for bulk put / delete
-export const CHUNK_SIZE = 100;
+export const CHUNK_SIZE = parseInt(process.env.CHUNK_SIZE) || 100;
 
 export type PutResponse = {
   createdMonitors: string[];