[Tests] updated artifact script

packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts

@@ -104,7 +104,7 @@ describe('opensearchArchiver: createGenerateDocRecordsStream()', () => {
         await delay(200);
         return {
           _scroll_id: 'index1ScrollId',
-          hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_foo' }] },
+          hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_1' }] },
         };
       },
       async (name, params) => {

packages/osd-opensearch/src/artifact.js

@@ -29,7 +29,6 @@
  * Modifications Copyright OpenSearch Contributors. See
  * GitHub history for details.
  */
-
 const fetch = require('node-fetch');
 const AbortController = require('abort-controller');
 const fs = require('fs');
@@ -40,8 +39,12 @@ const { createHash } = require('crypto');
 const path = require('path');
 
 const asyncPipeline = promisify(pipeline);
-const DAILY_SNAPSHOTS_BASE_URL = '';
+const DAILY_SNAPSHOTS_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch';
+// TODO: [RENAMEME] currently do not have an existing replacement
+// issue: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/475
 const PERMANENT_SNAPSHOTS_BASE_URL = '';
+// Since we do not have a manifest URL, limiting how many RC checks to 5 should be more than enough
+const MAX_RC_CHECK = 5;
 
 const { cache } = require('./utils');
 const { resolveCustomSnapshotUrl } = require('./custom_snapshots');
@@ -96,6 +99,21 @@ async function fetchSnapshotManifest(url, log) {
   return { abc, resp, json };
 }
 
+async function verifySnapshotUrl(url, log) {
+  log.info('Verifying snapshot URL at %s', chalk.bold(url));
+
+  const abc = new AbortController();
+  const resp = await retry(
+    log,
+    async () => await fetch(url, { signal: abc.signal }, { method: 'HEAD' })
+  );
+
+  return { abc, resp };
+}
+
+/**
+ * @deprecated This method should not be used, uses logic for resources we do not have access to.
+ */
 async function getArtifactSpecForSnapshot(urlVersion, license, log) {
   const desiredVersion = urlVersion.replace('-SNAPSHOT', '');
   const desiredLicense = license === 'oss' ? 'oss' : 'default';
@@ -153,10 +171,87 @@ async function getArtifactSpecForSnapshot(urlVersion, license, log) {
   };
 }
 
+async function getArtifactSpecForSnapshotFromUrl(urlVersion, log) {
+  const desiredVersion = urlVersion.replace('-SNAPSHOT', '');
+
+  if (process.env.OPENSEARCH_SNAPSHOT_MANIFEST) {
+    return await getArtifactSpecForSnapshot(urlVersion, 'oss', log);
+  }
+
+  // [RENAMEME] Need replacement for other platforms.
+  // issue: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/475
+  const platform = process.platform === 'win32' ? 'windows' : process.platform;
+  const arch = process.arch === 'arm64' ? 'arm64' : 'x64';
+  if (platform !== 'linux' || arch !== 'x64') {
+    throw createCliError(`Snapshots are only available for Linux x64`);
+  }
+
+  const latestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}`;
+  const latestFile = `opensearch-${desiredVersion}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`;
+  const completeLatestUrl = `${latestUrl}/${latestFile}`;
+
+  let { abc, resp } = await verifySnapshotUrl(completeLatestUrl, log);
+
+  if (resp.ok) {
+    return {
+      url: completeLatestUrl,
+      checksumUrl: completeLatestUrl + '.sha512',
+      checksumType: 'sha512',
+      filename: latestFile,
+    };
+  }
+
+  log.info(
+    'Daily general-availability snapshot URL not found for current version, falling back to release-candidate snapshot URL.'
+  );
+
+  let completeUrl = null;
+  let snapshotFile = null;
+
+  // This checks and uses an RC if a RC exists at a higher increment than RC1 or it tries to use RC1
+  // This is in replacement of having a manifest URL, so the exact RC number is unknown but expect it not to be a large number
+  let rcCheck = MAX_RC_CHECK;
+  do {
+    const secondaryLatestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}-rc${rcCheck}`;
+    const secondaryLatestFile = `opensearch-${desiredVersion}-rc${rcCheck}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`;
+    const completeSecondaryLatestUrl = `${secondaryLatestUrl}/${secondaryLatestFile}`;
+    ({ abc, resp } = await verifySnapshotUrl(completeSecondaryLatestUrl, log));
+
+    if (resp.ok) {
+      completeUrl = completeSecondaryLatestUrl;
+      snapshotFile = secondaryLatestFile;
+      break;
+    }
+  } while (rcCheck-- >= 1);
+
+  if (resp.status === 404 || !completeUrl || !snapshotFile) {
+    abc.abort();
+    throw createCliError(`Snapshots for ${desiredVersion} are not available`);
+  }
+
+  if (!resp.ok) {
+    abc.abort();
+    throw new Error(`Unable to read snapshot url: ${resp.statusText}`);
+  }
+
+  return {
+    url: completeUrl,
+    checksumUrl: completeUrl + '.sha512',
+    checksumType: 'sha512',
+    filename: snapshotFile,
+  };
+}
+
 exports.Artifact = class Artifact {
   /**
-   * Fetch an Artifact from the Artifact API for a license level and version
-   * @param {('oss'|'basic'|'trial')} license
+   * Fetch an Artifact from the Artifact API for a license level and version.
+   * Only OSS license should be used but the param was left to mitigate impact
+   * until a later iteration.
+   *
+   * TODO: [RENAMEME] remove license param
+   * issue: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/475
+   *
+   * @param {('oss')} license
    * @param {string} version
    * @param {ToolingLog} log
    */
@@ -168,7 +263,7 @@ exports.Artifact = class Artifact {
       return new Artifact(customSnapshotArtifactSpec, log);
     }
 
-    const artifactSpec = await getArtifactSpecForSnapshot(urlVersion, license, log);
+    const artifactSpec = await getArtifactSpecForSnapshotFromUrl(urlVersion, log);
     return new Artifact(artifactSpec, log);
   }
 

packages/osd-opensearch/src/artifact.test.js

@@ -40,26 +40,32 @@ import { Artifact } from './artifact';
 const log = new ToolingLog();
 let MOCKS;
 
+const DAILY_SNAPSHOT_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch';
+
+const ORIGINAL_PLATFROM = process.platform;
+const ORIGINAL_ARCHITECTURE = process.arch;
 const PLATFORM = process.platform === 'win32' ? 'windows' : process.platform;
 const ARCHITECTURE = process.arch === 'arm64' ? 'arm64' : 'x64';
 const MOCK_VERSION = 'test-version';
-const MOCK_URL = 'http://127.0.0.1:12345';
-const MOCK_FILENAME = 'test-filename';
+const MOCK_RC_VERSION = `test-version-rc4`;
+const MOCK_FILENAME = 'opensearch-test-version-SNAPSHOT-linux-x64-latest.tar.gz';
+const MOCK_RC_FILENAME = `opensearch-test-version-rc4-SNAPSHOT-linux-x64-latest.tar.gz`;
+const MOCK_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/${MOCK_FILENAME}`;
+const MOCK_RC_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_RC_VERSION}/${MOCK_RC_FILENAME}`;
 
-const DAILY_SNAPSHOT_BASE_URL = '';
-const PERMANENT_SNAPSHOT_BASE_URL = '';
+const itif = process.platform === 'linux' && process.arch === 'x64' ? it : it.skip;
 
 const createArchive = (params = {}) => {
-  const license = params.license || 'default';
   const architecture = params.architecture || ARCHITECTURE;
+  const useRCVersion = params.useRCVersion || false;
 
   return {
-    license: 'default',
+    license: 'oss',
     architecture,
-    version: MOCK_VERSION,
-    url: MOCK_URL + `/${license}`,
+    version: !useRCVersion ? MOCK_VERSION : MOCK_RC_VERSION,
+    url: !useRCVersion ? MOCK_URL : MOCK_RC_URL,
     platform: PLATFORM,
-    filename: MOCK_FILENAME + `-${architecture}.${license}`,
+    filename: !useRCVersion ? MOCK_FILENAME : MOCK_RC_FILENAME,
     ...params,
   };
 };
@@ -92,99 +98,87 @@ beforeEach(() => {
   jest.resetAllMocks();
 
   MOCKS = {
-    valid: {
-      archives: [createArchive({ license: 'oss' }), createArchive({ license: 'default' })],
+    GA: {
+      archives: [createArchive({ useRCVersion: false })],
+    },
+    RC: {
+      archives: [createArchive({ useRCVersion: true })],
     },
     multipleArch: {
       archives: [
-        createArchive({ architecture: 'fake_arch', license: 'oss' }),
-        createArchive({ architecture: ARCHITECTURE, license: 'oss' }),
+        createArchive({ architecture: 'fake_arch', useRCVersion: false }),
+        createArchive({ architecture: ARCHITECTURE, useRCVersion: false }),
       ],
     },
   };
 });
 
-const artifactTest = (requestedLicense, expectedLicense, fetchTimesCalled = 1) => {
+const artifactTest = (fetchTimesCalled = 1) => {
   return async () => {
-    const artifact = await Artifact.getSnapshot(requestedLicense, MOCK_VERSION, log);
+    const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log);
+    const expectedUrl = fetchTimesCalled === 1 ? MOCK_URL : MOCK_RC_URL;
+    const expectedFilename = fetchTimesCalled === 1 ? MOCK_FILENAME : MOCK_RC_FILENAME;
     expect(fetch).toHaveBeenCalledTimes(fetchTimesCalled);
-    expect(fetch.mock.calls[0][0]).toEqual(
-      `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest-verified.json`
-    );
-    if (fetchTimesCalled === 2) {
-      expect(fetch.mock.calls[1][0]).toEqual(
-        `${PERMANENT_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest.json`
-      );
+    expect(fetch.mock.calls[0][0]).toEqual(MOCK_URL);
+    if (fetchTimesCalled !== 1) {
+      expect(fetch.mock.calls[fetchTimesCalled - 1][0]).toEqual(MOCK_RC_URL);
     }
-    expect(artifact.getUrl()).toEqual(MOCK_URL + `/${expectedLicense}`);
-    expect(artifact.getChecksumUrl()).toEqual(MOCK_URL + `/${expectedLicense}.sha512`);
+    expect(artifact.getUrl()).toEqual(expectedUrl);
+    expect(artifact.getChecksumUrl()).toEqual(expectedUrl + '.sha512');
     expect(artifact.getChecksumType()).toEqual('sha512');
-    expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.${expectedLicense}`);
+    expect(artifact.getFilename()).toEqual(expectedFilename);
   };
 };
 
 describe('Artifact', () => {
   describe('getSnapshot()', () => {
-    describe('with default snapshot', () => {
-      beforeEach(() => {
-        mockFetch(MOCKS.valid);
-      });
-
-      it('should return artifact metadata for a daily oss artifact', artifactTest('oss', 'oss'));
+    itif('should return artifact metadata for a daily GA artifact', () => {
+      mockFetch(MOCKS.GA);
+      artifactTest();
+    });
 
-      it(
-        'should return artifact metadata for a daily default artifact',
-        artifactTest('default', 'default')
-      );
+    itif('should return artifact metadata for a RC artifact', () => {
+      fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 })));
+      fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 })));
+      mockFetch(MOCKS.RC);
+      artifactTest(3);
+    });
 
-      it(
-        'should default to default license with anything other than "oss"',
-        artifactTest('INVALID_LICENSE', 'default')
+    itif('should throw when an artifact cannot be found for the specified parameters', async () => {
+      fetch.mockReturnValue(Promise.resolve(new Response('', { status: 404 })));
+      await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow(
+        'Snapshots for INVALID_VERSION are not available'
       );
-
-      it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => {
-        await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow(
-          "couldn't find an artifact"
-        );
-      });
     });
 
-    describe('with missing default snapshot', () => {
-      beforeEach(() => {
-        fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 })));
-        mockFetch(MOCKS.valid);
+    describe('with snapshots for multiple architectures', () => {
+      afterAll(() => {
+        Object.defineProperties(process, {
+          platform: {
+            value: ORIGINAL_PLATFROM,
+          },
+          arch: {
+            value: ORIGINAL_ARCHITECTURE,
+          },
+        });
       });
 
-      it(
-        'should return artifact metadata for a permanent oss artifact',
-        artifactTest('oss', 'oss', 2)
-      );
-
-      it(
-        'should return artifact metadata for a permanent default artifact',
-        artifactTest('default', 'default', 2)
-      );
-
-      it(
-        'should default to default license with anything other than "oss"',
-        artifactTest('INVALID_LICENSE', 'default', 2)
-      );
-
-      it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => {
-        await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow(
-          "couldn't find an artifact"
+      it('should throw when on a non-Linux platform', async () => {
+        Object.defineProperty(process, 'platform', {
+          value: 'win32',
+        });
+        await expect(Artifact.getSnapshot('default', 'INVALID_PLATFORM', log)).rejects.toThrow(
+          'Snapshots are only available for Linux x64'
         );
       });
-    });
 
-    describe('with snapshots for multiple architectures', () => {
-      beforeEach(() => {
-        mockFetch(MOCKS.multipleArch);
-      });
-
-      it('should return artifact metadata for the correct architecture', async () => {
-        const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log);
-        expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.oss`);
+      it('should throw when on a non-x64 arch', async () => {
+        Object.defineProperty(process, 'arch', {
+          value: 'arm64',
+        });
+        await expect(Artifact.getSnapshot('default', 'INVALID_ARCH', log)).rejects.toThrow(
+          'Snapshots are only available for Linux x64'
+        );
       });
     });
 
@@ -193,7 +187,7 @@ describe('Artifact', () => {
 
       beforeEach(() => {
         process.env.OPENSEARCH_SNAPSHOT_MANIFEST = CUSTOM_URL;
-        mockFetch(MOCKS.valid);
+        mockFetch(MOCKS.GA);
       });
 
       it('should use the custom URL when looking for a snapshot', async () => {
@@ -205,23 +199,5 @@ describe('Artifact', () => {
         delete process.env.OPENSEARCH_SNAPSHOT_MANIFEST;
       });
     });
-
-    describe('with latest unverified snapshot', () => {
-      beforeEach(() => {
-        process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED = 1;
-        mockFetch(MOCKS.valid);
-      });
-
-      it('should use the daily unverified URL when looking for a snapshot', async () => {
-        await Artifact.getSnapshot('oss', MOCK_VERSION, log);
-        expect(fetch.mock.calls[0][0]).toEqual(
-          `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest.json`
-        );
-      });
-
-      afterEach(() => {
-        delete process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED;
-      });
-    });
   });
 });

src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts

@@ -95,8 +95,7 @@ describe('createOrUpgradeSavedConfig()', () => {
     await osd.stop();
   }, 30000);
 
-  // TODO: [RENAMEME] Test can be enabled once there is a valid snapshot URL
-  xit('upgrades the previous version on each increment', async function () {
+  it('upgrades the previous version on each increment', async function () {
     // ------------------------------------
     // upgrade to 5.4.0
     await createOrUpgradeSavedConfig({