Fix flaky test by verifying that a search has been saved by checking for a success toast

src/core_plugins/kibana/public/dashboard/components/__snapshots__/exit_full_screen_button.test.js.snap

@@ -14,6 +14,7 @@ exports[`is rendered 1`] = `
     <button
       aria-label="Exit full screen mode"
       class="kuiButton exitFullScreenMode"
+      data-test-subj="exitFullScreenModeButton"
       type="hollow"
     >
       <span
@@ -22,11 +23,9 @@ exports[`is rendered 1`] = `
         <span>
           <span
             class="exitFullScreenModeLogo"
-            data-test-subj="exitFullScreenModeLogo"
           />
           <span
             class="exitFullScreenModeText"
-            data-test-subj="exitFullScreenModeText"
           >
             Exit full screen
             <span

src/core_plugins/kibana/public/dashboard/components/exit_full_screen_button.js

@@ -64,9 +64,10 @@ export class ExitFullScreenButton extends PureComponent {
             aria-label="Exit full screen mode"
             className="exitFullScreenMode"
             onClick={this.props.onExitFullScreenMode}
+            data-test-subj="exitFullScreenModeButton"
           >
-            <span className="exitFullScreenModeLogo" data-test-subj="exitFullScreenModeLogo"/>
-            <span className="exitFullScreenModeText" data-test-subj="exitFullScreenModeText">
+            <span className="exitFullScreenModeLogo" />
+            <span className="exitFullScreenModeText">
               Exit full screen
               <span className="kuiIcon fa fa-angle-left"/>
             </span>

src/ui/public/notify/partials/toaster.html

@@ -1,6 +1,6 @@
 <div class="toaster-container">
   <ul class="toaster">
-    <li ng-repeat="notif in list" kbn-toast notif="notif">
+    <li ng-repeat="notif in list" kbn-toast notif="notif" data-test-subj="bootstrapToast">
       <div ng-class="notif.getAlertClass()">
 
         <span ng-show="notif.count > 1" class="euiBadge euiBadge--default">

src/ui/public/styles/base.less

@@ -4,6 +4,14 @@
 @import (reference) "~ui/styles/bootstrap/bootstrap";
 @import "./react-input-range";
 
+// NOTE: This is an experiment to see if functional tests are less flaky with animations disabled.
+* {
+  -webkit-animation-duration: 0s !important;
+  animation-duration: 0s !important;
+  -webkit-transition-duration: 0s !important;
+  transition-duration: 0s !important;
+}
+
 .small {
   font-size: 0.9em !important;
 }

test/functional/apps/dashboard/_bwc_shared_urls.js

@@ -73,7 +73,7 @@ export default function ({ getService, getPageObjects }) {
       });
 
       it('loads a saved dashboard', async function () {
-        await PageObjects.dashboard.saveDashboard('saved with colors', { storeTimeWithDashboard: true });
+        await PageObjects.dashboard.saveDashboardAndVerify('saved with colors', { storeTimeWithDashboard: true });
 
         const id = await PageObjects.dashboard.getDashboardIdFromCurrentUrl();
         const url = `${kibanaBaseUrl}#/dashboard/${id}`;

test/functional/apps/dashboard/_create_and_add_embeddables.js

@@ -42,7 +42,7 @@ export default function ({ getService, getPageObjects }) {
         await dashboardAddPanel.clickAddNewEmbeddableLink();
         await PageObjects.visualize.clickAreaChart();
         await PageObjects.visualize.clickNewSearch();
-        await PageObjects.visualize.saveVisualization('visualization from add new link');
+        await PageObjects.visualize.saveVisualizationAndVerify('visualization from add new link');
 
         return retry.try(async () => {
           const panelCount = await PageObjects.dashboard.getPanelCount();

test/functional/apps/dashboard/_dashboard_filter_bar.js

@@ -100,7 +100,7 @@ export default function ({ getService, getPageObjects }) {
       });
 
       it('are preserved after saving a dashboard', async () => {
-        await PageObjects.dashboard.saveDashboard('with filters');
+        await PageObjects.dashboard.saveDashboardAndVerify('with filters');
         await PageObjects.header.waitUntilLoadingHasFinished();
 
         const filters = await PageObjects.dashboard.getFilters();

test/functional/apps/dashboard/_dashboard_filtering.js

@@ -241,7 +241,7 @@ export default function ({ getService, getPageObjects }) {
         await renderable.waitForRender();
         await dashboardExpect.pieSliceCount(3);
 
-        await PageObjects.visualize.saveVisualization('Rendering Test: animal sounds pie');
+        await PageObjects.visualize.saveVisualizationAndVerify('Rendering Test: animal sounds pie');
         await PageObjects.header.clickDashboard();
         await PageObjects.header.waitUntilLoadingHasFinished();
         await PageObjects.dashboard.waitForRenderComplete();
@@ -256,7 +256,7 @@ export default function ({ getService, getPageObjects }) {
         await PageObjects.header.waitUntilLoadingHasFinished();
         await dashboardExpect.pieSliceCount(1);
 
-        await PageObjects.visualize.saveVisualization('animal sounds pie');
+        await PageObjects.visualize.saveVisualizationAndVerify('animal sounds pie');
         await PageObjects.header.clickDashboard();
 
         await dashboardExpect.pieSliceCount(1);

test/functional/apps/dashboard/_dashboard_listing.js

@@ -38,7 +38,7 @@ export default function ({ getService, getPageObjects }) {
 
       it('creates a new dashboard', async function () {
         await PageObjects.dashboard.clickCreateDashboardPrompt();
-        await PageObjects.dashboard.saveDashboard(dashboardName);
+        await PageObjects.dashboard.saveDashboardAndVerify(dashboardName);
 
         await PageObjects.dashboard.gotoDashboardLandingPage();
         const countOfDashboards = await PageObjects.dashboard.getDashboardCountWithName(dashboardName);
@@ -90,7 +90,7 @@ export default function ({ getService, getPageObjects }) {
       before(async () => {
         await PageObjects.dashboard.clearSearchValue();
         await PageObjects.dashboard.clickNewDashboard();
-        await PageObjects.dashboard.saveDashboard('Two Words');
+        await PageObjects.dashboard.saveDashboardAndVerify('Two Words');
       });
 
       it('matches on the first word', async function () {
@@ -168,7 +168,7 @@ export default function ({ getService, getPageObjects }) {
 
       it('stays on listing page if title matches two dashboards', async function () {
         await PageObjects.dashboard.clickNewDashboard();
-        await PageObjects.dashboard.saveDashboard('two words', { needsConfirm: true });
+        await PageObjects.dashboard.saveDashboardAndVerify('two words', { needsConfirm: true });
         await PageObjects.dashboard.gotoDashboardLandingPage();
         const currentUrl = await remote.getCurrentUrl();
         const newUrl = currentUrl + '&title=two%20words';
@@ -188,7 +188,7 @@ export default function ({ getService, getPageObjects }) {
 
       it('matches a title with many special characters', async function () {
         await PageObjects.dashboard.clickNewDashboard();
-        await PageObjects.dashboard.saveDashboard('i am !@#$%^&*()_+~`,.<>{}[]; so special');
+        await PageObjects.dashboard.saveDashboardAndVerify('i am !@#$%^&*()_+~`,.<>{}[]; so special');
         await PageObjects.dashboard.gotoDashboardLandingPage();
         const currentUrl = await remote.getCurrentUrl();
         // Need to encode that one.

test/functional/apps/dashboard/_dashboard_save.js

@@ -36,7 +36,7 @@ export default function ({ getService, getPageObjects }) {
 
     it('warns on duplicate name for new dashboard', async function () {
       await PageObjects.dashboard.clickNewDashboard();
-      await PageObjects.dashboard.saveDashboard(dashboardName);
+      await PageObjects.dashboard.saveDashboardAndVerify(dashboardName);
 
       let isWarningDisplayed = await PageObjects.dashboard.isDuplicateTitleWarningDisplayed();
       expect(isWarningDisplayed).to.equal(false);
@@ -60,13 +60,15 @@ export default function ({ getService, getPageObjects }) {
       await PageObjects.dashboard.gotoDashboardLandingPage();
       await PageObjects.dashboard.clickNewDashboard();
       await PageObjects.dashboard.enterDashboardTitleAndClickSave(dashboardName);
-
       await PageObjects.dashboard.clickSave();
 
-      // This is important since saving a new dashboard will cause a refresh of the page. We have to
-      // wait till it finishes reloading or it might reload the url after simulating the
-      // dashboard landing page click.
-      await PageObjects.header.waitUntilLoadingHasFinished();
+      await Promise.all([
+        PageObjects.dashboard.verifySaveSuccess(),
+        // This is important since saving a new dashboard will cause a refresh of the page. We have to
+        // wait till it finishes reloading or it might reload the url after simulating the
+        // dashboard landing page click.
+        PageObjects.header.waitUntilLoadingHasFinished(),
+      ]);
 
       const countOfDashboards = await PageObjects.dashboard.getDashboardCountWithName(dashboardName);
       expect(countOfDashboards).to.equal(2);
@@ -77,8 +79,7 @@ export default function ({ getService, getPageObjects }) {
         await PageObjects.dashboard.selectDashboard(dashboardName);
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
         await PageObjects.dashboard.clickEdit();
-        await PageObjects.dashboard.saveDashboard(dashboardName);
-
+        await PageObjects.dashboard.saveDashboardAndVerify(dashboardName);
         const isWarningDisplayed = await PageObjects.dashboard.isDuplicateTitleWarningDisplayed();
         expect(isWarningDisplayed).to.equal(false);
       }
@@ -95,7 +96,7 @@ export default function ({ getService, getPageObjects }) {
     });
 
     it('Does not warn when only the prefix matches', async function () {
-      await PageObjects.dashboard.saveDashboard(dashboardName.split(' ')[0]);
+      await PageObjects.dashboard.saveDashboardAndVerify(dashboardName.split(' ')[0]);
 
       const isWarningDisplayed = await PageObjects.dashboard.isDuplicateTitleWarningDisplayed();
       expect(isWarningDisplayed).to.equal(false);

test/functional/apps/dashboard/_dashboard_snapshots.js

@@ -44,18 +44,16 @@ export default function ({ getService, getPageObjects, updateBaselines }) {
       await PageObjects.dashboard.clickNewDashboard();
       await PageObjects.dashboard.setTimepickerInLogstashDataRange();
       await dashboardAddPanel.addVisualization('Rendering Test: tsvb-ts');
-      await PageObjects.common.closeToast();
 
-      await PageObjects.dashboard.saveDashboard('tsvb');
-      await PageObjects.common.closeToast();
+      await PageObjects.dashboard.saveDashboardAndVerify('tsvb');
 
       await PageObjects.dashboard.clickFullScreenMode();
       await dashboardPanelActions.toggleExpandPanel();
 
       await PageObjects.dashboard.waitForRenderComplete();
       const percentSimilar = await screenshot.compareAgainstBaseline('tsvb_dashboard', updateBaselines);
 
-      await PageObjects.dashboard.clickExitFullScreenLogoButton();
+      await PageObjects.dashboard.clickExitFullScreenButton();
 
       expect(percentSimilar).to.be.lessThan(0.05);
     });
@@ -65,17 +63,15 @@ export default function ({ getService, getPageObjects, updateBaselines }) {
       await PageObjects.dashboard.clickNewDashboard();
       await PageObjects.dashboard.setTimepickerInLogstashDataRange();
       await dashboardAddPanel.addVisualization('Rendering Test: area with not filter');
-      await PageObjects.common.closeToast();
-      await PageObjects.dashboard.saveDashboard('area');
-      await PageObjects.common.closeToast();
+      await PageObjects.dashboard.saveDashboardAndVerify('area');
 
       await PageObjects.dashboard.clickFullScreenMode();
       await dashboardPanelActions.toggleExpandPanel();
 
       await PageObjects.dashboard.waitForRenderComplete();
       const percentSimilar = await screenshot.compareAgainstBaseline('area_chart', updateBaselines);
 
-      await PageObjects.dashboard.clickExitFullScreenLogoButton();
+      await PageObjects.dashboard.clickExitFullScreenButton();
 
       // Testing some OS/browser differences were shown to cause .009 percent difference.
       expect(percentSimilar).to.be.lessThan(0.05);

test/functional/apps/dashboard/_dashboard_state.js

@@ -50,14 +50,14 @@ export default function ({ getService, getPageObjects }) {
       await PageObjects.dashboard.setTimepickerInHistoricalDataRange();
 
       await dashboardAddPanel.addVisualization(AREA_CHART_VIS_NAME);
-      await PageObjects.dashboard.saveDashboard('Overridden colors');
+      await PageObjects.dashboard.saveDashboardAndVerify('Overridden colors');
 
       await PageObjects.dashboard.clickEdit();
 
       await PageObjects.visualize.openLegendOptionColors('Count');
       await PageObjects.visualize.selectNewLegendColorChoice('#EA6460');
 
-      await PageObjects.dashboard.saveDashboard('Overridden colors');
+      await PageObjects.dashboard.saveDashboardAndVerify('Overridden colors');
 
       await PageObjects.dashboard.gotoDashboardLandingPage();
       await PageObjects.dashboard.loadSavedDashboard('Overridden colors');
@@ -79,7 +79,7 @@ export default function ({ getService, getPageObjects }) {
       await PageObjects.dashboard.clickNewDashboard();
 
       await dashboardAddPanel.addSavedSearch('my search');
-      await PageObjects.dashboard.saveDashboard('No local edits');
+      await PageObjects.dashboard.saveDashboardAndVerify('No local edits');
 
       const inViewMode = await testSubjects.exists('dashboardEditMode');
       expect(inViewMode).to.be(true);
@@ -101,7 +101,7 @@ export default function ({ getService, getPageObjects }) {
     it('Saved search with column changes will not update when the saved object changes', async () => {
       await PageObjects.discover.removeHeaderColumn('bytes');
       await PageObjects.dashboard.clickEdit();
-      await PageObjects.dashboard.saveDashboard('Has local edits');
+      await PageObjects.dashboard.saveDashboardAndVerify('Has local edits');
 
       await PageObjects.header.clickDiscover();
       await PageObjects.discover.clickFieldListItemAdd('clientip');
@@ -123,7 +123,7 @@ export default function ({ getService, getPageObjects }) {
       await PageObjects.dashboard.setTimepickerInHistoricalDataRange();
 
       await dashboardAddPanel.addVisualization('Visualization TileMap');
-      await PageObjects.dashboard.saveDashboard('No local edits');
+      await PageObjects.dashboard.saveDashboardAndVerify('No local edits');
 
       await dashboardPanelActions.openInspector();
       const tileMapData = await PageObjects.visualize.getInspectorTableData();
@@ -140,7 +140,7 @@ export default function ({ getService, getPageObjects }) {
       await PageObjects.visualize.clickGo();
       await PageObjects.header.waitUntilLoadingHasFinished();
 
-      await PageObjects.visualize.saveVisualization('Visualization TileMap');
+      await PageObjects.visualize.saveVisualizationAndVerify('Visualization TileMap');
 
       await PageObjects.header.clickDashboard();
 

test/functional/apps/dashboard/_dashboard_time.js

@@ -42,8 +42,7 @@ export default function ({ getPageObjects, getService }) {
       it('is saved', async () => {
         await PageObjects.dashboard.clickNewDashboard();
         await PageObjects.dashboard.addVisualizations([PageObjects.dashboard.getTestVisualizationNames()[0]]);
-        const isDashboardSaved = await PageObjects.dashboard.saveDashboard(dashboardName, { storeTimeWithDashboard: false });
-        expect(isDashboardSaved).to.eql(true);
+        await PageObjects.dashboard.saveDashboardAndVerify(dashboardName, { storeTimeWithDashboard: false });
       });
 
       it('Does not set the time picker on open', async () => {
@@ -62,8 +61,7 @@ export default function ({ getPageObjects, getService }) {
       it('is saved with quick time', async function () {
         await PageObjects.dashboard.clickEdit();
         await PageObjects.header.setQuickTime('Today');
-        const isDashboardSaved = await PageObjects.dashboard.saveDashboard(dashboardName, { storeTimeWithDashboard: true });
-        expect(isDashboardSaved).to.eql(true);
+        await PageObjects.dashboard.saveDashboardAndVerify(dashboardName, { storeTimeWithDashboard: true });
       });
 
       it('sets quick time on open', async function () {
@@ -78,8 +76,7 @@ export default function ({ getPageObjects, getService }) {
       it('is saved with absolute time', async function () {
         await PageObjects.dashboard.clickEdit();
         await PageObjects.header.setAbsoluteRange(fromTime, toTime);
-        const isDashboardSaved = await PageObjects.dashboard.saveDashboard(dashboardName, { storeTimeWithDashboard: true });
-        expect(isDashboardSaved).to.eql(true);
+        await PageObjects.dashboard.saveDashboardAndVerify(dashboardName, { storeTimeWithDashboard: true });
       });
 
       it('sets absolute time on open', async function () {

test/functional/apps/dashboard/_embeddable_rendering.js

@@ -32,7 +32,7 @@ export default function ({ getService, getPageObjects }) {
   const remote = getService('remote');
   const dashboardExpect = getService('dashboardExpect');
   const dashboardAddPanel = getService('dashboardAddPanel');
-  const PageObjects = getPageObjects(['common', 'dashboard', 'header', 'visualize', 'discover']);
+  const PageObjects = getPageObjects(['dashboard', 'header', 'visualize', 'discover']);
 
   const expectAllDataRenders = async () => {
     await dashboardExpect.pieSliceCount(16);
@@ -114,7 +114,13 @@ export default function ({ getService, getPageObjects }) {
       const panelCount = await PageObjects.dashboard.getPanelCount();
       expect(panelCount).to.be(28);
 
-      await PageObjects.dashboard.saveDashboard('embeddable rendering test', { storeTimeWithDashboard: true });
+      // Saving a dashboard causes the panels to refresh, which can be heavy on the browser.
+      // If the re-render takes too long, then the toast might disappear before selenium
+      // gets a chance to interact with it. This can cause flakiness, so we'll just ignore the
+      // success toast this time.
+      await PageObjects.dashboard.saveDashboard('embeddable rendering test', {
+        storeTimeWithDashboard: true,
+      });
     });
 
     it('initial render test', async () => {

test/functional/apps/dashboard/_full_screen_mode.js

@@ -21,13 +21,16 @@ import expect from 'expect.js';
 
 export default function ({ getService, getPageObjects }) {
   const retry = getService('retry');
-  const remote = getService('remote');
   const dashboardPanelActions = getService('dashboardPanelActions');
-  const PageObjects = getPageObjects(['dashboard', 'common']);
+  const PageObjects = getPageObjects(['dashboard', 'common', 'header']);
 
   describe('full screen mode', async () => {
     before(async () => {
       await PageObjects.dashboard.loadSavedDashboard('few panels');
+
+      const fromTime = '2018-04-09 21:56:08.000';
+      const toTime = '2018-04-11 21:56:08.000';
+      await PageObjects.header.setAbsoluteRange(fromTime, toTime);
     });
 
     it('option not available in edit mode', async () => {
@@ -37,7 +40,7 @@ export default function ({ getService, getPageObjects }) {
     });
 
     it('available in view mode', async () => {
-      await PageObjects.dashboard.saveDashboard('full screen test', { saveAsNew: true });
+      await PageObjects.dashboard.saveDashboardAndVerify('full screen test', { saveAsNew: true });
       const exists = await PageObjects.dashboard.fullScreenModeMenuItemExists();
       expect(exists).to.be(true);
     });
@@ -55,21 +58,23 @@ export default function ({ getService, getPageObjects }) {
     });
 
     it('displays exit full screen logo button', async () => {
-      const exists = await PageObjects.dashboard.exitFullScreenLogoButtonExists();
+      const exists = await PageObjects.dashboard.exitFullScreenButtonExists();
       expect(exists).to.be(true);
     });
 
     it('displays exit full screen logo button when panel is expanded', async () => {
       await dashboardPanelActions.toggleExpandPanel();
 
-      const exists = await PageObjects.dashboard.exitFullScreenTextButtonExists();
+      const exists = await PageObjects.dashboard.exitFullScreenButtonExists();
       expect(exists).to.be(true);
     });
 
-    it('exits when the text button is clicked on', async () => {
-      const logoButton = await PageObjects.dashboard.getExitFullScreenLogoButton();
-      await remote.moveMouseTo(logoButton);
-      await PageObjects.dashboard.clickExitFullScreenTextButton();
+    it('exits when the full screen logo button is clicked on', async () => {
+      await retry.try(async () => {
+        await PageObjects.dashboard.exitFullScreenButtonExists();
+      });
+
+      await PageObjects.dashboard.clickExitFullScreenButton();
 
       await retry.try(async () => {
         const isChromeVisible = await PageObjects.common.isChromeVisible();