Stabilize and bring back flaky tests (#13034) (#13160)

* Stabalize tests and bring back flaky test

* Try to fix flaky map radius visualize test

* fix flaky tile map zoom test

* Try to fix flaky zoom test by avoiding zooming in and out with spy panel open

* Remove duplicate wrapped retry around find methods

The retry was moved internally, around a check for stale elements as
well, so this should no longer be neccessary.

test/functional/apps/dashboard/_dashboard_save.js

@@ -1,6 +1,7 @@
 import expect from 'expect.js';
 
-export default function ({ getPageObjects }) {
+export default function ({ getService, getPageObjects }) {
+  const retry = getService('retry');
   const PageObjects = getPageObjects(['dashboard', 'header', 'common']);
 
   describe('dashboard save', function describeIndexTests() {
@@ -88,8 +89,11 @@ export default function ({ getPageObjects }) {
       await PageObjects.dashboard.clickEdit();
       await PageObjects.dashboard.enterDashboardTitleAndClickSave(dashboardName.toUpperCase());
 
-      const isConfirmOpen = await PageObjects.common.isConfirmModalOpen();
-      expect(isConfirmOpen).to.equal(true);
+      // We expect isConfirmModalOpen to be open, hence the retry if not found.
+      await retry.try(async () => {
+        const isConfirmOpen = await PageObjects.common.isConfirmModalOpen();
+        expect(isConfirmOpen).to.equal(true);
+      });
 
       await PageObjects.common.clickCancelOnModal();
     });

test/functional/apps/dashboard/_dashboard_time.js

@@ -29,7 +29,7 @@ export default function ({ getPageObjects }) {
         await PageObjects.header.clickToastOK();
       });
 
-      it.skip('Does not set the time picker on open', async function () {
+      it('Does not set the time picker on open', async function () {
         await PageObjects.header.setAbsoluteRange(fromTime, toTime);
 
         await PageObjects.dashboard.loadSavedDashboard(dashboardName);
@@ -81,7 +81,7 @@ export default function ({ getPageObjects }) {
     // when it's opened. However, if the user then changes the time, navigates to visualize, then navigates
     // back to dashboard, the overridden time should be preserved. The time is *only* reset on open, not
     // during navigation or page refreshes.
-    describe.skip('time changes', function () {
+    describe('time changes', function () {
       it('preserved during navigation', async function () {
         await PageObjects.header.setQuickTime('Today');
         await PageObjects.header.clickVisualize();

test/functional/apps/dashboard/_view_edit.js

@@ -164,7 +164,7 @@ export default function ({ getService, getPageObjects }) {
           expect(query).to.equal(originalQuery);
         });
 
-        it.skip('when a filter is deleted', async function () {
+        it('when a filter is deleted', async function () {
           await PageObjects.dashboard.setTimepickerInDataRange();
           await PageObjects.dashboard.filterOnPieSlice();
           await PageObjects.dashboard.saveDashboard(dashboardName);

test/functional/apps/discover/_discover.js

@@ -230,7 +230,7 @@ export default function ({ getService, getPageObjects }) {
     });
 
     describe('data-shared-item', function () {
-      it.skip('should have correct data-shared-item title and description', async () => {
+      it('should have correct data-shared-item title and description', async () => {
         const expected = {
           title: 'A Saved Search',
           description: 'A Saved Search Description'

test/functional/apps/discover/_field_data.js

@@ -1,39 +1,28 @@
 import expect from 'expect.js';
 
 export default function ({ getService, getPageObjects }) {
-  const log = getService('log');
   const retry = getService('retry');
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
   const screenshots = getService('screenshots');
   const PageObjects = getPageObjects(['common', 'header', 'discover']);
 
   describe('discover app', function describeIndexTests() {
-    before(function () {
+    before(async function () {
       const fromTime = '2015-09-19 06:31:44.000';
       const toTime = '2015-09-23 18:31:44.000';
 
+      await esArchiver.loadIfNeeded('logstash_functional');
+      await esArchiver.load('discover');
+      await kibanaServer.waitForStabilization();
       // delete .kibana index and update configDoc
-      return kibanaServer.uiSettings.replace({
+      await kibanaServer.uiSettings.replace({
         'dateFormat:tz': 'UTC',
         'defaultIndex': 'logstash-*'
-      })
-      .then(function loadkibanaIndexPattern() {
-        log.debug('load kibana index with default index pattern');
-        return esArchiver.load('discover');
-      })
-      // and load a set of makelogs data
-      .then(function loadIfEmptyMakelogs() {
-        return esArchiver.loadIfNeeded('logstash_functional');
-      })
-      .then(function () {
-        log.debug('discover');
-        return PageObjects.common.navigateToApp('discover');
-      })
-      .then(function () {
-        log.debug('setAbsoluteRange');
-        return PageObjects.header.setAbsoluteRange(fromTime, toTime);
       });
+
+      await PageObjects.common.navigateToApp('discover');
+      await PageObjects.header.setAbsoluteRange(fromTime, toTime);
     });
 
 

test/functional/apps/visualize/_area_chart.js

@@ -68,7 +68,7 @@ export default function ({ getService, getPageObjects }) {
           expect(message).to.be(`Visualization Editor: Saved Visualization "${vizNamewithSpecialChars}"`);
         })
         .then(function testVisualizeWaitForToastMessageGone() {
-          return PageObjects.visualize.waitForToastMessageGone();
+          return PageObjects.header.waitForToastMessageGone();
         });
       });
 
@@ -79,7 +79,7 @@ export default function ({ getService, getPageObjects }) {
         log.debug(`Saved viz message with umlaut = ${message}`);
         expect(message).to.be(`Visualization Editor: Saved Visualization "${vizNamewithSpecialChars}"`);
 
-        await PageObjects.visualize.waitForToastMessageGone();
+        await PageObjects.header.waitForToastMessageGone();
       });
 
       it('should save and load', function () {
@@ -90,7 +90,7 @@ export default function ({ getService, getPageObjects }) {
           expect(message).to.be('Visualization Editor: Saved Visualization \"' + vizName1 + '\"');
         })
         .then(function testVisualizeWaitForToastMessageGone() {
-          return PageObjects.visualize.waitForToastMessageGone();
+          return PageObjects.header.waitForToastMessageGone();
         })
         .then(function loadSavedVisualization() {
           return PageObjects.visualize.loadSavedVisualization(vizName1);

test/functional/apps/visualize/_data_table.js

@@ -59,7 +59,7 @@ export default function ({ getService, getPageObjects }) {
           expect(message).to.be('Visualization Editor: Saved Visualization \"' + vizName1 + '\"');
         })
         .then(function testVisualizeWaitForToastMessageGone() {
-          return PageObjects.visualize.waitForToastMessageGone();
+          return PageObjects.header.waitForToastMessageGone();
         })
         .then(function () {
           return PageObjects.visualize.loadSavedVisualization(vizName1);

test/functional/apps/visualize/_heatmap_chart.js

@@ -3,6 +3,7 @@ import expect from 'expect.js';
 export default function ({ getService, getPageObjects }) {
   const log = getService('log');
   const screenshots = getService('screenshots');
+  const retry = getService('retry');
   const PageObjects = getPageObjects(['common', 'visualize', 'header']);
 
   describe('visualize app', function describeIndexTests() {
@@ -57,7 +58,7 @@ export default function ({ getService, getPageObjects }) {
           expect(message).to.be('Visualization Editor: Saved Visualization \"' + vizName1 + '\"');
         })
         .then(function testVisualizeWaitForToastMessageGone() {
-          return PageObjects.visualize.waitForToastMessageGone();
+          return PageObjects.header.waitForToastMessageGone();
         })
         .then(function () {
           return PageObjects.visualize.loadSavedVisualization(vizName1);
@@ -70,20 +71,14 @@ export default function ({ getService, getPageObjects }) {
         });
       });
 
-      it('should show correct chart, take screenshot', function () {
+      it('should show correct chart, take screenshot', async function () {
         const expectedChartValues = ['0 - 400', '0 - 400', '400 - 800', '1,200 - 1,600',
           '1,200 - 1,600', '400 - 800', '0 - 400', '0 - 400', '0 - 400', '0 - 400', '400 - 800',
           '1,200 - 1,600', '1,200 - 1,600', '400 - 800', '0 - 400', '0 - 400', '0 - 400', '0 - 400',
           '400 - 800', '1,200 - 1,600', '1,200 - 1,600', '400 - 800', '0 - 400', '0 - 400' ];
 
-        // Most recent failure on Jenkins usually indicates the bar chart is still being drawn?
-        // return arguments[0].getAttribute(arguments[1]);","args":[{"ELEMENT":"592"},"fill"]}] arguments[0].getAttribute is not a function
-        // try sleeping a bit before getting that data
-        return PageObjects.common.sleep(5000)
-        .then(function () {
-          return PageObjects.visualize.getHeatmapData();
-        })
-        .then(function showData(data) {
+        await retry.try(async () => {
+          const data = await PageObjects.visualize.getHeatmapData();
           log.debug('data=' + data);
           log.debug('data.length=' + data.length);
           screenshots.take('Visualize-heatmap-chart');

test/functional/apps/visualize/_line_chart.js

@@ -131,7 +131,7 @@ export default function ({ getService, getPageObjects }) {
           expect(message).to.be('Visualization Editor: Saved Visualization \"' + vizName1 + '\"');
         })
         .then(function testVisualizeWaitForToastMessageGone() {
-          return PageObjects.visualize.waitForToastMessageGone();
+          return PageObjects.header.waitForToastMessageGone();
         })
         .then(function () {
           return PageObjects.visualize.loadSavedVisualization(vizName1);

test/functional/apps/visualize/_pie_chart.js

@@ -65,7 +65,7 @@ export default function ({ getService, getPageObjects }) {
           expect(message).to.be('Visualization Editor: Saved Visualization \"' + vizName1 + '\"');
         })
         .then(function testVisualizeWaitForToastMessageGone() {
-          return PageObjects.visualize.waitForToastMessageGone();
+          return PageObjects.header.waitForToastMessageGone();
         })
         .then(function () {
           return PageObjects.visualize.loadSavedVisualization(vizName1);

test/functional/apps/visualize/_point_series_options.js

@@ -2,6 +2,7 @@ import expect from 'expect.js';
 
 export default function ({ getService, getPageObjects }) {
   const log = getService('log');
+  const retry = getService('retry');
   const screenshots = getService('screenshots');
   const PageObjects = getPageObjects(['common', 'visualize', 'header', 'pointSeries']);
   const pointSeriesVis = PageObjects.pointSeries;
@@ -82,7 +83,7 @@ export default function ({ getService, getPageObjects }) {
 
     describe('secondary value axis', function () {
 
-      it('should show correct chart, take screenshot', function () {
+      it('should show correct chart, take screenshot', async function () {
         const expectedChartValues = [
           [ 37, 202, 740, 1437, 1371, 751, 188, 31, 42, 202, 683,
             1361, 1415, 707, 177, 27, 32, 175, 707, 1408, 1355, 726, 201, 29 ],
@@ -92,27 +93,18 @@ export default function ({ getService, getPageObjects }) {
             12807319386, 13375732998, 13190755620, 12627508458, 12731510199, 13153337344  ],
         ];
 
-        // Most recent failure on Jenkins usually indicates the bar chart is still being drawn?
-        // return arguments[0].getAttribute(arguments[1]);","args":[{"ELEMENT":"592"},"fill"]}] arguments[0].getAttribute is not a function
-        // try sleeping a bit before getting that data
-        return PageObjects.common.sleep(2000)
-          .then(function () {
-            return PageObjects.visualize.getLineChartData('fill="#00a69b"');
-          })
-          .then(function showData(data) {
-            log.debug('count data=' + data);
-            log.debug('data.length=' + data.length);
-            screenshots.take('Visualize-secondary-value-axis');
-            expect(data).to.eql(expectedChartValues[0]);
-          })
-          .then(function () {
-            return PageObjects.visualize.getLineChartData('fill="#57c17b"', 'ValueAxis-2');
-          })
-          .then(function showData(data) {
-            log.debug('average memory data=' + data);
-            log.debug('data.length=' + data.length);
-            expect(data).to.eql(expectedChartValues[1]);
-          });
+        await retry.try(async () => {
+          const data = await PageObjects.visualize.getLineChartData('fill="#00a69b"');
+          log.debug('count data=' + data);
+          log.debug('data.length=' + data.length);
+          screenshots.take('Visualize-secondary-value-axis');
+          expect(data).to.eql(expectedChartValues[0]);
+
+          const avgMemoryData = await PageObjects.visualize.getLineChartData('fill="#57c17b"', 'ValueAxis-2');
+          log.debug('average memory data=' + avgMemoryData);
+          log.debug('data.length=' + avgMemoryData.length);
+          expect(avgMemoryData).to.eql(expectedChartValues[1]);
+        });
       });
 
       it('should put secondary axis on the right', function () {

test/functional/apps/visualize/_tag_cloud.js

@@ -53,14 +53,10 @@ export default function ({ getService, getPageObjects }) {
     describe('tile cloud chart', function indexPatternCreation() {
       const vizName1 = 'Visualization tagCloud';
 
-      it('should show correct tag cloud data', function () {
-        return PageObjects.common.sleep(2000)
-          .then(function () {
-            return PageObjects.visualize.getTextTag().then(function (results) {
-              log.debug(results);
-              expect(results).to.eql([ '32212254720', '21474836480','20401094656','19327352832','18253611008' ]);
-            });
-          });
+      it('should show correct tag cloud data', async function () {
+        const data = await PageObjects.visualize.getTextTag();
+        log.debug(data);
+        expect(data).to.eql([ '32212254720', '21474836480','20401094656','19327352832','18253611008' ]);
       });
 
 
@@ -71,7 +67,7 @@ export default function ({ getService, getPageObjects }) {
           expect(message).to.be('Visualization Editor: Saved Visualization \"' + vizName1 + '\"');
         })
           .then(function testVisualizeWaitForToastMessageGone() {
-            return PageObjects.visualize.waitForToastMessageGone();
+            return PageObjects.header.waitForToastMessageGone();
           })
           .then(function () {
             return PageObjects.visualize.loadSavedVisualization(vizName1);

test/functional/apps/visualize/_tile_map.js

@@ -90,54 +90,34 @@ export default function ({ getService, getPageObjects }) {
 
     describe('tile map chart', function indexPatternCreation() {
 
-      it('should show correct tile map data on default zoom level', function () {
+      it('should show correct tile map data on default zoom level', async function () {
         const expectedTableData = ['9 5,787 { "lat": 37.22448418632405, "lon": -103.01935195013255 }',
           'd 5,600 { "lat": 37.44271478370398, "lon": -81.72692197253595 }',
           'c 1,319 { "lat": 47.72720855392425, "lon": -109.84745063951028 }',
           'b 999 { "lat": 62.04130042948433, "lon": -155.28087269195967 }',
           'f 187 { "lat": 45.656166475784175, "lon": -82.45831044201545 }',
           '8 108 { "lat": 18.85260305600241, "lon": -156.5148810390383 }'];
+        //level 1
+        await PageObjects.visualize.clickMapZoomOut();
+        //level 0
+        await PageObjects.visualize.clickMapZoomOut();
 
-        return PageObjects.visualize.openSpyPanel()
-        .then(function () {
-          //level 1
-          return PageObjects.visualize.clickMapZoomOut();
-        })
-        .then(function () {
-          //level 0
-          return PageObjects.visualize.clickMapZoomOut();
-        })
-        .then(function () {
-          return PageObjects.settings.setPageSize('All');
-        })
-        .then(function getDataTableData() {
-          return PageObjects.visualize.getDataTableData()
-          .then(function showData(actualTableData) {
-            compareTableData(expectedTableData, actualTableData.trim().split('\n'));
-            return PageObjects.visualize.closeSpyPanel();
-          });
-        });
+        await PageObjects.visualize.openSpyPanel();
+        await PageObjects.settings.setPageSize('All');
+        const actualTableData = await PageObjects.visualize.getDataTableData();
+        compareTableData(expectedTableData, actualTableData.trim().split('\n'));
+        await PageObjects.visualize.closeSpyPanel();
       });
 
-      it('should not be able to zoom out beyond 0', function () {
-        return PageObjects.visualize.getMapZoomOutEnabled()
-        // we can tell we're at level 1 because zoom out is disabled
-        .then(function () {
-          return retry.try(function tryingForTime() {
-            return PageObjects.visualize.getMapZoomOutEnabled()
-            .then(function (enabled) {
-              //should be able to zoom more as current config has 0 as min level.
-              expect(enabled).to.be(false);
-            });
-          });
-        })
-        .then(function takeScreenshot() {
-          log.debug('Take screenshot (success)');
-          screenshots.take('map-at-zoom-0');
-        });
+      it('should not be able to zoom out beyond 0', async function () {
+        await PageObjects.visualize.zoomAllTheWayOut();
+        const enabled = await PageObjects.visualize.getMapZoomOutEnabled();
+        expect(enabled).to.be(false);
+        screenshots.take('map-at-zoom-0');
       });
 
-      it('Fit data bounds should zoom to level 3', function () {
+      // See https://github.com/elastic/kibana/issues/13137 if this test starts failing intermittently
+      it('Fit data bounds should zoom to level 3', async function () {
         const expectedPrecision2ZoomCircles = [
           { color: '#750000', radius: 192 },
           { color: '#750000', radius: 191 },
@@ -187,12 +167,11 @@ export default function ({ getService, getPageObjects }) {
           { color: '#b99939', radius: 9 }
         ];
 
-        return PageObjects.visualize.clickMapFitDataBounds()
-        .then(function () {
-          return PageObjects.visualize.getTileMapData();
-        })
-        .then(function (data) {
+        await retry.try(async() => {
+          await PageObjects.visualize.clickMapFitDataBounds();
+          const data = await PageObjects.visualize.getTileMapData();
           expect(data).to.eql(expectedPrecision2ZoomCircles);
+          screenshots.take('map-at-zoom-3');
         });
       });
 
@@ -238,7 +217,7 @@ export default function ({ getService, getPageObjects }) {
           expect(message).to.be('Visualization Editor: Saved Visualization \"' + vizName1 + '\"');
         })
         .then(function testVisualizeWaitForToastMessageGone() {
-          return PageObjects.visualize.waitForToastMessageGone();
+          return PageObjects.header.waitForToastMessageGone();
         })
         .then(function () {
           return PageObjects.visualize.openSpyPanel();