Fix template scale in find element by image

lib/basedriver/commands/find.js

@@ -4,14 +4,18 @@ import _ from 'lodash';
 import { errors } from '../../..';
 import { MATCH_TEMPLATE_MODE } from './images';
 import { W3C_ELEMENT_KEY, MJSONWP_ELEMENT_KEY } from '../../protocol/protocol';
-import { ImageElement } from '../image-element';
+import { ImageElement, DEFAULT_TEMPLATE_IMAGE_SCALE } from '../image-element';
 
 
 const commands = {}, helpers = {}, extensions = {};
 
 const IMAGE_STRATEGY = '-image';
 const CUSTOM_STRATEGY = '-custom';
 
+// Used to compar ratio and screen width
+// Pixel is basically under 1080 for example. 100K is probably enough fo a while.
+const FLOAT_PERCISION = 100000;
+
 // Override the following function for your own driver, and the rest is taken
 // care of!
 
@@ -183,6 +187,9 @@ commands.findByCustom = async function (selector, multiple) {
  * image is merely to check staleness. If so we can bypass a lot of logic
  * @property {boolean} [multiple=false] - Whether we are finding one element or
  * multiple
+ * @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we
+ * ignore defaultImageTemplateScale. It can be used when you would like to
+ * scale b64Template with defaultImageTemplateScale setting.
  */
 
 /**
@@ -198,11 +205,13 @@ commands.findByCustom = async function (selector, multiple) {
 helpers.findByImage = async function (b64Template, {
   shouldCheckStaleness = false,
   multiple = false,
+  ignoreDefaultImageTemplateScale = false,
 }) {
   const {
     imageMatchThreshold: threshold,
     fixImageTemplateSize,
-    fixImageTemplateScale
+    fixImageTemplateScale,
+    defaultImageTemplateScale
   } = this.settings.getSettings();
 
   log.info(`Finding image element with match threshold ${threshold}`);
@@ -225,9 +234,10 @@ helpers.findByImage = async function (b64Template, {
     try {
       const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);
 
-      if (fixImageTemplateScale) {
-        b64Template = await this.fixImageTemplateScale(b64Template, scale);
-      }
+      b64Template = await this.fixImageTemplateScale(b64Template, {
+        defaultImageTemplateScale, ignoreDefaultImageTemplateScale,
+        fixImageTemplateScale, ...scale
+      });
 
       rect = (await this.compareImages(MATCH_TEMPLATE_MODE, b64Screenshot,
         b64Template, {threshold})).rect;
@@ -357,42 +367,119 @@ helpers.getScreenshotForImageFind = async function (screenWidth, screenHeight) {
   // of coordinates returned by the image match algorithm, since we match based
   // on the screenshot coordinates not the device coordinates themselves. There
   // are two potential types of mismatch: aspect ratio mismatch and scale
-  // mismatch.
+  // mismatch. we need to detect and fix both
+
+  const scale = {xScale: 1.0, yScale: 1.0};
+
+  const screenAR = screenWidth / screenHeight;
+  const shotAR = shotWidth / shotHeight;
+  if (Math.round(screenAR * FLOAT_PERCISION) === Math.round(shotAR * FLOAT_PERCISION)) {
+    log.info('Screenshot aspect ratio matched screen aspect ratio');
+  } else {
+    log.warn(`When trying to find an element, determined that the screen ` +
+             `aspect ratio and screenshot aspect ratio are different. Screen ` +
+             `is ${screenWidth}x${screenHeight} whereas screenshot is ` +
+             `${shotWidth}x${shotHeight}.`);
+
+    // Select smaller ratio
+    // this.getScreenshot(shotWidth, shotHeight) is 540x397,
+    // this.getDeviceSize(screenWidth, screenHeight) is 1080x1920.
+    // The ratio is {xScale: 0.5, yScale: 0.2}.
+    // In this case, we must choose `yScale: 0.2` as scaleFactor.
+    // Because if Appium selects the both ratio, the screenshot will be distorted.
+    // If Appium selects the xScale, height will be bigger than real screenshot size
+    // which is used to image comparison by OpenCV as a base image.
+    const xScale = (1.0 * shotWidth) / screenWidth;
+    const yScale = (1.0 * shotHeight) / screenHeight;
+    const scaleFactor = xScale >= yScale ? yScale : xScale;
+
+    log.warn(`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` +
+             `screen aspect ratio so that image element coordinates have a ` +
+             `greater chance of being correct.`);
+    imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor);
+
+    scale.xScale *= scaleFactor;
+    scale.yScale *= scaleFactor;
+  }
 
   // Resize based on the screen dimensions only if both width and height are mismatched
   // since except for that, it might be a situation which is different window rect and
   // screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and
   // `"deviceScreenSize"=>"1080x1920"`
-  let scale;
   if (screenWidth !== shotWidth && screenHeight !== shotHeight) {
     log.info(`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +
              `screen at ${screenWidth}x${screenHeight}`);
     imgObj = imgObj.resize(screenWidth, screenHeight);
 
-    scale = {xScale: (1.0 * screenWidth) / shotWidth, yScale: (1.0 * screenHeight) / shotHeight};
+    scale.xScale *= (1.0 * screenWidth) / shotWidth;
+    scale.yScale *= (1.0 * screenHeight) / shotHeight;
   }
 
   b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');
   return {b64Screenshot, scale};
 };
 
+/**
+ * @typedef {Object} ImageTemplateSettings
+ * @property {boolean} fixImageTemplateScale - fixImageTemplateScale in device-settings
+ * @property {float} defaultImageTemplateScale - defaultImageTemplateScale in device-settings
+ * @property {boolean} ignoreDefaultImageTemplateScale - Ignore defaultImageTemplateScale if it has true.
+ * If b64Template has been scaled to defaultImageTemplateScale or should ignore the scale,
+ * this parameter should be true. e.g. click in image-element module
+ * @property {float} xScale - Scale ratio for width
+ * @property {float} yScale - Scale ratio for height
+
+ */
 /**
  * Get a image that will be used for template maching.
  * Returns scaled image if scale ratio is provided.
  *
+ *
  * @param {string} b64Template - base64-encoded image used as a template to be
  * matched in the screenshot
- * @param {ScreenshotScale} scale - scale of screen. Default to `{}`
+ * @param {ImageTemplateSettings} opts - Image template scale related options
  *
  * @returns {string} base64-encoded scaled template screenshot
  */
-helpers.fixImageTemplateScale = async function (b64Template, scale = {}) {
-  if (!scale) {
+const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;
+helpers.fixImageTemplateScale = async function (b64Template, opts = {}) {
+  if (!opts) {
+    return b64Template;
+  }
+
+  let {
+    fixImageTemplateScale = false,
+    defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,
+    ignoreDefaultImageTemplateScale = false,
+    xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
+    yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE
+  } = opts;
+
+  if (ignoreDefaultImageTemplateScale) {
+    defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;
+  }
+
+  // Default
+  if (defaultImageTemplateScale === DEFAULT_TEMPLATE_IMAGE_SCALE && !fixImageTemplateScale) {
+    return b64Template;
+  }
+
+  // Calculate xScale and yScale Appium should scale
+  if (fixImageTemplateScale) {
+    xScale *= defaultImageTemplateScale;
+    yScale *= defaultImageTemplateScale;
+  } else {
+    xScale = yScale = 1 * defaultImageTemplateScale;
+  }
+
+  // xScale and yScale can be NaN if defaultImageTemplateScale is string, for example
+  if (!parseFloat(xScale) || !parseFloat(yScale)) {
     return b64Template;
   }
 
-  const {xScale, yScale} = scale;
-  if (!xScale || !yScale) {
+  // Return if the scale is default, 1, value
+  if (Math.round(xScale * FLOAT_PERCISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PERCISION)
+      && Math.round(yScale * FLOAT_PERCISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PERCISION))) {
     return b64Template;
   }
 

lib/basedriver/device-settings.js

@@ -1,7 +1,7 @@
 import _ from 'lodash';
 import log from './logger';
 import { DEFAULT_MATCH_THRESHOLD } from './commands/images';
-import { IMAGE_EL_TAP_STRATEGY_W3C } from './image-element';
+import { IMAGE_EL_TAP_STRATEGY_W3C, DEFAULT_TEMPLATE_IMAGE_SCALE } from './image-element';
 
 const GLOBAL_DEFAULT_SETTINGS = {
   // value between 0 and 1 representing match strength, below which an image
@@ -25,6 +25,14 @@ const GLOBAL_DEFAULT_SETTINGS = {
   //      if a user use `width=750 × height=1334` pixels's base template image.
   fixImageTemplateScale: false,
 
+  // Users might have scaled template image to reduce their storage size.
+  // This setting allows users to scale a template image they send to Appium server
+  // so that Appium server compare actual scale users orginaly have.
+  // e.g. A user have `270 × 32 pixels` image oridinally `1080 × 126 pixels`
+  //      The user set {defaultImageTemplateScale: 4.0} to scale the small image
+  //      to the original one so that Appium can compare it as the original one.
+  defaultImageTemplateScale: DEFAULT_TEMPLATE_IMAGE_SCALE,
+
   // whether Appium should re-check that an image element can be matched
   // against the current screenshot before clicking it
   checkForImageElementStaleness: true,
@@ -44,6 +52,8 @@ const BASEDRIVER_HANDLED_SETTINGS = [
   'imageMatchThreshold',
   'fixImageFindScreenshotDims',
   'fixImageTemplateSize',
+  'fixImageTemplateScale',
+  'defaultImageTemplateScale',
   'checkForImageElementStaleness',
   'autoUpdateImageElementPosition',
   'imageElementTapStrategy',

lib/basedriver/image-element.js

@@ -13,6 +13,7 @@ const IMAGE_TAP_STRATEGIES = [
   IMAGE_EL_TAP_STRATEGY_MJSONWP,
   IMAGE_EL_TAP_STRATEGY_W3C
 ];
+const DEFAULT_TEMPLATE_IMAGE_SCALE = 1.0;
 
 /**
  * @typedef {Object} Rect
@@ -127,7 +128,10 @@ class ImageElement {
       log.info('Checking image element for staleness before clicking');
       try {
         newImgEl = await driver.findByImage(this.template, {
-          shouldCheckStaleness: true
+          shouldCheckStaleness: true,
+          // Set ignoreDefaultImageTemplateScale because this.template is device screenshot based image
+          // managed inside Appium after finidng image by template which managed by a user
+          ignoreDefaultImageTemplateScale: true
         });
       } catch (err) {
         throw new errors.StaleElementReferenceError();
@@ -246,4 +250,5 @@ function getImgElFromArgs (args) {
 export {
   ImageElement, getImgElFromArgs, makeImageElementCache,
   IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C,
+  DEFAULT_TEMPLATE_IMAGE_SCALE
 };

test/basedriver/commands/find-specs.js

@@ -113,25 +113,6 @@ describe('finding elements by image', function () {
       sinon.stub(d, 'fixImageTemplateScale').returns(newTemplate);
       d.fixImageTemplateScale.callCount.should.eql(0);
     });
-    it('should not fix template size scale if no scale value', async function () {
-      const newTemplate = 'iVBORbaz';
-      await helpers.fixImageTemplateScale(newTemplate).should.eventually.eql(newTemplate);
-    });
-    it('should not fix template size scale if it is null', async function () {
-      const newTemplate = 'iVBORbaz';
-      await helpers.fixImageTemplateScale(newTemplate, null)
-        .should.eventually.eql(newTemplate);
-    });
-    it('should not fix template size scale if it is not number', async function () {
-      const newTemplate = 'iVBORbaz';
-      await helpers.fixImageTemplateScale(newTemplate, 'wrong-scale')
-        .should.eventually.eql(newTemplate);
-    });
-    it('should fix template size scale', async function () {
-      const actual = 'iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAYAAADgzO9IAAAAWElEQVR4AU3BQRWAQAhAwa/PGBsEgrC16AFBKEIPXW7OXO+Rmey9iQjMjHFzrLUwM7qbqmLcHKpKRFBVuDvj4agq3B1VRUQYT2bS3QwRQVUZF/CaGRHB3wc1vSZbHO5+BgAAAABJRU5ErkJggg==';
-      await helpers.fixImageTemplateScale(TINY_PNG, { xScale: 1.5, yScale: 1.5 })
-        .should.eventually.eql(actual);
-    });
 
     it('should throw an error if template match fails', async function () {
       const d = new TestDriver();
@@ -165,6 +146,81 @@ describe('finding elements by image', function () {
     });
   });
 
+  describe('fixImageTemplateScale', function () {
+    it('should not fix template size scale if no scale value', async function () {
+      const newTemplate = 'iVBORbaz';
+      await helpers.fixImageTemplateScale(newTemplate, {fixImageTemplateScale: true})
+        .should.eventually.eql(newTemplate);
+    });
+
+    it('should not fix template size scale if it is null', async function () {
+      const newTemplate = 'iVBORbaz';
+      await helpers.fixImageTemplateScale(newTemplate, null)
+        .should.eventually.eql(newTemplate);
+    });
+
+    it('should not fix template size scale if it is not number', async function () {
+      const newTemplate = 'iVBORbaz';
+      await helpers.fixImageTemplateScale(newTemplate, 'wrong-scale')
+        .should.eventually.eql(newTemplate);
+    });
+
+    it('should fix template size scale', async function () {
+      const actual = 'iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAYAAADgzO9IAAAAWElEQVR4AU3BQRWAQAhAwa/PGBsEgrC16AFBKEIPXW7OXO+Rmey9iQjMjHFzrLUwM7qbqmLcHKpKRFBVuDvj4agq3B1VRUQYT2bS3QwRQVUZF/CaGRHB3wc1vSZbHO5+BgAAAABJRU5ErkJggg==';
+      await helpers.fixImageTemplateScale(TINY_PNG, {
+        fixImageTemplateScale: true, xScale: 1.5, yScale: 1.5
+      }).should.eventually.eql(actual);
+    });
+
+    it('should not fix template size scale because of fixImageTemplateScale is false', async function () {
+      await helpers.fixImageTemplateScale(TINY_PNG, {
+        fixImageTemplateScale: false, xScale: 1.5, yScale: 1.5
+      }).should.eventually.eql(TINY_PNG);
+    });
+
+    it('should fix template size scale with default scale', async function () {
+      const actual = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABwUlEQVR4AaXBPUsrQQCG0SeX+cBdkTjwTpG1NPgLpjY/fW1stt4UYmm2cJqwMCsaw70uJJ3CBc9Z/P3Cl+12S9u2tG1L27bEGLm/v2ez2bDZbJDEd/7wS4YT7z3X19fc3Nxwd3dHXdd47xnHkefnZ8ZxpKoq6rqmqiqMMcwMJ1VV0TQN0zThnOPj44O6rsk503UdkmiahqZpWK1WGGOYGU7quqZpGqy1SCLnTM6Z19dXcs5IYpomrLVI4uLigpnhpKoqVqsVkjgcDjw9PdF1HTlnuq5DEs45JHE4HDgznByPR97e3pimiVIK4zhyPB7x3hNCIITA5eUl3nsWiwVnhpNSCsMwsNvtGIaB/X5PKQVJpJSQxHq9RhLOOc4MJ9M0sdvt2G639H3PTBIxRiQhCUnEGLHWcmY4KaUwDAN93/P4+MhyuSSlhCRSSkjCOYe1FmstZ6bve2YvLy/s93tmy+USSUhCEpIIIfAd8/DwwOz9/Z1SCpJIKSGJ9XqNJJxz/MS0bcvs6uoKScQYkYQkJBFjxFrLT0zbtsxub29JKSGJlBKScM5hrcVay09MzplZjJHPz0+894QQCCHwP/7wS/8A4e6nAg+R8LwAAAAASUVORK5CYII=';
+      await helpers.fixImageTemplateScale(TINY_PNG, {
+        defaultImageTemplateScale: 4.0
+      }).should.eventually.eql(actual);
+    });
+
+    it('should fix template size scale with default scale and image scale', async function () {
+      const actual = 'iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAACaUlEQVR4AbXBMWvrWBSF0c9BsFPtW91UR1U6+///FKlKKt8qqnyqnMozggkI8xgMj6x1uv+L/6zryrIsrOvKsiys68qyLFwuF87nM5fLhfP5zOVy4Xw+84wXftkLv2ziQBK26b0TEVQVu4jANrvM5Hq9spOEJCQhCUlI4mjiQBK26b1TVewkYRvb7DKTMQaZiW1s01rDNraRxNHEgSRaa1QVO0m01jjKTDKTXe+d3jtVxU4SjyYOJGGbnSRs03snM8lMMpPb7UZmkplEBFXFThK2eTRxIAnbSMI2VcX39zdjDMYYZCaZyRiDMQZVxU4StqkqHk0cSEISf5KZ7DKTMQbLsrCTRGuN3jtVxaOJg6qiqqgqqoqqoqoYY5CZ7GwTEdzvd97f34kIeu/YRhKPJg6qiswkM7ndbmQmmUlmkpnsbBMR2CYimOeZ3ju2kcSjiYOqIjP5+vpi2za2bWPbNo5aa7TW2PXe6b3Te6e1hiQeTRxUFbfbjW3bGGNwvV4ZY2Ab27TWsI1tbGMb27TWsI0kHk0cVBWZybZtXK9XPj8/+fj4YJ5nIoLWGraJCOZ5RhKSkIQkJPFo4qCqyEy2bWOMwefnJ+u6cjqdsM3ONvM8cz6feca0ris/rtcrmcnONhHB/X7n/f2diKD3jm0k8axpWRZ+ZCaZyc42EYFtIoJ5num9YxtJPGta15U/sY1tdm9vb/Te6b1jG0k8a1qWhR+2sU1rjdYatrGNbWxjm9YaknjWtK4rPyKCiKC1hm0igojg9fUVSUhCEpJ41rQsC0e22dkmIrhcLvyNF/7H6XTib73wy174Zf8AJEsePtlPj10AAAAASUVORK5CYII=';
+      await helpers.fixImageTemplateScale(TINY_PNG, {
+        defaultImageTemplateScale: 4.0,
+        fixImageTemplateScale: true,
+        xScale: 1.5, yScale: 1.5
+      }).should.eventually.eql(actual);
+    });
+
+    it('should not fix template size scale with default scale and image scale', async function () {
+      const actual = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABwUlEQVR4AaXBPUsrQQCG0SeX+cBdkTjwTpG1NPgLpjY/fW1stt4UYmm2cJqwMCsaw70uJJ3CBc9Z/P3Cl+12S9u2tG1L27bEGLm/v2ez2bDZbJDEd/7wS4YT7z3X19fc3Nxwd3dHXdd47xnHkefnZ8ZxpKoq6rqmqiqMMcwMJ1VV0TQN0zThnOPj44O6rsk503UdkmiahqZpWK1WGGOYGU7quqZpGqy1SCLnTM6Z19dXcs5IYpomrLVI4uLigpnhpKoqVqsVkjgcDjw9PdF1HTlnuq5DEs45JHE4HDgznByPR97e3pimiVIK4zhyPB7x3hNCIITA5eUl3nsWiwVnhpNSCsMwsNvtGIaB/X5PKQVJpJSQxHq9RhLOOc4MJ9M0sdvt2G639H3PTBIxRiQhCUnEGLHWcmY4KaUwDAN93/P4+MhyuSSlhCRSSkjCOYe1FmstZ6bve2YvLy/s93tmy+USSUhCEpIIIfAd8/DwwOz9/Z1SCpJIKSGJ9XqNJJxz/MS0bcvs6uoKScQYkYQkJBFjxFrLT0zbtsxub29JKSGJlBKScM5hrcVay09MzplZjJHPz0+894QQCCHwP/7wS/8A4e6nAg+R8LwAAAAASUVORK5CYII=';
+      await helpers.fixImageTemplateScale(TINY_PNG, {
+        defaultImageTemplateScale: 4.0,
+        fixImageTemplateScale: false,
+        xScale: 1.5, yScale: 1.5
+      }).should.eventually.eql(actual);
+    });
+
+    it('should not fix template size scale because of ignoreDefaultImageTemplateScale', async function () {
+      await helpers.fixImageTemplateScale(TINY_PNG, {
+        defaultImageTemplateScale: 4.0,
+        ignoreDefaultImageTemplateScale: true,
+      }).should.eventually.eql(TINY_PNG);
+    });
+
+    it('should ignore defaultImageTemplateScale to fix template size scale because of ignoreDefaultImageTemplateScale', async function () {
+      const actual = 'iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAYAAADgzO9IAAAAWElEQVR4AU3BQRWAQAhAwa/PGBsEgrC16AFBKEIPXW7OXO+Rmey9iQjMjHFzrLUwM7qbqmLcHKpKRFBVuDvj4agq3B1VRUQYT2bS3QwRQVUZF/CaGRHB3wc1vSZbHO5+BgAAAABJRU5ErkJggg==';
+      await helpers.fixImageTemplateScale(TINY_PNG, {
+        defaultImageTemplateScale: 4.0,
+        ignoreDefaultImageTemplateScale: true,
+        fixImageTemplateScale: true,
+        xScale: 1.5, yScale: 1.5
+      }).should.eventually.eql(actual);
+    });
+  });
+
   describe('ensureTemplateSize', function () {
     it('should not resize the template if it is smaller than the screen', async function () {
       const screen = TINY_PNG_DIMS.map(n => n * 2);
@@ -225,21 +281,27 @@ describe('finding elements by image', function () {
 
       // try first with portrait screen
       let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];
+      let expectedScale = { xScale: 0.67, yScale: 1 };
+
       const {b64Screenshot, scale} = await d.getScreenshotForImageFind(...screen);
       b64Screenshot.should.not.eql(TINY_PNG);
       let screenshotObj = await imageUtil.getJimpImage(b64Screenshot);
       screenshotObj.bitmap.width.should.eql(screen[0]);
       screenshotObj.bitmap.height.should.eql(screen[1]);
-      scale.should.eql({ xScale: 2, yScale: 3 });
+      scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());
+      scale.yScale.should.eql(expectedScale.yScale);
 
       // then with landscape screen
       screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];
+      expectedScale = { xScale: 1, yScale: 0.67 };
+
       const {b64Screenshot: newScreen, scale: newScale} = await d.getScreenshotForImageFind(...screen);
       newScreen.should.not.eql(TINY_PNG);
       screenshotObj = await imageUtil.getJimpImage(newScreen);
       screenshotObj.bitmap.width.should.eql(screen[0]);
       screenshotObj.bitmap.height.should.eql(screen[1]);
-      newScale.should.eql({ xScale: 3, yScale: 2 });
+      newScale.xScale.should.eql(expectedScale.xScale);
+      newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());
     });
   });
 });