How to use d.ensureTemplateSize method in Appium Base Driver

1import log from '../logger';2import { logger, imageUtil } from 'appium-support';3import _ from 'lodash';4import { errors } from '../../..';5import { MATCH_TEMPLATE_MODE } from './images';6import { ImageElement, DEFAULT_TEMPLATE_IMAGE_SCALE } from '../image-element';7const commands = {}, helpers = {}, extensions = {};8const IMAGE_STRATEGY = '-image';9const CUSTOM_STRATEGY = '-custom';10// Used to compare ratio and screen width11// Pixel is basically under 1080 for example. 100K is probably enough fo a while.12const FLOAT_PRECISION = 100000;13// Override the following function for your own driver, and the rest is taken14// care of!15// helpers.findElOrEls = async function (strategy, selector, mult, context) {}16//   strategy: locator strategy17//   selector: the actual selector for finding an element18//   mult: multiple elements or just one?19//   context: finding an element from the root context? or starting from another element20//21// Returns an object which adheres to the way the JSON Wire Protocol represents elements:22// { ELEMENT: # }    eg: { ELEMENT: 3 }  or { ELEMENT: 1.023 }23helpers.findElOrElsWithProcessing = async function findElOrElsWithProcessing (strategy, selector, mult, context) {24  this.validateLocatorStrategy(strategy);25  try {26    return await this.findElOrEls(strategy, selector, mult, context);27  } catch (err) {28    if (this.opts.printPageSourceOnFindFailure) {29      const src = await this.getPageSource();30      log.debug(`Error finding element${mult ? 's' : ''}: ${err.message}`);31      log.debug(`Page source requested through 'printPageSourceOnFindFailure':`);32      log.debug(src);33    }34    // still want the error to occur35    throw err;36  }37};38commands.findElement = async function findElement (strategy, selector) {39  if (strategy === IMAGE_STRATEGY) {40    return await this.findByImage(selector, {multiple: false});41  } else if (strategy === CUSTOM_STRATEGY) {42    return await this.findByCustom(selector, false);43  }44  return await this.findElOrElsWithProcessing(strategy, selector, false);45};46commands.findElements = async function findElements (strategy, selector) {47  if (strategy === IMAGE_STRATEGY) {48    return await this.findByImage(selector, {multiple: true});49  } else if (strategy === CUSTOM_STRATEGY) {50    return await this.findByCustom(selector, true);51  }52  return await this.findElOrElsWithProcessing(strategy, selector, true);53};54commands.findElementFromElement = async function findElementFromElement (strategy, selector, elementId) {55  return await this.findElOrElsWithProcessing(strategy, selector, false, elementId);56};57commands.findElementsFromElement = async function findElementsFromElement (strategy, selector, elementId) {58  return await this.findElOrElsWithProcessing(strategy, selector, true, elementId);59};60/**61 * Find an element using a custom plugin specified by the customFindModules cap.62 *63 * @param {string} selector - the selector which the plugin will use to find64 * elements65 * @param {boolean} multiple - whether we want one element or multiple66 *67 * @returns {WebElement} - WebDriver element or list of elements68 */69commands.findByCustom = async function findByCustom (selector, multiple) {70  const plugins = this.opts.customFindModules;71  // first ensure the user has registered one or more find plugins72  if (!plugins) {73    // TODO this info should go in docs instead; update when docs for this74    // feature exist75    throw new Error('Finding an element using a plugin is currently an ' +76      'incubating feature. To use it you must manually install one or more ' +77      'plugin modules in a way that they can be required by Appium, for ' +78      'example installing them from the Appium directory, installing them ' +79      'globally, or installing them elsewhere and passing an absolute path as ' +80      'the capability. Then construct an object where the key is the shortcut ' +81      'name for this plugin and the value is the module name or absolute path, ' +82      'for example: {"p1": "my-find-plugin"}, and pass this in as the ' +83      "'customFindModules' capability.");84  }85  // then do some basic checking of the type of the capability86  if (!_.isPlainObject(plugins)) {87    throw new Error("Invalid format for the 'customFindModules' capability. " +88      'It should be an object with keys corresponding to the short names and ' +89      'values corresponding to the full names of the element finding plugins');90  }91  // get the name of the particular plugin used for this invocation of find,92  // and separate it from the selector we will pass to the plugin93  let [plugin, realSelector] = selector.split(':');94  // if the user didn't specify a plugin for this find invocation, and we had95  // multiple plugins registered, that's a problem96  if (_.size(plugins) > 1 && !realSelector) {97    throw new Error(`Multiple element finding plugins were registered ` +98      `(${_.keys(plugins)}), but your selector did not indicate which plugin ` +99      `to use. Ensure you put the short name of the plugin followed by ':' as ` +100      `the initial part of the selector string.`);101  }102  // but if they did not specify a plugin and we only have one plugin, just use103  // that one104  if (_.size(plugins) === 1 && !realSelector) {105    realSelector = plugin;106    plugin = _.keys(plugins)[0];107  }108  if (!plugins[plugin]) {109    throw new Error(`Selector specified use of element finding plugin ` +110      `'${plugin}' but it was not registered in the 'customFindModules' ` +111      `capability.`);112  }113  let finder;114  try {115    log.debug(`Find plugin '${plugin}' requested; will attempt to use it ` +116      `from '${plugins[plugin]}'`);117    finder = require(plugins[plugin]);118  } catch (err) {119    throw new Error(`Could not load your custom find module '${plugin}'. Did ` +120      `you put it somewhere Appium can 'require' it? Original error: ${err}`);121  }122  if (!finder || !_.isFunction(finder.find)) {123    throw new Error('Your custom find module did not appear to be constructed ' +124        'correctly. It needs to export an object with a `find` method.');125  }126  const customFinderLog = logger.getLogger(plugin);127  let elements;128  const condition = async () => {129    // get a list of matched elements from the custom finder, which can130    // potentially use the entire suite of methods the current driver provides.131    // the finder should always return a list of elements, but may use the132    // knowledge of whether we are looking for one or many to perform internal133    // optimizations134    elements = await finder.find(this, customFinderLog, realSelector, multiple);135    // if we're looking for multiple elements, or if we're looking for only136    // one and found it, we're done137    if (!_.isEmpty(elements) || multiple) {138      return true;139    }140    // otherwise we should retry, so return false to trigger the retry loop141    return false;142  };143  try {144    // make sure we respect implicit wait145    await this.implicitWaitForCondition(condition);146  } catch (err) {147    if (err.message.match(/Condition unmet/)) {148      throw new errors.NoSuchElementError();149    }150    throw err;151  }152  return multiple ? elements : elements[0];153};154/**155 * @typedef {Object} FindByImageOptions156 * @property {boolean} [shouldCheckStaleness=false] - whether this call to find an157 * image is merely to check staleness. If so we can bypass a lot of logic158 * @property {boolean} [multiple=false] - Whether we are finding one element or159 * multiple160 * @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we161 * ignore defaultImageTemplateScale. It can be used when you would like to162 * scale b64Template with defaultImageTemplateScale setting.163 */164/**165 * Find a screen rect represented by an ImageElement corresponding to an image166 * template sent in by the client167 *168 * @param {string} b64Template - base64-encoded image used as a template to be169 * matched in the screenshot170 * @param {FindByImageOptions} - additional options171 *172 * @returns {WebElement} - WebDriver element with a special id prefix173 */174helpers.findByImage = async function findByImage (b64Template, {175  shouldCheckStaleness = false,176  multiple = false,177  ignoreDefaultImageTemplateScale = false,178}) {179  const {180    imageMatchThreshold: threshold,181    fixImageTemplateSize,182    fixImageTemplateScale,183    defaultImageTemplateScale,184    getMatchedImageResult: visualize185  } = this.settings.getSettings();186  log.info(`Finding image element with match threshold ${threshold}`);187  if (!this.getWindowSize) {188    throw new Error("This driver does not support the required 'getWindowSize' command");189  }190  const {width: screenWidth, height: screenHeight} = await this.getWindowSize();191  // someone might have sent in a template that's larger than the screen192  // dimensions. If so let's check and cut it down to size since the algorithm193  // will not work unless we do. But because it requires some potentially194  // expensive commands, only do this if the user has requested it in settings.195  if (fixImageTemplateSize) {196    b64Template = await this.ensureTemplateSize(b64Template, screenWidth,197      screenHeight);198  }199  let rect = null;200  let b64Matched = null;201  const condition = async () => {202    try {203      const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);204      b64Template = await this.fixImageTemplateScale(b64Template, {205        defaultImageTemplateScale, ignoreDefaultImageTemplateScale,206        fixImageTemplateScale, ...scale207      });208      const comparedImage = await this.compareImages(MATCH_TEMPLATE_MODE, b64Screenshot, b64Template, {threshold, visualize});209      rect = comparedImage.rect;210      b64Matched = comparedImage.visualization;211      return true;212    } catch (err) {213      // if compareImages fails, we'll get a specific error, but we should214      // retry, so trap that and just return false to trigger the next round of215      // implicitly waiting. For other errors, throw them to get out of the216      // implicit wait loop217      if (err.message.match(/Cannot find any occurrences/)) {218        return false;219      }220      throw err;221    }222  };223  try {224    await this.implicitWaitForCondition(condition);225  } catch (err) {226    // this `implicitWaitForCondition` method will throw a 'Condition unmet'227    // error if an element is not found eventually. In that case, we will228    // handle the element not found response below. In the case where get some229    // _other_ kind of error, it means something blew up totally apart from the230    // implicit wait timeout. We should not mask that error and instead throw231    // it straightaway232    if (!err.message.match(/Condition unmet/)) {233      throw err;234    }235  }236  if (!rect) {237    if (multiple) {238      return [];239    }240    throw new errors.NoSuchElementError();241  }242  log.info(`Image template matched: ${JSON.stringify(rect)}`);243  if (b64Matched) {244    log.info(`Matched base64 data: ${b64Matched.substring(0, 200)}...`);245  }246  const imgEl = new ImageElement(b64Template, rect, b64Matched);247  // if we're just checking staleness, return straightaway so we don't add248  // a new element to the cache. shouldCheckStaleness does not support multiple249  // elements, since it is a purely internal mechanism250  if (shouldCheckStaleness) {251    return imgEl;252  }253  const protocolEl = this.registerImageElement(imgEl);254  return multiple ? [protocolEl] : protocolEl;255};256/**257 * Ensure that the image template sent in for a find is of a suitable size258 *259 * @param {string} b64Template - base64-encoded image260 * @param {int} screenWidth - width of screen261 * @param {int} screenHeight - height of screen262 *263 * @returns {string} base64-encoded image, potentially resized264 */265helpers.ensureTemplateSize = async function ensureTemplateSize (b64Template, screenWidth, screenHeight) {266  let imgObj = await imageUtil.getJimpImage(b64Template);267  let {width: tplWidth, height: tplHeight} = imgObj.bitmap;268  log.info(`Template image is ${tplWidth}x${tplHeight}. Screen size is ${screenWidth}x${screenHeight}`);269  // if the template fits inside the screen dimensions, we're good270  if (tplWidth <= screenWidth && tplHeight <= screenHeight) {271    return b64Template;272  }273  log.info(`Scaling template image from ${tplWidth}x${tplHeight} to match ` +274           `screen at ${screenWidth}x${screenHeight}`);275  // otherwise, scale it to fit inside the screen dimensions276  imgObj = imgObj.scaleToFit(screenWidth, screenHeight);277  return (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');278};279/**280 * @typedef {Object} Screenshot281 * @property {string} b64Screenshot - base64 based screenshot string282 */283/**284 * @typedef {Object} ScreenshotScale285 * @property {float} xScale - Scale ratio for width286 * @property {float} yScale - Scale ratio for height287 */288/**289 * Get the screenshot image that will be used for find by element, potentially290 * altering it in various ways based on user-requested settings291 *292 * @param {int} screenWidth - width of screen293 * @param {int} screenHeight - height of screen294 *295 * @returns {Screenshot, ?ScreenshotScale} base64-encoded screenshot and ScreenshotScale296 */297helpers.getScreenshotForImageFind = async function getScreenshotForImageFind (screenWidth, screenHeight) {298  if (!this.getScreenshot) {299    throw new Error("This driver does not support the required 'getScreenshot' command");300  }301  let b64Screenshot = await this.getScreenshot();302  // if the user has requested not to correct for aspect or size differences303  // between the screenshot and the screen, just return the screenshot now304  if (!this.settings.getSettings().fixImageFindScreenshotDims) {305    log.info(`Not verifying screenshot dimensions match screen`);306    return {b64Screenshot};307  }308  // otherwise, do some verification on the screenshot to make sure it matches309  // the screen size and aspect ratio310  log.info('Verifying screenshot size and aspect ratio');311  let imgObj = await imageUtil.getJimpImage(b64Screenshot);312  let {width: shotWidth, height: shotHeight} = imgObj.bitmap;313  if (screenWidth === shotWidth && screenHeight === shotHeight) {314    // the height and width of the screenshot and the device screen match, which315    // means we should be safe when doing template matches316    log.info('Screenshot size matched screen size');317    return {b64Screenshot};318  }319  // otherwise, if they don't match, it could spell problems for the accuracy320  // of coordinates returned by the image match algorithm, since we match based321  // on the screenshot coordinates not the device coordinates themselves. There322  // are two potential types of mismatch: aspect ratio mismatch and scale323  // mismatch. We need to detect and fix both324  const scale = {xScale: 1.0, yScale: 1.0};325  const screenAR = screenWidth / screenHeight;326  const shotAR = shotWidth / shotHeight;327  if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {328    log.info(`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +329      `screen aspect ratio '${screenAR}' (${screenWidth}x${screenHeight})`);330  } else {331    log.warn(`When trying to find an element, determined that the screen ` +332             `aspect ratio and screenshot aspect ratio are different. Screen ` +333             `is ${screenWidth}x${screenHeight} whereas screenshot is ` +334             `${shotWidth}x${shotHeight}.`);335    // In the case where the x-scale and y-scale are different, we need to decide336    // which one to respect, otherwise the screenshot and template will end up337    // being resized in a way that changes its aspect ratio (distorts it). For example, let's say:338    // this.getScreenshot(shotWidth, shotHeight) is 540x397,339    // this.getDeviceSize(screenWidth, screenHeight) is 1080x1920.340    // The ratio would then be {xScale: 0.5, yScale: 0.2}.341    // In this case, we must should `yScale: 0.2` as scaleFactor, because342    // if we select the xScale, the height will be bigger than real screenshot size343    // which is used to image comparison by OpenCV as a base image.344    // All of this is primarily useful when the screenshot is a horizontal slice taken out of the345    // screen (for example not including top/bottom nav bars)346    const xScale = (1.0 * shotWidth) / screenWidth;347    const yScale = (1.0 * shotHeight) / screenHeight;348    const scaleFactor = xScale >= yScale ? yScale : xScale;349    log.warn(`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` +350             `screen aspect ratio so that image element coordinates have a ` +351             `greater chance of being correct.`);352    imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor);353    scale.xScale *= scaleFactor;354    scale.yScale *= scaleFactor;355    shotWidth = imgObj.bitmap.width;356    shotHeight = imgObj.bitmap.height;357  }358  // Resize based on the screen dimensions only if both width and height are mismatched359  // since except for that, it might be a situation which is different window rect and360  // screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and361  // `"deviceScreenSize"=>"1080x1920"`362  if (screenWidth !== shotWidth && screenHeight !== shotHeight) {363    log.info(`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +364             `screen at ${screenWidth}x${screenHeight}`);365    imgObj = imgObj.resize(screenWidth, screenHeight);366    scale.xScale *= (1.0 * screenWidth) / shotWidth;367    scale.yScale *= (1.0 * screenHeight) / shotHeight;368  }369  b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');370  return {b64Screenshot, scale};371};372/**373 * @typedef {Object} ImageTemplateSettings374 * @property {boolean} fixImageTemplateScale - fixImageTemplateScale in device-settings375 * @property {float} defaultImageTemplateScale - defaultImageTemplateScale in device-settings376 * @property {boolean} ignoreDefaultImageTemplateScale - Ignore defaultImageTemplateScale if it has true.377 * If b64Template has been scaled to defaultImageTemplateScale or should ignore the scale,378 * this parameter should be true. e.g. click in image-element module379 * @property {float} xScale - Scale ratio for width380 * @property {float} yScale - Scale ratio for height381 */382/**383 * Get a image that will be used for template maching.384 * Returns scaled image if scale ratio is provided.385 *386 *387 * @param {string} b64Template - base64-encoded image used as a template to be388 * matched in the screenshot389 * @param {ImageTemplateSettings} opts - Image template scale related options390 *391 * @returns {string} base64-encoded scaled template screenshot392 */393const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;394helpers.fixImageTemplateScale = async function fixImageTemplateScale (b64Template, opts = {}) {395  if (!opts) {396    return b64Template;397  }398  let {399    fixImageTemplateScale = false,400    defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,401    ignoreDefaultImageTemplateScale = false,402    xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,403    yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE404  } = opts;405  if (ignoreDefaultImageTemplateScale) {406    defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;407  }408  // Default409  if (defaultImageTemplateScale === DEFAULT_TEMPLATE_IMAGE_SCALE && !fixImageTemplateScale) {410    return b64Template;411  }412  // Calculate xScale and yScale Appium should scale413  if (fixImageTemplateScale) {414    xScale *= defaultImageTemplateScale;415    yScale *= defaultImageTemplateScale;416  } else {417    xScale = yScale = 1 * defaultImageTemplateScale;418  }419  // xScale and yScale can be NaN if defaultImageTemplateScale is string, for example420  if (!parseFloat(xScale) || !parseFloat(yScale)) {421    return b64Template;422  }423  // Return if the scale is default, 1, value424  if (Math.round(xScale * FLOAT_PRECISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)425      && Math.round(yScale * FLOAT_PRECISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION))) {426    return b64Template;427  }428  let imgTempObj = await imageUtil.getJimpImage(b64Template);429  let {width: baseTempWidth, height: baseTempHeigh} = imgTempObj.bitmap;430  const scaledWidth = baseTempWidth * xScale;431  const scaledHeight = baseTempHeigh * yScale;432  log.info(`Scaling template image from ${baseTempWidth}x${baseTempHeigh}` +433            ` to ${scaledWidth}x${scaledHeight}`);434  log.info(`The ratio is ${xScale} and ${yScale}`);435  imgTempObj = await imgTempObj.resize(scaledWidth, scaledHeight);436  return (await imgTempObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');437};438Object.assign(extensions, commands, helpers);439export { commands, helpers, IMAGE_STRATEGY, CUSTOM_STRATEGY };...

1import _ from 'lodash';2import LRU from 'lru-cache';3import { imageUtil, util } from '@appium/support';4import { errors } from '@appium/base-driver';5import { ImageElement, DEFAULT_TEMPLATE_IMAGE_SCALE,6         IMAGE_EL_TAP_STRATEGY_W3C } from './image-element';7import { MATCH_TEMPLATE_MODE, compareImages, DEFAULT_MATCH_THRESHOLD } from './compare';8import log from './logger';9const MJSONWP_ELEMENT_KEY = 'ELEMENT';10const W3C_ELEMENT_KEY = util.W3C_WEB_ELEMENT_IDENTIFIER;11const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;12// Used to compare ratio and screen width13// Pixel is basically under 1080 for example. 100K is probably enough fo a while.14const FLOAT_PRECISION = 100000;15const MAX_CACHE_SIZE = 1024 * 1024 * 40; // 40mb16const DEFAULT_SETTINGS = {17  // value between 0 and 1 representing match strength, below which an image18  // element will not be found19  imageMatchThreshold: DEFAULT_MATCH_THRESHOLD,20  // One of possible image matching methods.21  // Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html22  // for more details.23  // TM_CCOEFF_NORMED by default24  imageMatchMethod: '',25  // if the image returned by getScreenshot differs in size or aspect ratio26  // from the screen, attempt to fix it automatically27  fixImageFindScreenshotDims: true,28  // whether Appium should ensure that an image template sent in during image29  // element find should have its size adjusted so the match algorithm will not30  // complain31  fixImageTemplateSize: false,32  // whether Appium should ensure that an image template sent in during image33  // element find should have its scale adjusted to display size so the match34  // algorithm will not complain.35  // e.g. iOS has `width=375, height=667` window rect, but its screenshot is36  //      `width=750 × height=1334` pixels. This setting help to adjust the scale37  //      if a user use `width=750 × height=1334` pixels's base template image.38  fixImageTemplateScale: false,39  // Users might have scaled template image to reduce their storage size.40  // This setting allows users to scale a template image they send to Appium server41  // so that the Appium server compares the actual scale users originally had.42  // e.g. If a user has an image of 270 x 32 pixels which was originally 1080 x 126 pixels,43  //      the user can set {defaultImageTemplateScale: 4.0} to scale the small image44  //      to the original one so that Appium can compare it as the original one.45  defaultImageTemplateScale: DEFAULT_TEMPLATE_IMAGE_SCALE,46  // whether Appium should re-check that an image element can be matched47  // against the current screenshot before clicking it48  checkForImageElementStaleness: true,49  // whether before clicking on an image element Appium should re-determine the50  // position of the element on screen51  autoUpdateImageElementPosition: false,52  // which method to use for tapping by coordinate for image elements. the53  // options are 'w3c' or 'mjsonwp'54  imageElementTapStrategy: IMAGE_EL_TAP_STRATEGY_W3C,55  // which method to use to save the matched image area in ImageElement class.56  // It is used for debugging purpose.57  getMatchedImageResult: false,58};59export default class ImageElementFinder {60  constructor (driver, max = MAX_CACHE_SIZE) {61    this.driver = driver;62    this.imgElCache = new LRU({63      max,64      length: (el) => el.template.length,65    });66  }67  setDriver (driver) {68    this.driver = driver;69  }70  registerImageElement (imgEl) {71    this.imgElCache.set(imgEl.id, imgEl);72    const protoKey = this.driver.isW3CProtocol() ? W3C_ELEMENT_KEY : MJSONWP_ELEMENT_KEY;73    return imgEl.asElement(protoKey);74  }75  /**76   * @typedef {Object} FindByImageOptions77   * @property {boolean} [shouldCheckStaleness=false] - whether this call to find an78   * image is merely to check staleness. If so we can bypass a lot of logic79   * @property {boolean} [multiple=false] - Whether we are finding one element or80   * multiple81   * @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we82   * ignore defaultImageTemplateScale. It can be used when you would like to83   * scale b64Template with defaultImageTemplateScale setting.84   */85  /**86   * Find a screen rect represented by an ImageElement corresponding to an image87   * template sent in by the client88   *89   * @param {string} b64Template - base64-encoded image used as a template to be90   * matched in the screenshot91   * @param {FindByImageOptions} - additional options92   *93   * @returns {WebElement} - WebDriver element with a special id prefix94   */95  async findByImage (b64Template, {96    shouldCheckStaleness = false,97    multiple = false,98    ignoreDefaultImageTemplateScale = false,99  }) {100    if (!this.driver) {101      throw new Error(`Can't find without a driver!`);102    }103    const settings = Object.assign({}, DEFAULT_SETTINGS, this.driver.settings.getSettings());104    const {105      imageMatchThreshold: threshold,106      imageMatchMethod,107      fixImageTemplateSize,108      fixImageTemplateScale,109      defaultImageTemplateScale,110      getMatchedImageResult: visualize111    } = settings;112    log.info(`Finding image element with match threshold ${threshold}`);113    if (!this.driver.getWindowSize) {114      throw new Error("This driver does not support the required 'getWindowSize' command");115    }116    const {width: screenWidth, height: screenHeight} = await this.driver.getWindowSize();117    // someone might have sent in a template that's larger than the screen118    // dimensions. If so let's check and cut it down to size since the algorithm119    // will not work unless we do. But because it requires some potentially120    // expensive commands, only do this if the user has requested it in settings.121    if (fixImageTemplateSize) {122      b64Template = await this.ensureTemplateSize(b64Template, screenWidth,123        screenHeight);124    }125    const results = [];126    const condition = async () => {127      try {128        const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);129        b64Template = await this.fixImageTemplateScale(b64Template, {130          defaultImageTemplateScale, ignoreDefaultImageTemplateScale,131          fixImageTemplateScale, ...scale132        });133        const comparisonOpts = {134          threshold,135          visualize,136          multiple,137        };138        if (imageMatchMethod) {139          comparisonOpts.method = imageMatchMethod;140        }141        if (multiple) {142          results.push(...(await compareImages(MATCH_TEMPLATE_MODE,143                                               b64Screenshot,144                                               b64Template,145                                               comparisonOpts)));146        } else {147          results.push(await compareImages(MATCH_TEMPLATE_MODE,148                                           b64Screenshot,149                                           b64Template,150                                           comparisonOpts));151        }152        return true;153      } catch (err) {154        // if compareImages fails, we'll get a specific error, but we should155        // retry, so trap that and just return false to trigger the next round of156        // implicitly waiting. For other errors, throw them to get out of the157        // implicit wait loop158        if (err.message.match(/Cannot find any occurrences/)) {159          return false;160        }161        throw err;162      }163    };164    try {165      await this.driver.implicitWaitForCondition(condition);166    } catch (err) {167      // this `implicitWaitForCondition` method will throw a 'Condition unmet'168      // error if an element is not found eventually. In that case, we will169      // handle the element not found response below. In the case where get some170      // _other_ kind of error, it means something blew up totally apart from the171      // implicit wait timeout. We should not mask that error and instead throw172      // it straightaway173      if (!err.message.match(/Condition unmet/)) {174        throw err;175      }176    }177    if (_.isEmpty(results)) {178      if (multiple) {179        return [];180      }181      throw new errors.NoSuchElementError();182    }183    const elements = results.map(({rect, score, visualization}) => {184      log.info(`Image template matched: ${JSON.stringify(rect)}`);185      return new ImageElement(b64Template, rect, score, visualization, this);186    });187    // if we're just checking staleness, return straightaway so we don't add188    // a new element to the cache. shouldCheckStaleness does not support multiple189    // elements, since it is a purely internal mechanism190    if (shouldCheckStaleness) {191      return elements[0];192    }193    const registeredElements = elements.map((imgEl) => this.registerImageElement(imgEl));194    return multiple ? registeredElements : registeredElements[0];195  }196  /**197   * Ensure that the image template sent in for a find is of a suitable size198   *199   * @param {string} b64Template - base64-encoded image200   * @param {int} screenWidth - width of screen201   * @param {int} screenHeight - height of screen202   *203   * @returns {string} base64-encoded image, potentially resized204   */205  async ensureTemplateSize (b64Template, screenWidth, screenHeight) {206    let imgObj = await imageUtil.getJimpImage(b64Template);207    let {width: tplWidth, height: tplHeight} = imgObj.bitmap;208    log.info(`Template image is ${tplWidth}x${tplHeight}. Screen size is ${screenWidth}x${screenHeight}`);209    // if the template fits inside the screen dimensions, we're good210    if (tplWidth <= screenWidth && tplHeight <= screenHeight) {211      return b64Template;212    }213    log.info(`Scaling template image from ${tplWidth}x${tplHeight} to match ` +214             `screen at ${screenWidth}x${screenHeight}`);215    // otherwise, scale it to fit inside the screen dimensions216    imgObj = imgObj.scaleToFit(screenWidth, screenHeight);217    return (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');218  }219  /**220   * @typedef {Object} Screenshot221   * @property {string} b64Screenshot - base64 based screenshot string222   */223  /**224   * @typedef {Object} ScreenshotScale225   * @property {float} xScale - Scale ratio for width226   * @property {float} yScale - Scale ratio for height227   */228  /**229   * Get the screenshot image that will be used for find by element, potentially230   * altering it in various ways based on user-requested settings231   *232   * @param {int} screenWidth - width of screen233   * @param {int} screenHeight - height of screen234   *235   * @returns {Screenshot, ?ScreenshotScale} base64-encoded screenshot and ScreenshotScale236   */237  async getScreenshotForImageFind (screenWidth, screenHeight) {238    if (!this.driver.getScreenshot) {239      throw new Error("This driver does not support the required 'getScreenshot' command");240    }241    const settings = Object.assign({}, DEFAULT_SETTINGS, this.driver.settings.getSettings());242    const {fixImageFindScreenshotDims} = settings;243    let b64Screenshot = await this.driver.getScreenshot();244    // if the user has requested not to correct for aspect or size differences245    // between the screenshot and the screen, just return the screenshot now246    if (!fixImageFindScreenshotDims) {247      log.info(`Not verifying screenshot dimensions match screen`);248      return {b64Screenshot};249    }250    if (screenWidth < 1 || screenHeight < 1) {251      log.warn(`The retrieved screen size ${screenWidth}x${screenHeight} does ` +252        `not seem to be valid. No changes will be applied to the screenshot`);253      return {b64Screenshot};254    }255    // otherwise, do some verification on the screenshot to make sure it matches256    // the screen size and aspect ratio257    log.info('Verifying screenshot size and aspect ratio');258    let imgObj = await imageUtil.getJimpImage(b64Screenshot);259    let {width: shotWidth, height: shotHeight} = imgObj.bitmap;260    if (shotWidth < 1 || shotHeight < 1) {261      log.warn(`The retrieved screenshot size ${shotWidth}x${shotHeight} does ` +262        `not seem to be valid. No changes will be applied to the screenshot`);263      return {b64Screenshot};264    }265    if (screenWidth === shotWidth && screenHeight === shotHeight) {266      // the height and width of the screenshot and the device screen match, which267      // means we should be safe when doing template matches268      log.info('Screenshot size matched screen size');269      return {b64Screenshot};270    }271    // otherwise, if they don't match, it could spell problems for the accuracy272    // of coordinates returned by the image match algorithm, since we match based273    // on the screenshot coordinates not the device coordinates themselves. There274    // are two potential types of mismatch: aspect ratio mismatch and scale275    // mismatch. We need to detect and fix both276    const scale = {xScale: 1.0, yScale: 1.0};277    const screenAR = screenWidth / screenHeight;278    const shotAR = shotWidth / shotHeight;279    if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {280      log.info(`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +281        `screen aspect ratio '${screenAR}' (${screenWidth}x${screenHeight})`);282    } else {283      log.warn(`When trying to find an element, determined that the screen ` +284               `aspect ratio and screenshot aspect ratio are different. Screen ` +285               `is ${screenWidth}x${screenHeight} whereas screenshot is ` +286               `${shotWidth}x${shotHeight}.`);287      // In the case where the x-scale and y-scale are different, we need to decide288      // which one to respect, otherwise the screenshot and template will end up289      // being resized in a way that changes its aspect ratio (distorts it). For example, let's say:290      // this.getScreenshot(shotWidth, shotHeight) is 540x397,291      // this.getDeviceSize(screenWidth, screenHeight) is 1080x1920.292      // The ratio would then be {xScale: 0.5, yScale: 0.2}.293      // In this case, we must should `yScale: 0.2` as scaleFactor, because294      // if we select the xScale, the height will be bigger than real screenshot size295      // which is used to image comparison by OpenCV as a base image.296      // All of this is primarily useful when the screenshot is a horizontal slice taken out of the297      // screen (for example not including top/bottom nav bars)298      const xScale = (1.0 * shotWidth) / screenWidth;299      const yScale = (1.0 * shotHeight) / screenHeight;300      const scaleFactor = xScale >= yScale ? yScale : xScale;301      log.warn(`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` +302               `screen aspect ratio so that image element coordinates have a ` +303               `greater chance of being correct.`);304      imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor);305      scale.xScale *= scaleFactor;306      scale.yScale *= scaleFactor;307      shotWidth = imgObj.bitmap.width;308      shotHeight = imgObj.bitmap.height;309    }310    // Resize based on the screen dimensions only if both width and height are mismatched311    // since except for that, it might be a situation which is different window rect and312    // screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and313    // `"deviceScreenSize"=>"1080x1920"`314    if (screenWidth !== shotWidth && screenHeight !== shotHeight) {315      log.info(`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +316               `screen at ${screenWidth}x${screenHeight}`);317      imgObj = imgObj.resize(screenWidth, screenHeight);318      scale.xScale *= (1.0 * screenWidth) / shotWidth;319      scale.yScale *= (1.0 * screenHeight) / shotHeight;320    }321    b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');322    return {b64Screenshot, scale};323  }324  /**325   * @typedef {Object} ImageTemplateSettings326   * @property {boolean} fixImageTemplateScale - fixImageTemplateScale in device-settings327   * @property {float} defaultImageTemplateScale - defaultImageTemplateScale in device-settings328   * @property {boolean} ignoreDefaultImageTemplateScale - Ignore defaultImageTemplateScale if it has true.329   * If b64Template has been scaled to defaultImageTemplateScale or should ignore the scale,330   * this parameter should be true. e.g. click in image-element module331   * @property {float} xScale - Scale ratio for width332   * @property {float} yScale - Scale ratio for height333   */334  /**335   * Get a image that will be used for template maching.336   * Returns scaled image if scale ratio is provided.337   *338   *339   * @param {string} b64Template - base64-encoded image used as a template to be340   * matched in the screenshot341   * @param {ImageTemplateSettings} opts - Image template scale related options342   *343   * @returns {string} base64-encoded scaled template screenshot344   */345  async fixImageTemplateScale (b64Template, opts = {}) {346    if (!opts) {347      return b64Template;348    }349    let {350      fixImageTemplateScale = false,351      defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,352      ignoreDefaultImageTemplateScale = false,353      xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,354      yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE355    } = opts;356    if (ignoreDefaultImageTemplateScale) {357      defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;358    }359    // Default360    if (defaultImageTemplateScale === DEFAULT_TEMPLATE_IMAGE_SCALE && !fixImageTemplateScale) {361      return b64Template;362    }363    // Calculate xScale and yScale Appium should scale364    if (fixImageTemplateScale) {365      xScale *= defaultImageTemplateScale;366      yScale *= defaultImageTemplateScale;367    } else {368      xScale = yScale = 1 * defaultImageTemplateScale;369    }370    // xScale and yScale can be NaN if defaultImageTemplateScale is string, for example371    if (!parseFloat(xScale) || !parseFloat(yScale)) {372      return b64Template;373    }374    // Return if the scale is default, 1, value375    if (Math.round(xScale * FLOAT_PRECISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)376        && Math.round(yScale * FLOAT_PRECISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION))) {377      return b64Template;378    }379    let imgTempObj = await imageUtil.getJimpImage(b64Template);380    let {width: baseTempWidth, height: baseTempHeigh} = imgTempObj.bitmap;381    const scaledWidth = baseTempWidth * xScale;382    const scaledHeight = baseTempHeigh * yScale;383    log.info(`Scaling template image from ${baseTempWidth}x${baseTempHeigh}` +384              ` to ${scaledWidth}x${scaledHeight}`);385    log.info(`The ratio is ${xScale} and ${yScale}`);386    imgTempObj = await imgTempObj.resize(scaledWidth, scaledHeight);387    return (await imgTempObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');388  }389}...

1import _ from 'lodash';2import { imageUtil } from '@appium/support';3import BaseDriver from '@appium/base-driver';4import ImageElementPlugin, { IMAGE_STRATEGY } from '../../index';5import ImageElementFinder from '../../lib/finder';6import ImageElement from '../../lib/image-element';7import sinon from 'sinon';8import { TINY_PNG, TINY_PNG_DIMS } from '../fixtures';9import chai from 'chai';10import chaiAsPromised from 'chai-as-promised';11const compareModule = require('../../lib/compare');12chai.use(chaiAsPromised);13const should = chai.should();14const plugin = new ImageElementPlugin();15class PluginDriver extends BaseDriver {16  async getWindowSize () {}17  async getScreenshot () {}18  findElement (strategy, selector) {19    return plugin.findElement(_.noop, this, strategy, selector);20  }21  findElements (strategy, selector) {22    return plugin.findElements(_.noop, this, strategy, selector);23  }24}25describe('finding elements by image', function () {26  describe('findElement', function () {27    it('should use a different special method to find element by image', async function () {28      const d = new PluginDriver();29      sinon.stub(plugin.finder, 'findByImage').returns(true);30      sinon.stub(d, 'findElOrElsWithProcessing').returns(false);31      await d.findElement(IMAGE_STRATEGY, 'foo').should.eventually.be.true;32      await d.findElements(IMAGE_STRATEGY, 'foo').should.eventually.be.true;33    });34    it('should not be able to find image element from any other element', async function () {35      const d = new PluginDriver();36      await d.findElementFromElement(IMAGE_STRATEGY, 'foo', 'elId')37        .should.eventually.be.rejectedWith(/Locator Strategy.+is not supported/);38      await d.findElementsFromElement(IMAGE_STRATEGY, 'foo', 'elId')39        .should.eventually.be.rejectedWith(/Locator Strategy.+is not supported/);40    });41  });42  describe('findByImage', function () {43    const rect = {x: 10, y: 20, width: 30, height: 40};44    const score = 0.9;45    const size = {width: 100, height: 200};46    const screenshot = 'iVBORfoo';47    const template = 'iVBORbar';48    let compareStub;49    let d = new PluginDriver();50    let f = new ImageElementFinder(d);51    function basicStub (driver, finder) {52      const sizeStub = sinon.stub(driver, 'getWindowSize').returns(size);53      const screenStub = sinon.stub(finder, 'getScreenshotForImageFind').returns(screenshot);54      return {sizeStub, screenStub};55    }56    function basicImgElVerify (imgElProto, finder) {57      const imgElId = imgElProto.ELEMENT;58      finder.imgElCache.has(imgElId).should.be.true;59      const imgEl = finder.imgElCache.get(imgElId);60      (imgEl instanceof ImageElement).should.be.true;61      imgEl.rect.should.eql(rect);62      imgEl.score.should.eql(score);63      return imgEl;64    }65    beforeEach(function () {66      compareStub = sinon.stub(compareModule, 'compareImages').returns({rect, score});67      d = new PluginDriver();68      f = new ImageElementFinder(d);69      basicStub(d, f);70    });71    afterEach(function () {72      compareStub.restore();73    });74    it('should find an image element happypath', async function () {75      const imgElProto = await f.findByImage(template, {multiple: false});76      basicImgElVerify(imgElProto, f);77    });78    it('should find image elements happypath', async function () {79      compareStub.restore();80      compareStub = sinon.stub(compareModule, 'compareImages').returns([{rect, score}]);81      const els = await f.findByImage(template, {multiple: true});82      els.should.have.length(1);83      basicImgElVerify(els[0], f);84    });85    it('should fail if driver does not support getWindowSize', async function () {86      d.getWindowSize = null;87      await f.findByImage(template, {multiple: false})88        .should.eventually.be.rejectedWith(/driver does not support/);89    });90    it('should fix template size if requested', async function () {91      const newTemplate = 'iVBORbaz';92      await d.settings.update({fixImageTemplateSize: true});93      sinon.stub(f, 'ensureTemplateSize').returns(newTemplate);94      const imgElProto = await f.findByImage(template, {multiple: false});95      const imgEl = basicImgElVerify(imgElProto, f);96      imgEl.template.should.eql(newTemplate);97      _.last(compareStub.args)[2].should.eql(newTemplate);98    });99    it('should fix template size scale if requested', async function () {100      const newTemplate = 'iVBORbaz';101      await d.settings.update({fixImageTemplateScale: true});102      sinon.stub(f, 'fixImageTemplateScale').returns(newTemplate);103      const imgElProto = await f.findByImage(template, {multiple: false});104      const imgEl = basicImgElVerify(imgElProto, f);105      imgEl.template.should.eql(newTemplate);106      _.last(compareStub.args)[2].should.eql(newTemplate);107    });108    it('should not fix template size scale if it is not requested', async function () {109      const newTemplate = 'iVBORbaz';110      await d.settings.update({});111      sinon.stub(f, 'fixImageTemplateScale').returns(newTemplate);112      f.fixImageTemplateScale.callCount.should.eql(0);113    });114    it('should throw an error if template match fails', async function () {115      compareStub.throws(new Error('Cannot find any occurrences'));116      await f.findByImage(template, {multiple: false})117        .should.eventually.be.rejectedWith(/element could not be located/);118    });119    it('should return empty array for multiple elements if template match fails', async function () {120      compareStub.throws(new Error('Cannot find any occurrences'));121      await f.findByImage(template, {multiple: true}).should.eventually.eql([]);122    });123    it('should respect implicit wait', async function () {124      d.setImplicitWait(10);125      compareStub.resetHistory();126      compareStub.onCall(0).throws(new Error('Cannot find any occurrences'));127      compareStub.returns({rect, score});128      const imgElProto = await f.findByImage(template, {multiple: false});129      basicImgElVerify(imgElProto, f);130      compareStub.callCount.should.eql(2);131    });132    it('should not add element to cache and return it directly when checking staleness', async function () {133      const imgEl = await f.findByImage(template, {multiple: false, shouldCheckStaleness: true});134      (imgEl instanceof ImageElement).should.be.true;135      f.imgElCache.has(imgEl.id).should.be.false;136      imgEl.rect.should.eql(rect);137    });138  });139  describe('fixImageTemplateScale', function () {140    const d = new PluginDriver();141    const f = new ImageElementFinder(d);142    const basicTemplate = 'iVBORbaz';143    it('should not fix template size scale if no scale value', async function () {144      await f.fixImageTemplateScale(basicTemplate, {fixImageTemplateScale: true})145        .should.eventually.eql(basicTemplate);146    });147    it('should not fix template size scale if it is null', async function () {148      await f.fixImageTemplateScale(basicTemplate, null)149        .should.eventually.eql(basicTemplate);150    });151    it('should not fix template size scale if it is not number', async function () {152      await f.fixImageTemplateScale(basicTemplate, 'wrong-scale')153        .should.eventually.eql(basicTemplate);154    });155    it('should fix template size scale', async function () {156      const actual = 'iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAYAAADgzO9IAAAAWElEQVR4AU3BQRWAQAhAwa/PGBsEgrC16AFBKEIPXW7OXO+Rmey9iQjMjHFzrLUwM7qbqmLcHKpKRFBVuDvj4agq3B1VRUQYT2bS3QwRQVUZF/CaGRHB3wc1vSZbHO5+BgAAAABJRU5ErkJggg==';157      await f.fixImageTemplateScale(TINY_PNG, {158        fixImageTemplateScale: true, xScale: 1.5, yScale: 1.5159      }).should.eventually.eql(actual);160    });161    it('should not fix template size scale because of fixImageTemplateScale being false', async function () {162      await f.fixImageTemplateScale(TINY_PNG, {163        fixImageTemplateScale: false, xScale: 1.5, yScale: 1.5164      }).should.eventually.eql(TINY_PNG);165    });166    it('should fix template size scale with default scale', async function () {167      const actual = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABwUlEQVR4AaXBPUsrQQCG0SeX+cBdkTjwTpG1NPgLpjY/fW1stt4UYmm2cJqwMCsaw70uJJ3CBc9Z/P3Cl+12S9u2tG1L27bEGLm/v2ez2bDZbJDEd/7wS4YT7z3X19fc3Nxwd3dHXdd47xnHkefnZ8ZxpKoq6rqmqiqMMcwMJ1VV0TQN0zThnOPj44O6rsk503UdkmiahqZpWK1WGGOYGU7quqZpGqy1SCLnTM6Z19dXcs5IYpomrLVI4uLigpnhpKoqVqsVkjgcDjw9PdF1HTlnuq5DEs45JHE4HDgznByPR97e3pimiVIK4zhyPB7x3hNCIITA5eUl3nsWiwVnhpNSCsMwsNvtGIaB/X5PKQVJpJSQxHq9RhLOOc4MJ9M0sdvt2G639H3PTBIxRiQhCUnEGLHWcmY4KaUwDAN93/P4+MhyuSSlhCRSSkjCOYe1FmstZ6bve2YvLy/s93tmy+USSUhCEpIIIfAd8/DwwOz9/Z1SCpJIKSGJ9XqNJJxz/MS0bcvs6uoKScQYkYQkJBFjxFrLT0zbtsxub29JKSGJlBKScM5hrcVay09MzplZjJHPz0+894QQCCHwP/7wS/8A4e6nAg+R8LwAAAAASUVORK5CYII=';168      await f.fixImageTemplateScale(TINY_PNG, {169        defaultImageTemplateScale: 4.0170      }).should.eventually.eql(actual);171    });172    it('should fix template size scale with default scale and image scale', async function () {173      const actual = 'iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAACaUlEQVR4AbXBMWvrWBSF0c9BsFPtW91UR1U6+///FKlKKt8qqnyqnMozggkI8xgMj6x1uv+L/6zryrIsrOvKsiys68qyLFwuF87nM5fLhfP5zOVy4Xw+84wXftkLv2ziQBK26b0TEVQVu4jANrvM5Hq9spOEJCQhCUlI4mjiQBK26b1TVewkYRvb7DKTMQaZiW1s01rDNraRxNHEgSRaa1QVO0m01jjKTDKTXe+d3jtVxU4SjyYOJGGbnSRs03snM8lMMpPb7UZmkplEBFXFThK2eTRxIAnbSMI2VcX39zdjDMYYZCaZyRiDMQZVxU4StqkqHk0cSEISf5KZ7DKTMQbLsrCTRGuN3jtVxaOJg6qiqqgqqoqqoqoYY5CZ7GwTEdzvd97f34kIeu/YRhKPJg6qiswkM7ndbmQmmUlmkpnsbBMR2CYimOeZ3ju2kcSjiYOqIjP5+vpi2za2bWPbNo5aa7TW2PXe6b3Te6e1hiQeTRxUFbfbjW3bGGNwvV4ZY2Ab27TWsI1tbGMb27TWsI0kHk0cVBWZybZtXK9XPj8/+fj4YJ5nIoLWGraJCOZ5RhKSkIQkJPFo4qCqyEy2bWOMwefnJ+u6cjqdsM3ONvM8cz6feca0ris/rtcrmcnONhHB/X7n/f2diKD3jm0k8axpWRZ+ZCaZyc42EYFtIoJ5num9YxtJPGta15U/sY1tdm9vb/Te6b1jG0k8a1qWhR+2sU1rjdYatrGNbWxjm9YaknjWtK4rPyKCiKC1hm0igojg9fUVSUhCEpJ41rQsC0e22dkmIrhcLvyNF/7H6XTib73wy174Zf8AJEsePtlPj10AAAAASUVORK5CYII=';174      await f.fixImageTemplateScale(TINY_PNG, {175        defaultImageTemplateScale: 4.0,176        fixImageTemplateScale: true,177        xScale: 1.5, yScale: 1.5178      }).should.eventually.eql(actual);179    });180    it('should not fix template size scale with default scale and image scale', async function () {181      const actual = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABwUlEQVR4AaXBPUsrQQCG0SeX+cBdkTjwTpG1NPgLpjY/fW1stt4UYmm2cJqwMCsaw70uJJ3CBc9Z/P3Cl+12S9u2tG1L27bEGLm/v2ez2bDZbJDEd/7wS4YT7z3X19fc3Nxwd3dHXdd47xnHkefnZ8ZxpKoq6rqmqiqMMcwMJ1VV0TQN0zThnOPj44O6rsk503UdkmiahqZpWK1WGGOYGU7quqZpGqy1SCLnTM6Z19dXcs5IYpomrLVI4uLigpnhpKoqVqsVkjgcDjw9PdF1HTlnuq5DEs45JHE4HDgznByPR97e3pimiVIK4zhyPB7x3hNCIITA5eUl3nsWiwVnhpNSCsMwsNvtGIaB/X5PKQVJpJSQxHq9RhLOOc4MJ9M0sdvt2G639H3PTBIxRiQhCUnEGLHWcmY4KaUwDAN93/P4+MhyuSSlhCRSSkjCOYe1FmstZ6bve2YvLy/s93tmy+USSUhCEpIIIfAd8/DwwOz9/Z1SCpJIKSGJ9XqNJJxz/MS0bcvs6uoKScQYkYQkJBFjxFrLT0zbtsxub29JKSGJlBKScM5hrcVay09MzplZjJHPz0+894QQCCHwP/7wS/8A4e6nAg+R8LwAAAAASUVORK5CYII=';182      await f.fixImageTemplateScale(TINY_PNG, {183        defaultImageTemplateScale: 4.0,184        fixImageTemplateScale: false,185        xScale: 1.5, yScale: 1.5186      }).should.eventually.eql(actual);187    });188    it('should not fix template size scale because of ignoreDefaultImageTemplateScale', async function () {189      await f.fixImageTemplateScale(TINY_PNG, {190        defaultImageTemplateScale: 4.0,191        ignoreDefaultImageTemplateScale: true,192      }).should.eventually.eql(TINY_PNG);193    });194    it('should ignore defaultImageTemplateScale to fix template size scale because of ignoreDefaultImageTemplateScale', async function () {195      const actual = 'iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAYAAADgzO9IAAAAWElEQVR4AU3BQRWAQAhAwa/PGBsEgrC16AFBKEIPXW7OXO+Rmey9iQjMjHFzrLUwM7qbqmLcHKpKRFBVuDvj4agq3B1VRUQYT2bS3QwRQVUZF/CaGRHB3wc1vSZbHO5+BgAAAABJRU5ErkJggg==';196      await f.fixImageTemplateScale(TINY_PNG, {197        defaultImageTemplateScale: 4.0,198        ignoreDefaultImageTemplateScale: true,199        fixImageTemplateScale: true,200        xScale: 1.5, yScale: 1.5201      }).should.eventually.eql(actual);202    });203  });204  describe('ensureTemplateSize', function () {205    const d = new PluginDriver();206    const f = new ImageElementFinder(d);207    it('should not resize the template if it is smaller than the screen', async function () {208      const screen = TINY_PNG_DIMS.map((n) => n * 2);209      await f.ensureTemplateSize(TINY_PNG, ...screen)210        .should.eventually.eql(TINY_PNG);211    });212    it('should not resize the template if it is the same size as the screen', async function () {213      await f.ensureTemplateSize(TINY_PNG, ...TINY_PNG_DIMS)214        .should.eventually.eql(TINY_PNG);215    });216    it('should resize the template if it is bigger than the screen', async function () {217      const screen = TINY_PNG_DIMS.map((n) => n / 2);218      const newTemplate = await f.ensureTemplateSize(TINY_PNG, ...screen);219      newTemplate.should.not.eql(TINY_PNG);220      newTemplate.length.should.be.below(TINY_PNG.length);221    });222  });223  describe('getScreenshotForImageFind', function () {224    let d;225    let f;226    beforeEach(function () {227      d = new PluginDriver();228      f = new ImageElementFinder(d);229      sinon.stub(d, 'getScreenshot').returns(TINY_PNG);230    });231    it('should fail if driver does not support getScreenshot', async function () {232      const d = new BaseDriver();233      const f = new ImageElementFinder(d);234      await f.getScreenshotForImageFind()235        .should.eventually.be.rejectedWith(/driver does not support/);236    });237    it('should not adjust or verify screenshot if asked not to by settings', async function () {238      await d.settings.update({fixImageFindScreenshotDims: false});239      const screen = TINY_PNG_DIMS.map((n) => n + 1);240      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);241      b64Screenshot.should.eql(TINY_PNG);242      should.equal(scale, undefined);243    });244    it('should return screenshot without adjustment if it matches screen size', async function () {245      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...TINY_PNG_DIMS);246      b64Screenshot.should.eql(TINY_PNG);247      should.equal(scale, undefined);248    });249    it('should return scaled screenshot with same aspect ratio if matching screen aspect ratio', async function () {250      const screen = TINY_PNG_DIMS.map((n) => n * 1.5);251      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);252      b64Screenshot.should.not.eql(TINY_PNG);253      const screenshotObj = await imageUtil.getJimpImage(b64Screenshot);254      screenshotObj.bitmap.width.should.eql(screen[0]);255      screenshotObj.bitmap.height.should.eql(screen[1]);256      scale.should.eql({ xScale: 1.5, yScale: 1.5 });257    });258    it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio', async function () {259      // try first with portrait screen, screen = 8 x 12260      let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];261      let expectedScale = { xScale: 2.67, yScale: 4 };262      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);263      b64Screenshot.should.not.eql(TINY_PNG);264      let screenshotObj = await imageUtil.getJimpImage(b64Screenshot);265      screenshotObj.bitmap.width.should.eql(screen[0]);266      screenshotObj.bitmap.height.should.eql(screen[1]);267      scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());268      scale.yScale.should.eql(expectedScale.yScale);269      // then with landscape screen, screen = 12 x 8270      screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];271      expectedScale = { xScale: 4, yScale: 2.67 };272      const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(...screen);273      newScreen.should.not.eql(TINY_PNG);274      screenshotObj = await imageUtil.getJimpImage(newScreen);275      screenshotObj.bitmap.width.should.eql(screen[0]);276      screenshotObj.bitmap.height.should.eql(screen[1]);277      newScale.xScale.should.eql(expectedScale.xScale);278      newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());279    });280    it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio with fixImageTemplateScale', async function () {281      // try first with portrait screen, screen = 8 x 12282      let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];283      let expectedScale = { xScale: 2.67, yScale: 4 };284      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);285      b64Screenshot.should.not.eql(TINY_PNG);286      let screenshotObj = await imageUtil.getJimpImage(b64Screenshot);287      screenshotObj.bitmap.width.should.eql(screen[0]);288      screenshotObj.bitmap.height.should.eql(screen[1]);289      scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());290      scale.yScale.should.eql(expectedScale.yScale);291      // 8 x 12 stretched TINY_PNG292      await f.fixImageTemplateScale(b64Screenshot, {fixImageTemplateScale: true, scale})293        .should.eventually.eql('iVBORw0KGgoAAAANSUhEUgAAAAgAAAAMCAYAAABfnvydAAAAJ0lEQVR4AYXBAQEAIACDMKR/p0fTBrKdbZcPCRIkSJAgQYIECRIkPAzBA1TpeNwZAAAAAElFTkSuQmCC');294      // then with landscape screen, screen = 12 x 8295      screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];296      expectedScale = { xScale: 4, yScale: 2.67 };297      const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(...screen);298      newScreen.should.not.eql(TINY_PNG);299      screenshotObj = await imageUtil.getJimpImage(newScreen);300      screenshotObj.bitmap.width.should.eql(screen[0]);301      screenshotObj.bitmap.height.should.eql(screen[1]);302      newScale.xScale.should.eql(expectedScale.xScale);303      newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());304      // 12 x 8 stretched TINY_PNG305      await f.fixImageTemplateScale(newScreen, {fixImageTemplateScale: true, scale})306        .should.eventually.eql('iVBORw0KGgoAAAANSUhEUgAAAAwAAAAICAYAAADN5B7xAAAAI0lEQVR4AZXBAQEAMAyDMI5/T5W2ayB5245AIokkkkgiiST6+W4DTLyo5PUAAAAASUVORK5CYII=');307    });308  });...