How to use helpers.fixImageTemplateScale method in Appium Base Driver

1import log from '../logger';2import { logger, imageUtil } from 'appium-support';3import _ from 'lodash';4import { errors } from '../../..';5import { MATCH_TEMPLATE_MODE } from './images';6import { ImageElement, DEFAULT_TEMPLATE_IMAGE_SCALE } from '../image-element';7const commands = {}, helpers = {}, extensions = {};8const IMAGE_STRATEGY = '-image';9const CUSTOM_STRATEGY = '-custom';10// Used to compare ratio and screen width11// Pixel is basically under 1080 for example. 100K is probably enough fo a while.12const FLOAT_PRECISION = 100000;13// Override the following function for your own driver, and the rest is taken14// care of!15// helpers.findElOrEls = async function (strategy, selector, mult, context) {}16//   strategy: locator strategy17//   selector: the actual selector for finding an element18//   mult: multiple elements or just one?19//   context: finding an element from the root context? or starting from another element20//21// Returns an object which adheres to the way the JSON Wire Protocol represents elements:22// { ELEMENT: # }    eg: { ELEMENT: 3 }  or { ELEMENT: 1.023 }23helpers.findElOrElsWithProcessing = async function findElOrElsWithProcessing (strategy, selector, mult, context) {24  this.validateLocatorStrategy(strategy);25  try {26    return await this.findElOrEls(strategy, selector, mult, context);27  } catch (err) {28    if (this.opts.printPageSourceOnFindFailure) {29      const src = await this.getPageSource();30      log.debug(`Error finding element${mult ? 's' : ''}: ${err.message}`);31      log.debug(`Page source requested through 'printPageSourceOnFindFailure':`);32      log.debug(src);33    }34    // still want the error to occur35    throw err;36  }37};38commands.findElement = async function findElement (strategy, selector) {39  if (strategy === IMAGE_STRATEGY) {40    return await this.findByImage(selector, {multiple: false});41  } else if (strategy === CUSTOM_STRATEGY) {42    return await this.findByCustom(selector, false);43  }44  return await this.findElOrElsWithProcessing(strategy, selector, false);45};46commands.findElements = async function findElements (strategy, selector) {47  if (strategy === IMAGE_STRATEGY) {48    return await this.findByImage(selector, {multiple: true});49  } else if (strategy === CUSTOM_STRATEGY) {50    return await this.findByCustom(selector, true);51  }52  return await this.findElOrElsWithProcessing(strategy, selector, true);53};54commands.findElementFromElement = async function findElementFromElement (strategy, selector, elementId) {55  return await this.findElOrElsWithProcessing(strategy, selector, false, elementId);56};57commands.findElementsFromElement = async function findElementsFromElement (strategy, selector, elementId) {58  return await this.findElOrElsWithProcessing(strategy, selector, true, elementId);59};60/**61 * Find an element using a custom plugin specified by the customFindModules cap.62 *63 * @param {string} selector - the selector which the plugin will use to find64 * elements65 * @param {boolean} multiple - whether we want one element or multiple66 *67 * @returns {WebElement} - WebDriver element or list of elements68 */69commands.findByCustom = async function findByCustom (selector, multiple) {70  const plugins = this.opts.customFindModules;71  // first ensure the user has registered one or more find plugins72  if (!plugins) {73    // TODO this info should go in docs instead; update when docs for this74    // feature exist75    throw new Error('Finding an element using a plugin is currently an ' +76      'incubating feature. To use it you must manually install one or more ' +77      'plugin modules in a way that they can be required by Appium, for ' +78      'example installing them from the Appium directory, installing them ' +79      'globally, or installing them elsewhere and passing an absolute path as ' +80      'the capability. Then construct an object where the key is the shortcut ' +81      'name for this plugin and the value is the module name or absolute path, ' +82      'for example: {"p1": "my-find-plugin"}, and pass this in as the ' +83      "'customFindModules' capability.");84  }85  // then do some basic checking of the type of the capability86  if (!_.isPlainObject(plugins)) {87    throw new Error("Invalid format for the 'customFindModules' capability. " +88      'It should be an object with keys corresponding to the short names and ' +89      'values corresponding to the full names of the element finding plugins');90  }91  // get the name of the particular plugin used for this invocation of find,92  // and separate it from the selector we will pass to the plugin93  let [plugin, realSelector] = selector.split(':');94  // if the user didn't specify a plugin for this find invocation, and we had95  // multiple plugins registered, that's a problem96  if (_.size(plugins) > 1 && !realSelector) {97    throw new Error(`Multiple element finding plugins were registered ` +98      `(${_.keys(plugins)}), but your selector did not indicate which plugin ` +99      `to use. Ensure you put the short name of the plugin followed by ':' as ` +100      `the initial part of the selector string.`);101  }102  // but if they did not specify a plugin and we only have one plugin, just use103  // that one104  if (_.size(plugins) === 1 && !realSelector) {105    realSelector = plugin;106    plugin = _.keys(plugins)[0];107  }108  if (!plugins[plugin]) {109    throw new Error(`Selector specified use of element finding plugin ` +110      `'${plugin}' but it was not registered in the 'customFindModules' ` +111      `capability.`);112  }113  let finder;114  try {115    log.debug(`Find plugin '${plugin}' requested; will attempt to use it ` +116      `from '${plugins[plugin]}'`);117    finder = require(plugins[plugin]);118  } catch (err) {119    throw new Error(`Could not load your custom find module '${plugin}'. Did ` +120      `you put it somewhere Appium can 'require' it? Original error: ${err}`);121  }122  if (!finder || !_.isFunction(finder.find)) {123    throw new Error('Your custom find module did not appear to be constructed ' +124        'correctly. It needs to export an object with a `find` method.');125  }126  const customFinderLog = logger.getLogger(plugin);127  let elements;128  const condition = async () => {129    // get a list of matched elements from the custom finder, which can130    // potentially use the entire suite of methods the current driver provides.131    // the finder should always return a list of elements, but may use the132    // knowledge of whether we are looking for one or many to perform internal133    // optimizations134    elements = await finder.find(this, customFinderLog, realSelector, multiple);135    // if we're looking for multiple elements, or if we're looking for only136    // one and found it, we're done137    if (!_.isEmpty(elements) || multiple) {138      return true;139    }140    // otherwise we should retry, so return false to trigger the retry loop141    return false;142  };143  try {144    // make sure we respect implicit wait145    await this.implicitWaitForCondition(condition);146  } catch (err) {147    if (err.message.match(/Condition unmet/)) {148      throw new errors.NoSuchElementError();149    }150    throw err;151  }152  return multiple ? elements : elements[0];153};154/**155 * @typedef {Object} FindByImageOptions156 * @property {boolean} [shouldCheckStaleness=false] - whether this call to find an157 * image is merely to check staleness. If so we can bypass a lot of logic158 * @property {boolean} [multiple=false] - Whether we are finding one element or159 * multiple160 * @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we161 * ignore defaultImageTemplateScale. It can be used when you would like to162 * scale b64Template with defaultImageTemplateScale setting.163 */164/**165 * Find a screen rect represented by an ImageElement corresponding to an image166 * template sent in by the client167 *168 * @param {string} b64Template - base64-encoded image used as a template to be169 * matched in the screenshot170 * @param {FindByImageOptions} - additional options171 *172 * @returns {WebElement} - WebDriver element with a special id prefix173 */174helpers.findByImage = async function findByImage (b64Template, {175  shouldCheckStaleness = false,176  multiple = false,177  ignoreDefaultImageTemplateScale = false,178}) {179  const {180    imageMatchThreshold: threshold,181    imageMatchMethod,182    fixImageTemplateSize,183    fixImageTemplateScale,184    defaultImageTemplateScale,185    getMatchedImageResult: visualize186  } = this.settings.getSettings();187  log.info(`Finding image element with match threshold ${threshold}`);188  if (!this.getWindowSize) {189    throw new Error("This driver does not support the required 'getWindowSize' command");190  }191  const {width: screenWidth, height: screenHeight} = await this.getWindowSize();192  // someone might have sent in a template that's larger than the screen193  // dimensions. If so let's check and cut it down to size since the algorithm194  // will not work unless we do. But because it requires some potentially195  // expensive commands, only do this if the user has requested it in settings.196  if (fixImageTemplateSize) {197    b64Template = await this.ensureTemplateSize(b64Template, screenWidth,198      screenHeight);199  }200  const results = [];201  const condition = async () => {202    try {203      const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);204      b64Template = await this.fixImageTemplateScale(b64Template, {205        defaultImageTemplateScale, ignoreDefaultImageTemplateScale,206        fixImageTemplateScale, ...scale207      });208      const comparisonOpts = {209        threshold,210        visualize,211        multiple,212      };213      if (imageMatchMethod) {214        comparisonOpts.method = imageMatchMethod;215      }216      if (multiple) {217        results.push(...(await this.compareImages(MATCH_TEMPLATE_MODE,218                                                  b64Screenshot,219                                                  b64Template,220                                                  comparisonOpts)));221      } else {222        results.push(await this.compareImages(MATCH_TEMPLATE_MODE,223                                              b64Screenshot,224                                              b64Template,225                                              comparisonOpts));226      }227      return true;228    } catch (err) {229      // if compareImages fails, we'll get a specific error, but we should230      // retry, so trap that and just return false to trigger the next round of231      // implicitly waiting. For other errors, throw them to get out of the232      // implicit wait loop233      if (err.message.match(/Cannot find any occurrences/)) {234        return false;235      }236      throw err;237    }238  };239  try {240    await this.implicitWaitForCondition(condition);241  } catch (err) {242    // this `implicitWaitForCondition` method will throw a 'Condition unmet'243    // error if an element is not found eventually. In that case, we will244    // handle the element not found response below. In the case where get some245    // _other_ kind of error, it means something blew up totally apart from the246    // implicit wait timeout. We should not mask that error and instead throw247    // it straightaway248    if (!err.message.match(/Condition unmet/)) {249      throw err;250    }251  }252  if (_.isEmpty(results)) {253    if (multiple) {254      return [];255    }256    throw new errors.NoSuchElementError();257  }258  const elements = results.map(({rect, score, visualization}) => {259    log.info(`Image template matched: ${JSON.stringify(rect)}`);260    return new ImageElement(b64Template, rect, score, visualization);261  });262  // if we're just checking staleness, return straightaway so we don't add263  // a new element to the cache. shouldCheckStaleness does not support multiple264  // elements, since it is a purely internal mechanism265  if (shouldCheckStaleness) {266    return elements[0];267  }268  const registeredElements = elements.map((imgEl) => this.registerImageElement(imgEl));269  return multiple ? registeredElements : registeredElements[0];270};271/**272 * Ensure that the image template sent in for a find is of a suitable size273 *274 * @param {string} b64Template - base64-encoded image275 * @param {int} screenWidth - width of screen276 * @param {int} screenHeight - height of screen277 *278 * @returns {string} base64-encoded image, potentially resized279 */280helpers.ensureTemplateSize = async function ensureTemplateSize (b64Template, screenWidth, screenHeight) {281  let imgObj = await imageUtil.getJimpImage(b64Template);282  let {width: tplWidth, height: tplHeight} = imgObj.bitmap;283  log.info(`Template image is ${tplWidth}x${tplHeight}. Screen size is ${screenWidth}x${screenHeight}`);284  // if the template fits inside the screen dimensions, we're good285  if (tplWidth <= screenWidth && tplHeight <= screenHeight) {286    return b64Template;287  }288  log.info(`Scaling template image from ${tplWidth}x${tplHeight} to match ` +289           `screen at ${screenWidth}x${screenHeight}`);290  // otherwise, scale it to fit inside the screen dimensions291  imgObj = imgObj.scaleToFit(screenWidth, screenHeight);292  return (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');293};294/**295 * @typedef {Object} Screenshot296 * @property {string} b64Screenshot - base64 based screenshot string297 */298/**299 * @typedef {Object} ScreenshotScale300 * @property {float} xScale - Scale ratio for width301 * @property {float} yScale - Scale ratio for height302 */303/**304 * Get the screenshot image that will be used for find by element, potentially305 * altering it in various ways based on user-requested settings306 *307 * @param {int} screenWidth - width of screen308 * @param {int} screenHeight - height of screen309 *310 * @returns {Screenshot, ?ScreenshotScale} base64-encoded screenshot and ScreenshotScale311 */312helpers.getScreenshotForImageFind = async function getScreenshotForImageFind (screenWidth, screenHeight) {313  if (!this.getScreenshot) {314    throw new Error("This driver does not support the required 'getScreenshot' command");315  }316  let b64Screenshot = await this.getScreenshot();317  // if the user has requested not to correct for aspect or size differences318  // between the screenshot and the screen, just return the screenshot now319  if (!this.settings.getSettings().fixImageFindScreenshotDims) {320    log.info(`Not verifying screenshot dimensions match screen`);321    return {b64Screenshot};322  }323  if (screenWidth < 1 || screenHeight < 1) {324    log.warn(`The retrieved screen size ${screenWidth}x${screenHeight} does ` +325      `not seem to be valid. No changes will be applied to the screenshot`);326    return {b64Screenshot};327  }328  // otherwise, do some verification on the screenshot to make sure it matches329  // the screen size and aspect ratio330  log.info('Verifying screenshot size and aspect ratio');331  let imgObj = await imageUtil.getJimpImage(b64Screenshot);332  let {width: shotWidth, height: shotHeight} = imgObj.bitmap;333  if (shotWidth < 1 || shotHeight < 1) {334    log.warn(`The retrieved screenshot size ${shotWidth}x${shotHeight} does ` +335      `not seem to be valid. No changes will be applied to the screenshot`);336    return {b64Screenshot};337  }338  if (screenWidth === shotWidth && screenHeight === shotHeight) {339    // the height and width of the screenshot and the device screen match, which340    // means we should be safe when doing template matches341    log.info('Screenshot size matched screen size');342    return {b64Screenshot};343  }344  // otherwise, if they don't match, it could spell problems for the accuracy345  // of coordinates returned by the image match algorithm, since we match based346  // on the screenshot coordinates not the device coordinates themselves. There347  // are two potential types of mismatch: aspect ratio mismatch and scale348  // mismatch. We need to detect and fix both349  const scale = {xScale: 1.0, yScale: 1.0};350  const screenAR = screenWidth / screenHeight;351  const shotAR = shotWidth / shotHeight;352  if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {353    log.info(`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +354      `screen aspect ratio '${screenAR}' (${screenWidth}x${screenHeight})`);355  } else {356    log.warn(`When trying to find an element, determined that the screen ` +357             `aspect ratio and screenshot aspect ratio are different. Screen ` +358             `is ${screenWidth}x${screenHeight} whereas screenshot is ` +359             `${shotWidth}x${shotHeight}.`);360    // In the case where the x-scale and y-scale are different, we need to decide361    // which one to respect, otherwise the screenshot and template will end up362    // being resized in a way that changes its aspect ratio (distorts it). For example, let's say:363    // this.getScreenshot(shotWidth, shotHeight) is 540x397,364    // this.getDeviceSize(screenWidth, screenHeight) is 1080x1920.365    // The ratio would then be {xScale: 0.5, yScale: 0.2}.366    // In this case, we must should `yScale: 0.2` as scaleFactor, because367    // if we select the xScale, the height will be bigger than real screenshot size368    // which is used to image comparison by OpenCV as a base image.369    // All of this is primarily useful when the screenshot is a horizontal slice taken out of the370    // screen (for example not including top/bottom nav bars)371    const xScale = (1.0 * shotWidth) / screenWidth;372    const yScale = (1.0 * shotHeight) / screenHeight;373    const scaleFactor = xScale >= yScale ? yScale : xScale;374    log.warn(`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` +375             `screen aspect ratio so that image element coordinates have a ` +376             `greater chance of being correct.`);377    imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor);378    scale.xScale *= scaleFactor;379    scale.yScale *= scaleFactor;380    shotWidth = imgObj.bitmap.width;381    shotHeight = imgObj.bitmap.height;382  }383  // Resize based on the screen dimensions only if both width and height are mismatched384  // since except for that, it might be a situation which is different window rect and385  // screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and386  // `"deviceScreenSize"=>"1080x1920"`387  if (screenWidth !== shotWidth && screenHeight !== shotHeight) {388    log.info(`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +389             `screen at ${screenWidth}x${screenHeight}`);390    imgObj = imgObj.resize(screenWidth, screenHeight);391    scale.xScale *= (1.0 * screenWidth) / shotWidth;392    scale.yScale *= (1.0 * screenHeight) / shotHeight;393  }394  b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');395  return {b64Screenshot, scale};396};397/**398 * @typedef {Object} ImageTemplateSettings399 * @property {boolean} fixImageTemplateScale - fixImageTemplateScale in device-settings400 * @property {float} defaultImageTemplateScale - defaultImageTemplateScale in device-settings401 * @property {boolean} ignoreDefaultImageTemplateScale - Ignore defaultImageTemplateScale if it has true.402 * If b64Template has been scaled to defaultImageTemplateScale or should ignore the scale,403 * this parameter should be true. e.g. click in image-element module404 * @property {float} xScale - Scale ratio for width405 * @property {float} yScale - Scale ratio for height406 */407/**408 * Get a image that will be used for template maching.409 * Returns scaled image if scale ratio is provided.410 *411 *412 * @param {string} b64Template - base64-encoded image used as a template to be413 * matched in the screenshot414 * @param {ImageTemplateSettings} opts - Image template scale related options415 *416 * @returns {string} base64-encoded scaled template screenshot417 */418const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;419helpers.fixImageTemplateScale = async function fixImageTemplateScale (b64Template, opts = {}) {420  if (!opts) {421    return b64Template;422  }423  let {424    fixImageTemplateScale = false,425    defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,426    ignoreDefaultImageTemplateScale = false,427    xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,428    yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE429  } = opts;430  if (ignoreDefaultImageTemplateScale) {431    defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;432  }433  // Default434  if (defaultImageTemplateScale === DEFAULT_TEMPLATE_IMAGE_SCALE && !fixImageTemplateScale) {435    return b64Template;436  }437  // Calculate xScale and yScale Appium should scale438  if (fixImageTemplateScale) {439    xScale *= defaultImageTemplateScale;440    yScale *= defaultImageTemplateScale;441  } else {442    xScale = yScale = 1 * defaultImageTemplateScale;443  }444  // xScale and yScale can be NaN if defaultImageTemplateScale is string, for example445  if (!parseFloat(xScale) || !parseFloat(yScale)) {446    return b64Template;447  }448  // Return if the scale is default, 1, value449  if (Math.round(xScale * FLOAT_PRECISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)450      && Math.round(yScale * FLOAT_PRECISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION))) {451    return b64Template;452  }453  let imgTempObj = await imageUtil.getJimpImage(b64Template);454  let {width: baseTempWidth, height: baseTempHeigh} = imgTempObj.bitmap;455  const scaledWidth = baseTempWidth * xScale;456  const scaledHeight = baseTempHeigh * yScale;457  log.info(`Scaling template image from ${baseTempWidth}x${baseTempHeigh}` +458            ` to ${scaledWidth}x${scaledHeight}`);459  log.info(`The ratio is ${xScale} and ${yScale}`);460  imgTempObj = await imgTempObj.resize(scaledWidth, scaledHeight);461  return (await imgTempObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');462};463Object.assign(extensions, commands, helpers);464export { commands, helpers, IMAGE_STRATEGY, CUSTOM_STRATEGY };...