How to use d.getScreenshotForImageFind method in Appium Base Driver

Best JavaScript code snippet using appium-base-driver

find.js

Source:find.js Github

copy

Full Screen

1import log from '../logger';2import { logger, imageUtil } from 'appium-support';3import _ from 'lodash';4import { errors } from '../../..';5import { MATCH_TEMPLATE_MODE } from './images';6import { ImageElement, DEFAULT_TEMPLATE_IMAGE_SCALE } from '../image-element';7const commands = {}, helpers = {}, extensions = {};8const IMAGE_STRATEGY = '-image';9const CUSTOM_STRATEGY = '-custom';10// Used to compare ratio and screen width11// Pixel is basically under 1080 for example. 100K is probably enough fo a while.12const FLOAT_PRECISION = 100000;13// Override the following function for your own driver, and the rest is taken14// care of!15// helpers.findElOrEls = async function (strategy, selector, mult, context) {}16// strategy: locator strategy17// selector: the actual selector for finding an element18// mult: multiple elements or just one?19// context: finding an element from the root context? or starting from another element20//21// Returns an object which adheres to the way the JSON Wire Protocol represents elements:22// { ELEMENT: # } eg: { ELEMENT: 3 } or { ELEMENT: 1.023 }23helpers.findElOrElsWithProcessing = async function findElOrElsWithProcessing (strategy, selector, mult, context) {24 this.validateLocatorStrategy(strategy);25 try {26 return await this.findElOrEls(strategy, selector, mult, context);27 } catch (err) {28 if (this.opts.printPageSourceOnFindFailure) {29 const src = await this.getPageSource();30 log.debug(`Error finding element${mult ? 's' : ''}: ${err.message}`);31 log.debug(`Page source requested through 'printPageSourceOnFindFailure':`);32 log.debug(src);33 }34 // still want the error to occur35 throw err;36 }37};38commands.findElement = async function findElement (strategy, selector) {39 if (strategy === IMAGE_STRATEGY) {40 return await this.findByImage(selector, {multiple: false});41 } else if (strategy === CUSTOM_STRATEGY) {42 return await this.findByCustom(selector, false);43 }44 return await this.findElOrElsWithProcessing(strategy, selector, false);45};46commands.findElements = async function findElements (strategy, selector) {47 if (strategy === IMAGE_STRATEGY) {48 return await this.findByImage(selector, {multiple: true});49 } else if (strategy === CUSTOM_STRATEGY) {50 return await this.findByCustom(selector, true);51 }52 return await this.findElOrElsWithProcessing(strategy, selector, true);53};54commands.findElementFromElement = async function findElementFromElement (strategy, selector, elementId) {55 return await this.findElOrElsWithProcessing(strategy, selector, false, elementId);56};57commands.findElementsFromElement = async function findElementsFromElement (strategy, selector, elementId) {58 return await this.findElOrElsWithProcessing(strategy, selector, true, elementId);59};60/**61 * Find an element using a custom plugin specified by the customFindModules cap.62 *63 * @param {string} selector - the selector which the plugin will use to find64 * elements65 * @param {boolean} multiple - whether we want one element or multiple66 *67 * @returns {WebElement} - WebDriver element or list of elements68 */69commands.findByCustom = async function findByCustom (selector, multiple) {70 const plugins = this.opts.customFindModules;71 // first ensure the user has registered one or more find plugins72 if (!plugins) {73 // TODO this info should go in docs instead; update when docs for this74 // feature exist75 throw new Error('Finding an element using a plugin is currently an ' +76 'incubating feature. To use it you must manually install one or more ' +77 'plugin modules in a way that they can be required by Appium, for ' +78 'example installing them from the Appium directory, installing them ' +79 'globally, or installing them elsewhere and passing an absolute path as ' +80 'the capability. Then construct an object where the key is the shortcut ' +81 'name for this plugin and the value is the module name or absolute path, ' +82 'for example: {"p1": "my-find-plugin"}, and pass this in as the ' +83 "'customFindModules' capability.");84 }85 // then do some basic checking of the type of the capability86 if (!_.isPlainObject(plugins)) {87 throw new Error("Invalid format for the 'customFindModules' capability. " +88 'It should be an object with keys corresponding to the short names and ' +89 'values corresponding to the full names of the element finding plugins');90 }91 // get the name of the particular plugin used for this invocation of find,92 // and separate it from the selector we will pass to the plugin93 let [plugin, realSelector] = selector.split(':');94 // if the user didn't specify a plugin for this find invocation, and we had95 // multiple plugins registered, that's a problem96 if (_.size(plugins) > 1 && !realSelector) {97 throw new Error(`Multiple element finding plugins were registered ` +98 `(${_.keys(plugins)}), but your selector did not indicate which plugin ` +99 `to use. Ensure you put the short name of the plugin followed by ':' as ` +100 `the initial part of the selector string.`);101 }102 // but if they did not specify a plugin and we only have one plugin, just use103 // that one104 if (_.size(plugins) === 1 && !realSelector) {105 realSelector = plugin;106 plugin = _.keys(plugins)[0];107 }108 if (!plugins[plugin]) {109 throw new Error(`Selector specified use of element finding plugin ` +110 `'${plugin}' but it was not registered in the 'customFindModules' ` +111 `capability.`);112 }113 let finder;114 try {115 log.debug(`Find plugin '${plugin}' requested; will attempt to use it ` +116 `from '${plugins[plugin]}'`);117 finder = require(plugins[plugin]);118 } catch (err) {119 throw new Error(`Could not load your custom find module '${plugin}'. Did ` +120 `you put it somewhere Appium can 'require' it? Original error: ${err}`);121 }122 if (!finder || !_.isFunction(finder.find)) {123 throw new Error('Your custom find module did not appear to be constructed ' +124 'correctly. It needs to export an object with a `find` method.');125 }126 const customFinderLog = logger.getLogger(plugin);127 let elements;128 const condition = async () => {129 // get a list of matched elements from the custom finder, which can130 // potentially use the entire suite of methods the current driver provides.131 // the finder should always return a list of elements, but may use the132 // knowledge of whether we are looking for one or many to perform internal133 // optimizations134 elements = await finder.find(this, customFinderLog, realSelector, multiple);135 // if we're looking for multiple elements, or if we're looking for only136 // one and found it, we're done137 if (!_.isEmpty(elements) || multiple) {138 return true;139 }140 // otherwise we should retry, so return false to trigger the retry loop141 return false;142 };143 try {144 // make sure we respect implicit wait145 await this.implicitWaitForCondition(condition);146 } catch (err) {147 if (err.message.match(/Condition unmet/)) {148 throw new errors.NoSuchElementError();149 }150 throw err;151 }152 return multiple ? elements : elements[0];153};154/**155 * @typedef {Object} FindByImageOptions156 * @property {boolean} [shouldCheckStaleness=false] - whether this call to find an157 * image is merely to check staleness. If so we can bypass a lot of logic158 * @property {boolean} [multiple=false] - Whether we are finding one element or159 * multiple160 * @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we161 * ignore defaultImageTemplateScale. It can be used when you would like to162 * scale b64Template with defaultImageTemplateScale setting.163 */164/**165 * Find a screen rect represented by an ImageElement corresponding to an image166 * template sent in by the client167 *168 * @param {string} b64Template - base64-encoded image used as a template to be169 * matched in the screenshot170 * @param {FindByImageOptions} - additional options171 *172 * @returns {WebElement} - WebDriver element with a special id prefix173 */174helpers.findByImage = async function findByImage (b64Template, {175 shouldCheckStaleness = false,176 multiple = false,177 ignoreDefaultImageTemplateScale = false,178}) {179 const {180 imageMatchThreshold: threshold,181 imageMatchMethod,182 fixImageTemplateSize,183 fixImageTemplateScale,184 defaultImageTemplateScale,185 getMatchedImageResult: visualize186 } = this.settings.getSettings();187 log.info(`Finding image element with match threshold ${threshold}`);188 if (!this.getWindowSize) {189 throw new Error("This driver does not support the required 'getWindowSize' command");190 }191 const {width: screenWidth, height: screenHeight} = await this.getWindowSize();192 // someone might have sent in a template that's larger than the screen193 // dimensions. If so let's check and cut it down to size since the algorithm194 // will not work unless we do. But because it requires some potentially195 // expensive commands, only do this if the user has requested it in settings.196 if (fixImageTemplateSize) {197 b64Template = await this.ensureTemplateSize(b64Template, screenWidth,198 screenHeight);199 }200 const results = [];201 const condition = async () => {202 try {203 const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);204 b64Template = await this.fixImageTemplateScale(b64Template, {205 defaultImageTemplateScale, ignoreDefaultImageTemplateScale,206 fixImageTemplateScale, ...scale207 });208 const comparisonOpts = {209 threshold,210 visualize,211 multiple,212 };213 if (imageMatchMethod) {214 comparisonOpts.method = imageMatchMethod;215 }216 if (multiple) {217 results.push(...(await this.compareImages(MATCH_TEMPLATE_MODE,218 b64Screenshot,219 b64Template,220 comparisonOpts)));221 } else {222 results.push(await this.compareImages(MATCH_TEMPLATE_MODE,223 b64Screenshot,224 b64Template,225 comparisonOpts));226 }227 return true;228 } catch (err) {229 // if compareImages fails, we'll get a specific error, but we should230 // retry, so trap that and just return false to trigger the next round of231 // implicitly waiting. For other errors, throw them to get out of the232 // implicit wait loop233 if (err.message.match(/Cannot find any occurrences/)) {234 return false;235 }236 throw err;237 }238 };239 try {240 await this.implicitWaitForCondition(condition);241 } catch (err) {242 // this `implicitWaitForCondition` method will throw a 'Condition unmet'243 // error if an element is not found eventually. In that case, we will244 // handle the element not found response below. In the case where get some245 // _other_ kind of error, it means something blew up totally apart from the246 // implicit wait timeout. We should not mask that error and instead throw247 // it straightaway248 if (!err.message.match(/Condition unmet/)) {249 throw err;250 }251 }252 if (_.isEmpty(results)) {253 if (multiple) {254 return [];255 }256 throw new errors.NoSuchElementError();257 }258 const elements = results.map(({rect, score, visualization}) => {259 log.info(`Image template matched: ${JSON.stringify(rect)}`);260 return new ImageElement(b64Template, rect, score, visualization);261 });262 // if we're just checking staleness, return straightaway so we don't add263 // a new element to the cache. shouldCheckStaleness does not support multiple264 // elements, since it is a purely internal mechanism265 if (shouldCheckStaleness) {266 return elements[0];267 }268 const registeredElements = elements.map((imgEl) => this.registerImageElement(imgEl));269 return multiple ? registeredElements : registeredElements[0];270};271/**272 * Ensure that the image template sent in for a find is of a suitable size273 *274 * @param {string} b64Template - base64-encoded image275 * @param {int} screenWidth - width of screen276 * @param {int} screenHeight - height of screen277 *278 * @returns {string} base64-encoded image, potentially resized279 */280helpers.ensureTemplateSize = async function ensureTemplateSize (b64Template, screenWidth, screenHeight) {281 let imgObj = await imageUtil.getJimpImage(b64Template);282 let {width: tplWidth, height: tplHeight} = imgObj.bitmap;283 log.info(`Template image is ${tplWidth}x${tplHeight}. Screen size is ${screenWidth}x${screenHeight}`);284 // if the template fits inside the screen dimensions, we're good285 if (tplWidth <= screenWidth && tplHeight <= screenHeight) {286 return b64Template;287 }288 log.info(`Scaling template image from ${tplWidth}x${tplHeight} to match ` +289 `screen at ${screenWidth}x${screenHeight}`);290 // otherwise, scale it to fit inside the screen dimensions291 imgObj = imgObj.scaleToFit(screenWidth, screenHeight);292 return (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');293};294/**295 * @typedef {Object} Screenshot296 * @property {string} b64Screenshot - base64 based screenshot string297 */298/**299 * @typedef {Object} ScreenshotScale300 * @property {float} xScale - Scale ratio for width301 * @property {float} yScale - Scale ratio for height302 */303/**304 * Get the screenshot image that will be used for find by element, potentially305 * altering it in various ways based on user-requested settings306 *307 * @param {int} screenWidth - width of screen308 * @param {int} screenHeight - height of screen309 *310 * @returns {Screenshot, ?ScreenshotScale} base64-encoded screenshot and ScreenshotScale311 */312helpers.getScreenshotForImageFind = async function getScreenshotForImageFind (screenWidth, screenHeight) {313 if (!this.getScreenshot) {314 throw new Error("This driver does not support the required 'getScreenshot' command");315 }316 let b64Screenshot = await this.getScreenshot();317 // if the user has requested not to correct for aspect or size differences318 // between the screenshot and the screen, just return the screenshot now319 if (!this.settings.getSettings().fixImageFindScreenshotDims) {320 log.info(`Not verifying screenshot dimensions match screen`);321 return {b64Screenshot};322 }323 if (screenWidth < 1 || screenHeight < 1) {324 log.warn(`The retrieved screen size ${screenWidth}x${screenHeight} does ` +325 `not seem to be valid. No changes will be applied to the screenshot`);326 return {b64Screenshot};327 }328 // otherwise, do some verification on the screenshot to make sure it matches329 // the screen size and aspect ratio330 log.info('Verifying screenshot size and aspect ratio');331 let imgObj = await imageUtil.getJimpImage(b64Screenshot);332 let {width: shotWidth, height: shotHeight} = imgObj.bitmap;333 if (shotWidth < 1 || shotHeight < 1) {334 log.warn(`The retrieved screenshot size ${shotWidth}x${shotHeight} does ` +335 `not seem to be valid. No changes will be applied to the screenshot`);336 return {b64Screenshot};337 }338 if (screenWidth === shotWidth && screenHeight === shotHeight) {339 // the height and width of the screenshot and the device screen match, which340 // means we should be safe when doing template matches341 log.info('Screenshot size matched screen size');342 return {b64Screenshot};343 }344 // otherwise, if they don't match, it could spell problems for the accuracy345 // of coordinates returned by the image match algorithm, since we match based346 // on the screenshot coordinates not the device coordinates themselves. There347 // are two potential types of mismatch: aspect ratio mismatch and scale348 // mismatch. We need to detect and fix both349 const scale = {xScale: 1.0, yScale: 1.0};350 const screenAR = screenWidth / screenHeight;351 const shotAR = shotWidth / shotHeight;352 if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {353 log.info(`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +354 `screen aspect ratio '${screenAR}' (${screenWidth}x${screenHeight})`);355 } else {356 log.warn(`When trying to find an element, determined that the screen ` +357 `aspect ratio and screenshot aspect ratio are different. Screen ` +358 `is ${screenWidth}x${screenHeight} whereas screenshot is ` +359 `${shotWidth}x${shotHeight}.`);360 // In the case where the x-scale and y-scale are different, we need to decide361 // which one to respect, otherwise the screenshot and template will end up362 // being resized in a way that changes its aspect ratio (distorts it). For example, let's say:363 // this.getScreenshot(shotWidth, shotHeight) is 540x397,364 // this.getDeviceSize(screenWidth, screenHeight) is 1080x1920.365 // The ratio would then be {xScale: 0.5, yScale: 0.2}.366 // In this case, we must should `yScale: 0.2` as scaleFactor, because367 // if we select the xScale, the height will be bigger than real screenshot size368 // which is used to image comparison by OpenCV as a base image.369 // All of this is primarily useful when the screenshot is a horizontal slice taken out of the370 // screen (for example not including top/bottom nav bars)371 const xScale = (1.0 * shotWidth) / screenWidth;372 const yScale = (1.0 * shotHeight) / screenHeight;373 const scaleFactor = xScale >= yScale ? yScale : xScale;374 log.warn(`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` +375 `screen aspect ratio so that image element coordinates have a ` +376 `greater chance of being correct.`);377 imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor);378 scale.xScale *= scaleFactor;379 scale.yScale *= scaleFactor;380 shotWidth = imgObj.bitmap.width;381 shotHeight = imgObj.bitmap.height;382 }383 // Resize based on the screen dimensions only if both width and height are mismatched384 // since except for that, it might be a situation which is different window rect and385 // screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and386 // `"deviceScreenSize"=>"1080x1920"`387 if (screenWidth !== shotWidth && screenHeight !== shotHeight) {388 log.info(`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +389 `screen at ${screenWidth}x${screenHeight}`);390 imgObj = imgObj.resize(screenWidth, screenHeight);391 scale.xScale *= (1.0 * screenWidth) / shotWidth;392 scale.yScale *= (1.0 * screenHeight) / shotHeight;393 }394 b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');395 return {b64Screenshot, scale};396};397/**398 * @typedef {Object} ImageTemplateSettings399 * @property {boolean} fixImageTemplateScale - fixImageTemplateScale in device-settings400 * @property {float} defaultImageTemplateScale - defaultImageTemplateScale in device-settings401 * @property {boolean} ignoreDefaultImageTemplateScale - Ignore defaultImageTemplateScale if it has true.402 * If b64Template has been scaled to defaultImageTemplateScale or should ignore the scale,403 * this parameter should be true. e.g. click in image-element module404 * @property {float} xScale - Scale ratio for width405 * @property {float} yScale - Scale ratio for height406 */407/**408 * Get a image that will be used for template maching.409 * Returns scaled image if scale ratio is provided.410 *411 *412 * @param {string} b64Template - base64-encoded image used as a template to be413 * matched in the screenshot414 * @param {ImageTemplateSettings} opts - Image template scale related options415 *416 * @returns {string} base64-encoded scaled template screenshot417 */418const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;419helpers.fixImageTemplateScale = async function fixImageTemplateScale (b64Template, opts = {}) {420 if (!opts) {421 return b64Template;422 }423 let {424 fixImageTemplateScale = false,425 defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,426 ignoreDefaultImageTemplateScale = false,427 xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,428 yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE429 } = opts;430 if (ignoreDefaultImageTemplateScale) {431 defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;432 }433 // Default434 if (defaultImageTemplateScale === DEFAULT_TEMPLATE_IMAGE_SCALE && !fixImageTemplateScale) {435 return b64Template;436 }437 // Calculate xScale and yScale Appium should scale438 if (fixImageTemplateScale) {439 xScale *= defaultImageTemplateScale;440 yScale *= defaultImageTemplateScale;441 } else {442 xScale = yScale = 1 * defaultImageTemplateScale;443 }444 // xScale and yScale can be NaN if defaultImageTemplateScale is string, for example445 if (!parseFloat(xScale) || !parseFloat(yScale)) {446 return b64Template;447 }448 // Return if the scale is default, 1, value449 if (Math.round(xScale * FLOAT_PRECISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)450 && Math.round(yScale * FLOAT_PRECISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION))) {451 return b64Template;452 }453 let imgTempObj = await imageUtil.getJimpImage(b64Template);454 let {width: baseTempWidth, height: baseTempHeigh} = imgTempObj.bitmap;455 const scaledWidth = baseTempWidth * xScale;456 const scaledHeight = baseTempHeigh * yScale;457 log.info(`Scaling template image from ${baseTempWidth}x${baseTempHeigh}` +458 ` to ${scaledWidth}x${scaledHeight}`);459 log.info(`The ratio is ${xScale} and ${yScale}`);460 imgTempObj = await imgTempObj.resize(scaledWidth, scaledHeight);461 return (await imgTempObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');462};463Object.assign(extensions, commands, helpers);464export { commands, helpers, IMAGE_STRATEGY, CUSTOM_STRATEGY };...

Full Screen

Full Screen

finder.js

Source:finder.js Github

copy

Full Screen

1import _ from 'lodash';2import LRU from 'lru-cache';3import { imageUtil, util } from '@appium/support';4import { errors } from '@appium/base-driver';5import { ImageElement, DEFAULT_TEMPLATE_IMAGE_SCALE,6 IMAGE_EL_TAP_STRATEGY_W3C } from './image-element';7import { MATCH_TEMPLATE_MODE, compareImages, DEFAULT_MATCH_THRESHOLD } from './compare';8import log from './logger';9const MJSONWP_ELEMENT_KEY = 'ELEMENT';10const W3C_ELEMENT_KEY = util.W3C_WEB_ELEMENT_IDENTIFIER;11const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;12// Used to compare ratio and screen width13// Pixel is basically under 1080 for example. 100K is probably enough fo a while.14const FLOAT_PRECISION = 100000;15const MAX_CACHE_SIZE = 1024 * 1024 * 40; // 40mb16const DEFAULT_SETTINGS = {17 // value between 0 and 1 representing match strength, below which an image18 // element will not be found19 imageMatchThreshold: DEFAULT_MATCH_THRESHOLD,20 // One of possible image matching methods.21 // Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html22 // for more details.23 // TM_CCOEFF_NORMED by default24 imageMatchMethod: '',25 // if the image returned by getScreenshot differs in size or aspect ratio26 // from the screen, attempt to fix it automatically27 fixImageFindScreenshotDims: true,28 // whether Appium should ensure that an image template sent in during image29 // element find should have its size adjusted so the match algorithm will not30 // complain31 fixImageTemplateSize: false,32 // whether Appium should ensure that an image template sent in during image33 // element find should have its scale adjusted to display size so the match34 // algorithm will not complain.35 // e.g. iOS has `width=375, height=667` window rect, but its screenshot is36 // `width=750 × height=1334` pixels. This setting help to adjust the scale37 // if a user use `width=750 × height=1334` pixels's base template image.38 fixImageTemplateScale: false,39 // Users might have scaled template image to reduce their storage size.40 // This setting allows users to scale a template image they send to Appium server41 // so that the Appium server compares the actual scale users originally had.42 // e.g. If a user has an image of 270 x 32 pixels which was originally 1080 x 126 pixels,43 // the user can set {defaultImageTemplateScale: 4.0} to scale the small image44 // to the original one so that Appium can compare it as the original one.45 defaultImageTemplateScale: DEFAULT_TEMPLATE_IMAGE_SCALE,46 // whether Appium should re-check that an image element can be matched47 // against the current screenshot before clicking it48 checkForImageElementStaleness: true,49 // whether before clicking on an image element Appium should re-determine the50 // position of the element on screen51 autoUpdateImageElementPosition: false,52 // which method to use for tapping by coordinate for image elements. the53 // options are 'w3c' or 'mjsonwp'54 imageElementTapStrategy: IMAGE_EL_TAP_STRATEGY_W3C,55 // which method to use to save the matched image area in ImageElement class.56 // It is used for debugging purpose.57 getMatchedImageResult: false,58};59export default class ImageElementFinder {60 constructor (driver, max = MAX_CACHE_SIZE) {61 this.driver = driver;62 this.imgElCache = new LRU({63 max,64 length: (el) => el.template.length,65 });66 }67 setDriver (driver) {68 this.driver = driver;69 }70 registerImageElement (imgEl) {71 this.imgElCache.set(imgEl.id, imgEl);72 const protoKey = this.driver.isW3CProtocol() ? W3C_ELEMENT_KEY : MJSONWP_ELEMENT_KEY;73 return imgEl.asElement(protoKey);74 }75 /**76 * @typedef {Object} FindByImageOptions77 * @property {boolean} [shouldCheckStaleness=false] - whether this call to find an78 * image is merely to check staleness. If so we can bypass a lot of logic79 * @property {boolean} [multiple=false] - Whether we are finding one element or80 * multiple81 * @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we82 * ignore defaultImageTemplateScale. It can be used when you would like to83 * scale b64Template with defaultImageTemplateScale setting.84 */85 /**86 * Find a screen rect represented by an ImageElement corresponding to an image87 * template sent in by the client88 *89 * @param {string} b64Template - base64-encoded image used as a template to be90 * matched in the screenshot91 * @param {FindByImageOptions} - additional options92 *93 * @returns {WebElement} - WebDriver element with a special id prefix94 */95 async findByImage (b64Template, {96 shouldCheckStaleness = false,97 multiple = false,98 ignoreDefaultImageTemplateScale = false,99 }) {100 if (!this.driver) {101 throw new Error(`Can't find without a driver!`);102 }103 const settings = Object.assign({}, DEFAULT_SETTINGS, this.driver.settings.getSettings());104 const {105 imageMatchThreshold: threshold,106 imageMatchMethod,107 fixImageTemplateSize,108 fixImageTemplateScale,109 defaultImageTemplateScale,110 getMatchedImageResult: visualize111 } = settings;112 log.info(`Finding image element with match threshold ${threshold}`);113 if (!this.driver.getWindowSize) {114 throw new Error("This driver does not support the required 'getWindowSize' command");115 }116 const {width: screenWidth, height: screenHeight} = await this.driver.getWindowSize();117 // someone might have sent in a template that's larger than the screen118 // dimensions. If so let's check and cut it down to size since the algorithm119 // will not work unless we do. But because it requires some potentially120 // expensive commands, only do this if the user has requested it in settings.121 if (fixImageTemplateSize) {122 b64Template = await this.ensureTemplateSize(b64Template, screenWidth,123 screenHeight);124 }125 const results = [];126 const condition = async () => {127 try {128 const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);129 b64Template = await this.fixImageTemplateScale(b64Template, {130 defaultImageTemplateScale, ignoreDefaultImageTemplateScale,131 fixImageTemplateScale, ...scale132 });133 const comparisonOpts = {134 threshold,135 visualize,136 multiple,137 };138 if (imageMatchMethod) {139 comparisonOpts.method = imageMatchMethod;140 }141 if (multiple) {142 results.push(...(await compareImages(MATCH_TEMPLATE_MODE,143 b64Screenshot,144 b64Template,145 comparisonOpts)));146 } else {147 results.push(await compareImages(MATCH_TEMPLATE_MODE,148 b64Screenshot,149 b64Template,150 comparisonOpts));151 }152 return true;153 } catch (err) {154 // if compareImages fails, we'll get a specific error, but we should155 // retry, so trap that and just return false to trigger the next round of156 // implicitly waiting. For other errors, throw them to get out of the157 // implicit wait loop158 if (err.message.match(/Cannot find any occurrences/)) {159 return false;160 }161 throw err;162 }163 };164 try {165 await this.driver.implicitWaitForCondition(condition);166 } catch (err) {167 // this `implicitWaitForCondition` method will throw a 'Condition unmet'168 // error if an element is not found eventually. In that case, we will169 // handle the element not found response below. In the case where get some170 // _other_ kind of error, it means something blew up totally apart from the171 // implicit wait timeout. We should not mask that error and instead throw172 // it straightaway173 if (!err.message.match(/Condition unmet/)) {174 throw err;175 }176 }177 if (_.isEmpty(results)) {178 if (multiple) {179 return [];180 }181 throw new errors.NoSuchElementError();182 }183 const elements = results.map(({rect, score, visualization}) => {184 log.info(`Image template matched: ${JSON.stringify(rect)}`);185 return new ImageElement(b64Template, rect, score, visualization, this);186 });187 // if we're just checking staleness, return straightaway so we don't add188 // a new element to the cache. shouldCheckStaleness does not support multiple189 // elements, since it is a purely internal mechanism190 if (shouldCheckStaleness) {191 return elements[0];192 }193 const registeredElements = elements.map((imgEl) => this.registerImageElement(imgEl));194 return multiple ? registeredElements : registeredElements[0];195 }196 /**197 * Ensure that the image template sent in for a find is of a suitable size198 *199 * @param {string} b64Template - base64-encoded image200 * @param {int} screenWidth - width of screen201 * @param {int} screenHeight - height of screen202 *203 * @returns {string} base64-encoded image, potentially resized204 */205 async ensureTemplateSize (b64Template, screenWidth, screenHeight) {206 let imgObj = await imageUtil.getJimpImage(b64Template);207 let {width: tplWidth, height: tplHeight} = imgObj.bitmap;208 log.info(`Template image is ${tplWidth}x${tplHeight}. Screen size is ${screenWidth}x${screenHeight}`);209 // if the template fits inside the screen dimensions, we're good210 if (tplWidth <= screenWidth && tplHeight <= screenHeight) {211 return b64Template;212 }213 log.info(`Scaling template image from ${tplWidth}x${tplHeight} to match ` +214 `screen at ${screenWidth}x${screenHeight}`);215 // otherwise, scale it to fit inside the screen dimensions216 imgObj = imgObj.scaleToFit(screenWidth, screenHeight);217 return (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');218 }219 /**220 * @typedef {Object} Screenshot221 * @property {string} b64Screenshot - base64 based screenshot string222 */223 /**224 * @typedef {Object} ScreenshotScale225 * @property {float} xScale - Scale ratio for width226 * @property {float} yScale - Scale ratio for height227 */228 /**229 * Get the screenshot image that will be used for find by element, potentially230 * altering it in various ways based on user-requested settings231 *232 * @param {int} screenWidth - width of screen233 * @param {int} screenHeight - height of screen234 *235 * @returns {Screenshot, ?ScreenshotScale} base64-encoded screenshot and ScreenshotScale236 */237 async getScreenshotForImageFind (screenWidth, screenHeight) {238 if (!this.driver.getScreenshot) {239 throw new Error("This driver does not support the required 'getScreenshot' command");240 }241 const settings = Object.assign({}, DEFAULT_SETTINGS, this.driver.settings.getSettings());242 const {fixImageFindScreenshotDims} = settings;243 let b64Screenshot = await this.driver.getScreenshot();244 // if the user has requested not to correct for aspect or size differences245 // between the screenshot and the screen, just return the screenshot now246 if (!fixImageFindScreenshotDims) {247 log.info(`Not verifying screenshot dimensions match screen`);248 return {b64Screenshot};249 }250 if (screenWidth < 1 || screenHeight < 1) {251 log.warn(`The retrieved screen size ${screenWidth}x${screenHeight} does ` +252 `not seem to be valid. No changes will be applied to the screenshot`);253 return {b64Screenshot};254 }255 // otherwise, do some verification on the screenshot to make sure it matches256 // the screen size and aspect ratio257 log.info('Verifying screenshot size and aspect ratio');258 let imgObj = await imageUtil.getJimpImage(b64Screenshot);259 let {width: shotWidth, height: shotHeight} = imgObj.bitmap;260 if (shotWidth < 1 || shotHeight < 1) {261 log.warn(`The retrieved screenshot size ${shotWidth}x${shotHeight} does ` +262 `not seem to be valid. No changes will be applied to the screenshot`);263 return {b64Screenshot};264 }265 if (screenWidth === shotWidth && screenHeight === shotHeight) {266 // the height and width of the screenshot and the device screen match, which267 // means we should be safe when doing template matches268 log.info('Screenshot size matched screen size');269 return {b64Screenshot};270 }271 // otherwise, if they don't match, it could spell problems for the accuracy272 // of coordinates returned by the image match algorithm, since we match based273 // on the screenshot coordinates not the device coordinates themselves. There274 // are two potential types of mismatch: aspect ratio mismatch and scale275 // mismatch. We need to detect and fix both276 const scale = {xScale: 1.0, yScale: 1.0};277 const screenAR = screenWidth / screenHeight;278 const shotAR = shotWidth / shotHeight;279 if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {280 log.info(`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +281 `screen aspect ratio '${screenAR}' (${screenWidth}x${screenHeight})`);282 } else {283 log.warn(`When trying to find an element, determined that the screen ` +284 `aspect ratio and screenshot aspect ratio are different. Screen ` +285 `is ${screenWidth}x${screenHeight} whereas screenshot is ` +286 `${shotWidth}x${shotHeight}.`);287 // In the case where the x-scale and y-scale are different, we need to decide288 // which one to respect, otherwise the screenshot and template will end up289 // being resized in a way that changes its aspect ratio (distorts it). For example, let's say:290 // this.getScreenshot(shotWidth, shotHeight) is 540x397,291 // this.getDeviceSize(screenWidth, screenHeight) is 1080x1920.292 // The ratio would then be {xScale: 0.5, yScale: 0.2}.293 // In this case, we must should `yScale: 0.2` as scaleFactor, because294 // if we select the xScale, the height will be bigger than real screenshot size295 // which is used to image comparison by OpenCV as a base image.296 // All of this is primarily useful when the screenshot is a horizontal slice taken out of the297 // screen (for example not including top/bottom nav bars)298 const xScale = (1.0 * shotWidth) / screenWidth;299 const yScale = (1.0 * shotHeight) / screenHeight;300 const scaleFactor = xScale >= yScale ? yScale : xScale;301 log.warn(`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` +302 `screen aspect ratio so that image element coordinates have a ` +303 `greater chance of being correct.`);304 imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor);305 scale.xScale *= scaleFactor;306 scale.yScale *= scaleFactor;307 shotWidth = imgObj.bitmap.width;308 shotHeight = imgObj.bitmap.height;309 }310 // Resize based on the screen dimensions only if both width and height are mismatched311 // since except for that, it might be a situation which is different window rect and312 // screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and313 // `"deviceScreenSize"=>"1080x1920"`314 if (screenWidth !== shotWidth && screenHeight !== shotHeight) {315 log.info(`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +316 `screen at ${screenWidth}x${screenHeight}`);317 imgObj = imgObj.resize(screenWidth, screenHeight);318 scale.xScale *= (1.0 * screenWidth) / shotWidth;319 scale.yScale *= (1.0 * screenHeight) / shotHeight;320 }321 b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');322 return {b64Screenshot, scale};323 }324 /**325 * @typedef {Object} ImageTemplateSettings326 * @property {boolean} fixImageTemplateScale - fixImageTemplateScale in device-settings327 * @property {float} defaultImageTemplateScale - defaultImageTemplateScale in device-settings328 * @property {boolean} ignoreDefaultImageTemplateScale - Ignore defaultImageTemplateScale if it has true.329 * If b64Template has been scaled to defaultImageTemplateScale or should ignore the scale,330 * this parameter should be true. e.g. click in image-element module331 * @property {float} xScale - Scale ratio for width332 * @property {float} yScale - Scale ratio for height333 */334 /**335 * Get a image that will be used for template maching.336 * Returns scaled image if scale ratio is provided.337 *338 *339 * @param {string} b64Template - base64-encoded image used as a template to be340 * matched in the screenshot341 * @param {ImageTemplateSettings} opts - Image template scale related options342 *343 * @returns {string} base64-encoded scaled template screenshot344 */345 async fixImageTemplateScale (b64Template, opts = {}) {346 if (!opts) {347 return b64Template;348 }349 let {350 fixImageTemplateScale = false,351 defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,352 ignoreDefaultImageTemplateScale = false,353 xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,354 yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE355 } = opts;356 if (ignoreDefaultImageTemplateScale) {357 defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;358 }359 // Default360 if (defaultImageTemplateScale === DEFAULT_TEMPLATE_IMAGE_SCALE && !fixImageTemplateScale) {361 return b64Template;362 }363 // Calculate xScale and yScale Appium should scale364 if (fixImageTemplateScale) {365 xScale *= defaultImageTemplateScale;366 yScale *= defaultImageTemplateScale;367 } else {368 xScale = yScale = 1 * defaultImageTemplateScale;369 }370 // xScale and yScale can be NaN if defaultImageTemplateScale is string, for example371 if (!parseFloat(xScale) || !parseFloat(yScale)) {372 return b64Template;373 }374 // Return if the scale is default, 1, value375 if (Math.round(xScale * FLOAT_PRECISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)376 && Math.round(yScale * FLOAT_PRECISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION))) {377 return b64Template;378 }379 let imgTempObj = await imageUtil.getJimpImage(b64Template);380 let {width: baseTempWidth, height: baseTempHeigh} = imgTempObj.bitmap;381 const scaledWidth = baseTempWidth * xScale;382 const scaledHeight = baseTempHeigh * yScale;383 log.info(`Scaling template image from ${baseTempWidth}x${baseTempHeigh}` +384 ` to ${scaledWidth}x${scaledHeight}`);385 log.info(`The ratio is ${xScale} and ${yScale}`);386 imgTempObj = await imgTempObj.resize(scaledWidth, scaledHeight);387 return (await imgTempObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');388 }389}...

Full Screen

Full Screen

finder-specs.js

Source:finder-specs.js Github

copy

Full Screen

1import _ from 'lodash';2import { imageUtil } from '@appium/support';3import BaseDriver from '@appium/base-driver';4import ImageElementPlugin, { IMAGE_STRATEGY } from '../../index';5import ImageElementFinder from '../../lib/finder';6import ImageElement from '../../lib/image-element';7import sinon from 'sinon';8import { TINY_PNG, TINY_PNG_DIMS } from '../fixtures';9import chai from 'chai';10import chaiAsPromised from 'chai-as-promised';11const compareModule = require('../../lib/compare');12chai.use(chaiAsPromised);13const should = chai.should();14const plugin = new ImageElementPlugin();15class PluginDriver extends BaseDriver {16 async getWindowSize () {}17 async getScreenshot () {}18 findElement (strategy, selector) {19 return plugin.findElement(_.noop, this, strategy, selector);20 }21 findElements (strategy, selector) {22 return plugin.findElements(_.noop, this, strategy, selector);23 }24}25describe('finding elements by image', function () {26 describe('findElement', function () {27 it('should use a different special method to find element by image', async function () {28 const d = new PluginDriver();29 sinon.stub(plugin.finder, 'findByImage').returns(true);30 sinon.stub(d, 'findElOrElsWithProcessing').returns(false);31 await d.findElement(IMAGE_STRATEGY, 'foo').should.eventually.be.true;32 await d.findElements(IMAGE_STRATEGY, 'foo').should.eventually.be.true;33 });34 it('should not be able to find image element from any other element', async function () {35 const d = new PluginDriver();36 await d.findElementFromElement(IMAGE_STRATEGY, 'foo', 'elId')37 .should.eventually.be.rejectedWith(/Locator Strategy.+is not supported/);38 await d.findElementsFromElement(IMAGE_STRATEGY, 'foo', 'elId')39 .should.eventually.be.rejectedWith(/Locator Strategy.+is not supported/);40 });41 });42 describe('findByImage', function () {43 const rect = {x: 10, y: 20, width: 30, height: 40};44 const score = 0.9;45 const size = {width: 100, height: 200};46 const screenshot = 'iVBORfoo';47 const template = 'iVBORbar';48 let compareStub;49 let d = new PluginDriver();50 let f = new ImageElementFinder(d);51 function basicStub (driver, finder) {52 const sizeStub = sinon.stub(driver, 'getWindowSize').returns(size);53 const screenStub = sinon.stub(finder, 'getScreenshotForImageFind').returns(screenshot);54 return {sizeStub, screenStub};55 }56 function basicImgElVerify (imgElProto, finder) {57 const imgElId = imgElProto.ELEMENT;58 finder.imgElCache.has(imgElId).should.be.true;59 const imgEl = finder.imgElCache.get(imgElId);60 (imgEl instanceof ImageElement).should.be.true;61 imgEl.rect.should.eql(rect);62 imgEl.score.should.eql(score);63 return imgEl;64 }65 beforeEach(function () {66 compareStub = sinon.stub(compareModule, 'compareImages').returns({rect, score});67 d = new PluginDriver();68 f = new ImageElementFinder(d);69 basicStub(d, f);70 });71 afterEach(function () {72 compareStub.restore();73 });74 it('should find an image element happypath', async function () {75 const imgElProto = await f.findByImage(template, {multiple: false});76 basicImgElVerify(imgElProto, f);77 });78 it('should find image elements happypath', async function () {79 compareStub.restore();80 compareStub = sinon.stub(compareModule, 'compareImages').returns([{rect, score}]);81 const els = await f.findByImage(template, {multiple: true});82 els.should.have.length(1);83 basicImgElVerify(els[0], f);84 });85 it('should fail if driver does not support getWindowSize', async function () {86 d.getWindowSize = null;87 await f.findByImage(template, {multiple: false})88 .should.eventually.be.rejectedWith(/driver does not support/);89 });90 it('should fix template size if requested', async function () {91 const newTemplate = 'iVBORbaz';92 await d.settings.update({fixImageTemplateSize: true});93 sinon.stub(f, 'ensureTemplateSize').returns(newTemplate);94 const imgElProto = await f.findByImage(template, {multiple: false});95 const imgEl = basicImgElVerify(imgElProto, f);96 imgEl.template.should.eql(newTemplate);97 _.last(compareStub.args)[2].should.eql(newTemplate);98 });99 it('should fix template size scale if requested', async function () {100 const newTemplate = 'iVBORbaz';101 await d.settings.update({fixImageTemplateScale: true});102 sinon.stub(f, 'fixImageTemplateScale').returns(newTemplate);103 const imgElProto = await f.findByImage(template, {multiple: false});104 const imgEl = basicImgElVerify(imgElProto, f);105 imgEl.template.should.eql(newTemplate);106 _.last(compareStub.args)[2].should.eql(newTemplate);107 });108 it('should not fix template size scale if it is not requested', async function () {109 const newTemplate = 'iVBORbaz';110 await d.settings.update({});111 sinon.stub(f, 'fixImageTemplateScale').returns(newTemplate);112 f.fixImageTemplateScale.callCount.should.eql(0);113 });114 it('should throw an error if template match fails', async function () {115 compareStub.throws(new Error('Cannot find any occurrences'));116 await f.findByImage(template, {multiple: false})117 .should.eventually.be.rejectedWith(/element could not be located/);118 });119 it('should return empty array for multiple elements if template match fails', async function () {120 compareStub.throws(new Error('Cannot find any occurrences'));121 await f.findByImage(template, {multiple: true}).should.eventually.eql([]);122 });123 it('should respect implicit wait', async function () {124 d.setImplicitWait(10);125 compareStub.resetHistory();126 compareStub.onCall(0).throws(new Error('Cannot find any occurrences'));127 compareStub.returns({rect, score});128 const imgElProto = await f.findByImage(template, {multiple: false});129 basicImgElVerify(imgElProto, f);130 compareStub.callCount.should.eql(2);131 });132 it('should not add element to cache and return it directly when checking staleness', async function () {133 const imgEl = await f.findByImage(template, {multiple: false, shouldCheckStaleness: true});134 (imgEl instanceof ImageElement).should.be.true;135 f.imgElCache.has(imgEl.id).should.be.false;136 imgEl.rect.should.eql(rect);137 });138 });139 describe('fixImageTemplateScale', function () {140 const d = new PluginDriver();141 const f = new ImageElementFinder(d);142 const basicTemplate = 'iVBORbaz';143 it('should not fix template size scale if no scale value', async function () {144 await f.fixImageTemplateScale(basicTemplate, {fixImageTemplateScale: true})145 .should.eventually.eql(basicTemplate);146 });147 it('should not fix template size scale if it is null', async function () {148 await f.fixImageTemplateScale(basicTemplate, null)149 .should.eventually.eql(basicTemplate);150 });151 it('should not fix template size scale if it is not number', async function () {152 await f.fixImageTemplateScale(basicTemplate, 'wrong-scale')153 .should.eventually.eql(basicTemplate);154 });155 it('should fix template size scale', async function () {156 const actual = 'iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAYAAADgzO9IAAAAWElEQVR4AU3BQRWAQAhAwa/PGBsEgrC16AFBKEIPXW7OXO+Rmey9iQjMjHFzrLUwM7qbqmLcHKpKRFBVuDvj4agq3B1VRUQYT2bS3QwRQVUZF/CaGRHB3wc1vSZbHO5+BgAAAABJRU5ErkJggg==';157 await f.fixImageTemplateScale(TINY_PNG, {158 fixImageTemplateScale: true, xScale: 1.5, yScale: 1.5159 }).should.eventually.eql(actual);160 });161 it('should not fix template size scale because of fixImageTemplateScale being false', async function () {162 await f.fixImageTemplateScale(TINY_PNG, {163 fixImageTemplateScale: false, xScale: 1.5, yScale: 1.5164 }).should.eventually.eql(TINY_PNG);165 });166 it('should fix template size scale with default scale', async function () {167 const actual = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABwUlEQVR4AaXBPUsrQQCG0SeX+cBdkTjwTpG1NPgLpjY/fW1stt4UYmm2cJqwMCsaw70uJJ3CBc9Z/P3Cl+12S9u2tG1L27bEGLm/v2ez2bDZbJDEd/7wS4YT7z3X19fc3Nxwd3dHXdd47xnHkefnZ8ZxpKoq6rqmqiqMMcwMJ1VV0TQN0zThnOPj44O6rsk503UdkmiahqZpWK1WGGOYGU7quqZpGqy1SCLnTM6Z19dXcs5IYpomrLVI4uLigpnhpKoqVqsVkjgcDjw9PdF1HTlnuq5DEs45JHE4HDgznByPR97e3pimiVIK4zhyPB7x3hNCIITA5eUl3nsWiwVnhpNSCsMwsNvtGIaB/X5PKQVJpJSQxHq9RhLOOc4MJ9M0sdvt2G639H3PTBIxRiQhCUnEGLHWcmY4KaUwDAN93/P4+MhyuSSlhCRSSkjCOYe1FmstZ6bve2YvLy/s93tmy+USSUhCEpIIIfAd8/DwwOz9/Z1SCpJIKSGJ9XqNJJxz/MS0bcvs6uoKScQYkYQkJBFjxFrLT0zbtsxub29JKSGJlBKScM5hrcVay09MzplZjJHPz0+894QQCCHwP/7wS/8A4e6nAg+R8LwAAAAASUVORK5CYII=';168 await f.fixImageTemplateScale(TINY_PNG, {169 defaultImageTemplateScale: 4.0170 }).should.eventually.eql(actual);171 });172 it('should fix template size scale with default scale and image scale', async function () {173 const actual = 'iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAACaUlEQVR4AbXBMWvrWBSF0c9BsFPtW91UR1U6+///FKlKKt8qqnyqnMozggkI8xgMj6x1uv+L/6zryrIsrOvKsiys68qyLFwuF87nM5fLhfP5zOVy4Xw+84wXftkLv2ziQBK26b0TEVQVu4jANrvM5Hq9spOEJCQhCUlI4mjiQBK26b1TVewkYRvb7DKTMQaZiW1s01rDNraRxNHEgSRaa1QVO0m01jjKTDKTXe+d3jtVxU4SjyYOJGGbnSRs03snM8lMMpPb7UZmkplEBFXFThK2eTRxIAnbSMI2VcX39zdjDMYYZCaZyRiDMQZVxU4StqkqHk0cSEISf5KZ7DKTMQbLsrCTRGuN3jtVxaOJg6qiqqgqqoqqoqoYY5CZ7GwTEdzvd97f34kIeu/YRhKPJg6qiswkM7ndbmQmmUlmkpnsbBMR2CYimOeZ3ju2kcSjiYOqIjP5+vpi2za2bWPbNo5aa7TW2PXe6b3Te6e1hiQeTRxUFbfbjW3bGGNwvV4ZY2Ab27TWsI1tbGMb27TWsI0kHk0cVBWZybZtXK9XPj8/+fj4YJ5nIoLWGraJCOZ5RhKSkIQkJPFo4qCqyEy2bWOMwefnJ+u6cjqdsM3ONvM8cz6feca0ris/rtcrmcnONhHB/X7n/f2diKD3jm0k8axpWRZ+ZCaZyc42EYFtIoJ5num9YxtJPGta15U/sY1tdm9vb/Te6b1jG0k8a1qWhR+2sU1rjdYatrGNbWxjm9YaknjWtK4rPyKCiKC1hm0igojg9fUVSUhCEpJ41rQsC0e22dkmIrhcLvyNF/7H6XTib73wy174Zf8AJEsePtlPj10AAAAASUVORK5CYII=';174 await f.fixImageTemplateScale(TINY_PNG, {175 defaultImageTemplateScale: 4.0,176 fixImageTemplateScale: true,177 xScale: 1.5, yScale: 1.5178 }).should.eventually.eql(actual);179 });180 it('should not fix template size scale with default scale and image scale', async function () {181 const actual = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABwUlEQVR4AaXBPUsrQQCG0SeX+cBdkTjwTpG1NPgLpjY/fW1stt4UYmm2cJqwMCsaw70uJJ3CBc9Z/P3Cl+12S9u2tG1L27bEGLm/v2ez2bDZbJDEd/7wS4YT7z3X19fc3Nxwd3dHXdd47xnHkefnZ8ZxpKoq6rqmqiqMMcwMJ1VV0TQN0zThnOPj44O6rsk503UdkmiahqZpWK1WGGOYGU7quqZpGqy1SCLnTM6Z19dXcs5IYpomrLVI4uLigpnhpKoqVqsVkjgcDjw9PdF1HTlnuq5DEs45JHE4HDgznByPR97e3pimiVIK4zhyPB7x3hNCIITA5eUl3nsWiwVnhpNSCsMwsNvtGIaB/X5PKQVJpJSQxHq9RhLOOc4MJ9M0sdvt2G639H3PTBIxRiQhCUnEGLHWcmY4KaUwDAN93/P4+MhyuSSlhCRSSkjCOYe1FmstZ6bve2YvLy/s93tmy+USSUhCEpIIIfAd8/DwwOz9/Z1SCpJIKSGJ9XqNJJxz/MS0bcvs6uoKScQYkYQkJBFjxFrLT0zbtsxub29JKSGJlBKScM5hrcVay09MzplZjJHPz0+894QQCCHwP/7wS/8A4e6nAg+R8LwAAAAASUVORK5CYII=';182 await f.fixImageTemplateScale(TINY_PNG, {183 defaultImageTemplateScale: 4.0,184 fixImageTemplateScale: false,185 xScale: 1.5, yScale: 1.5186 }).should.eventually.eql(actual);187 });188 it('should not fix template size scale because of ignoreDefaultImageTemplateScale', async function () {189 await f.fixImageTemplateScale(TINY_PNG, {190 defaultImageTemplateScale: 4.0,191 ignoreDefaultImageTemplateScale: true,192 }).should.eventually.eql(TINY_PNG);193 });194 it('should ignore defaultImageTemplateScale to fix template size scale because of ignoreDefaultImageTemplateScale', async function () {195 const actual = 'iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAYAAADgzO9IAAAAWElEQVR4AU3BQRWAQAhAwa/PGBsEgrC16AFBKEIPXW7OXO+Rmey9iQjMjHFzrLUwM7qbqmLcHKpKRFBVuDvj4agq3B1VRUQYT2bS3QwRQVUZF/CaGRHB3wc1vSZbHO5+BgAAAABJRU5ErkJggg==';196 await f.fixImageTemplateScale(TINY_PNG, {197 defaultImageTemplateScale: 4.0,198 ignoreDefaultImageTemplateScale: true,199 fixImageTemplateScale: true,200 xScale: 1.5, yScale: 1.5201 }).should.eventually.eql(actual);202 });203 });204 describe('ensureTemplateSize', function () {205 const d = new PluginDriver();206 const f = new ImageElementFinder(d);207 it('should not resize the template if it is smaller than the screen', async function () {208 const screen = TINY_PNG_DIMS.map((n) => n * 2);209 await f.ensureTemplateSize(TINY_PNG, ...screen)210 .should.eventually.eql(TINY_PNG);211 });212 it('should not resize the template if it is the same size as the screen', async function () {213 await f.ensureTemplateSize(TINY_PNG, ...TINY_PNG_DIMS)214 .should.eventually.eql(TINY_PNG);215 });216 it('should resize the template if it is bigger than the screen', async function () {217 const screen = TINY_PNG_DIMS.map((n) => n / 2);218 const newTemplate = await f.ensureTemplateSize(TINY_PNG, ...screen);219 newTemplate.should.not.eql(TINY_PNG);220 newTemplate.length.should.be.below(TINY_PNG.length);221 });222 });223 describe('getScreenshotForImageFind', function () {224 let d;225 let f;226 beforeEach(function () {227 d = new PluginDriver();228 f = new ImageElementFinder(d);229 sinon.stub(d, 'getScreenshot').returns(TINY_PNG);230 });231 it('should fail if driver does not support getScreenshot', async function () {232 const d = new BaseDriver();233 const f = new ImageElementFinder(d);234 await f.getScreenshotForImageFind()235 .should.eventually.be.rejectedWith(/driver does not support/);236 });237 it('should not adjust or verify screenshot if asked not to by settings', async function () {238 await d.settings.update({fixImageFindScreenshotDims: false});239 const screen = TINY_PNG_DIMS.map((n) => n + 1);240 const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);241 b64Screenshot.should.eql(TINY_PNG);242 should.equal(scale, undefined);243 });244 it('should return screenshot without adjustment if it matches screen size', async function () {245 const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...TINY_PNG_DIMS);246 b64Screenshot.should.eql(TINY_PNG);247 should.equal(scale, undefined);248 });249 it('should return scaled screenshot with same aspect ratio if matching screen aspect ratio', async function () {250 const screen = TINY_PNG_DIMS.map((n) => n * 1.5);251 const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);252 b64Screenshot.should.not.eql(TINY_PNG);253 const screenshotObj = await imageUtil.getJimpImage(b64Screenshot);254 screenshotObj.bitmap.width.should.eql(screen[0]);255 screenshotObj.bitmap.height.should.eql(screen[1]);256 scale.should.eql({ xScale: 1.5, yScale: 1.5 });257 });258 it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio', async function () {259 // try first with portrait screen, screen = 8 x 12260 let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];261 let expectedScale = { xScale: 2.67, yScale: 4 };262 const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);263 b64Screenshot.should.not.eql(TINY_PNG);264 let screenshotObj = await imageUtil.getJimpImage(b64Screenshot);265 screenshotObj.bitmap.width.should.eql(screen[0]);266 screenshotObj.bitmap.height.should.eql(screen[1]);267 scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());268 scale.yScale.should.eql(expectedScale.yScale);269 // then with landscape screen, screen = 12 x 8270 screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];271 expectedScale = { xScale: 4, yScale: 2.67 };272 const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(...screen);273 newScreen.should.not.eql(TINY_PNG);274 screenshotObj = await imageUtil.getJimpImage(newScreen);275 screenshotObj.bitmap.width.should.eql(screen[0]);276 screenshotObj.bitmap.height.should.eql(screen[1]);277 newScale.xScale.should.eql(expectedScale.xScale);278 newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());279 });280 it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio with fixImageTemplateScale', async function () {281 // try first with portrait screen, screen = 8 x 12282 let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];283 let expectedScale = { xScale: 2.67, yScale: 4 };284 const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);285 b64Screenshot.should.not.eql(TINY_PNG);286 let screenshotObj = await imageUtil.getJimpImage(b64Screenshot);287 screenshotObj.bitmap.width.should.eql(screen[0]);288 screenshotObj.bitmap.height.should.eql(screen[1]);289 scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());290 scale.yScale.should.eql(expectedScale.yScale);291 // 8 x 12 stretched TINY_PNG292 await f.fixImageTemplateScale(b64Screenshot, {fixImageTemplateScale: true, scale})293 .should.eventually.eql('iVBORw0KGgoAAAANSUhEUgAAAAgAAAAMCAYAAABfnvydAAAAJ0lEQVR4AYXBAQEAIACDMKR/p0fTBrKdbZcPCRIkSJAgQYIECRIkPAzBA1TpeNwZAAAAAElFTkSuQmCC');294 // then with landscape screen, screen = 12 x 8295 screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];296 expectedScale = { xScale: 4, yScale: 2.67 };297 const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(...screen);298 newScreen.should.not.eql(TINY_PNG);299 screenshotObj = await imageUtil.getJimpImage(newScreen);300 screenshotObj.bitmap.width.should.eql(screen[0]);301 screenshotObj.bitmap.height.should.eql(screen[1]);302 newScale.xScale.should.eql(expectedScale.xScale);303 newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());304 // 12 x 8 stretched TINY_PNG305 await f.fixImageTemplateScale(newScreen, {fixImageTemplateScale: true, scale})306 .should.eventually.eql('iVBORw0KGgoAAAANSUhEUgAAAAwAAAAICAYAAADN5B7xAAAAI0lEQVR4AZXBAQEAMAyDMI5/T5W2ayB5245AIokkkkgiiST6+W4DTLyo5PUAAAAASUVORK5CYII=');307 });308 });...

Full Screen

Full Screen

Using AI Code Generation

copy

Full Screen

1var webdriver = require('selenium-webdriver'),2 By = webdriver.By;3var driver = new webdriver.Builder()4 .forBrowser('chrome')5 .build();6driver.findElement(By.name('q')).sendKeys('webdriver');7driver.findElement(By.name('btnG')).click();8driver.sleep(5000).then(function() {9 driver.getScreenshotForImageFind().then(function(image) {10 console.log(image);11 });12});13driver.quit();14driver.getScreenshot().then(function(image) {15 console.log(image);16});17driver.getScreenshot().then(function(image) {18 console.log(image);19});20driver.getScreenshot().then(function(image) {21 console.log(image);22});23driver.getScreenshot().then(function(image) {24 console.log(image);25});26driver.getScreenshot().then(function(image) {27 console.log(image);28});

Full Screen

Using AI Code Generation

copy

Full Screen

1var webdriverio = require('webdriverio');2var options = {3 desiredCapabilities: {4 }5};6var driver = webdriverio.remote(options);7 .init()8 .then(function() {9 return driver.getScreenshotForImageFind();10 })11 .then(function(screenshot) {12 var fs = require('fs');13 var stream = fs.createWriteStream("/Users/username/Downloads/screenshot.png");14 stream.write(new Buffer(screenshot, "base64"));15 stream.end();16 })17 .then(function() {18 return driver.waitForVisible('android=new UiSelector().text("Controls")', 10000);19 })20 .then(function() {21 return driver.click('android=new UiSelector().text("Controls")');22 })23 .then(function() {24 return driver.waitForVisible('android=new UiSelector().text("2. Dark Theme")', 10000);25 })26 .then(function() {27 return driver.click('android=new UiSelector().text("2. Dark Theme")');28 })29 .then(function() {30 return driver.waitForVisible('android=new UiSelector().text("Checkbox1")', 10000);31 })32 .then(function() {33 return driver.click('android=new UiSelector().text("Checkbox1")');34 })35 .then(function() {36 return driver.waitForVisible('android=new UiSelector().text("Checkbox2")', 10000);37 })38 .then(function() {39 return driver.click('android=new UiSelector().text("Checkbox2")');40 })41 .then(function() {42 return driver.waitForVisible('android=new UiSelector().text("Checkbox3")', 10000);43 })44 .then(function() {45 return driver.click('android=new UiSelector().text("Checkbox3")');46 })47 .then(function() {48 return driver.waitForVisible('android=new UiSelector().text("Radio0")', 10000);49 })50 .then(function() {

Full Screen

Using AI Code Generation

copy

Full Screen

1var d = driver.getDriver();2var image = d.getScreenshotForImageFind();3d.imageFind(image, "test.png", function(err, res) {4 console.log(res);5});6var d = driver.getDriver();7d.imageFind("test.png", function(err, res) {8 console.log(res);9});10var d = driver.getDriver();11d.imageFind("test.png", 0.5, function(err, res) {12 console.log(res);13});14var d = driver.getDriver();15d.imageFind("test.png", 0.5, 0.5, function(err, res) {16 console.log(res);17});18var d = driver.getDriver();19d.imageFind("test.png", 0.5, 0.5, 0.5, function(err, res) {20 console.log(res);21});22var d = driver.getDriver();23d.imageFind("test.png", 0.5, 0.5, 0.5, 0.5, function(err, res) {24 console.log(res);25});26var d = driver.getDriver();27d.imageFind("test.png", 0.5, 0.5, 0.5, 0.5, 0.5, function(err, res) {28 console.log(res);29});30var d = driver.getDriver();31d.imageFind("test.png", 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, function(err, res) {32 console.log(res);33});34var d = driver.getDriver();35d.imageFind("test.png

Full Screen

Using AI Code Generation

copy

Full Screen

1const d = driver.getDriver()2const screenshot = d.getScreenshotForImageFind()3const img = new Buffer(screenshot, 'base64')4const base64Image = img.toString('base64')5const fs = require('fs')6const resemble = require('node-resemble-js')7const options = {8 output: {9 errorColor: {10 },11 },12}13const image1 = fs.readFileSync('./images/image1.png')14const image2 = fs.readFileSync('./images/image2.png')15resemble(image1).compareTo(image2).ignoreNothing().onComplete(function(data) {16 fs.writeFileSync('./diff.png', data.getBuffer())17})18const { AppiumService } = require('./services/appium.service')19const { ImageComparisonService } = require('./services/image-comparison.service')20const { ImageComparisonUtils } = require('./utils/image-comparison.utils')21exports.config = {22 {23 'goog:chromeOptions': {24 }25 }26 mochaOpts: {27 },28 before: function(capabilities, specs) {29 global.should = chai.should()30 },31 after: function(result, capabilities, specs) {32 }33}34const { AppiumDriver } = require('./appium.driver')35const { ImageComparisonService }

Full Screen

Using AI Code Generation

copy

Full Screen

1'use strict';2const wd = require('wd');3const chai = require('chai');4const chaiAsPromised = require('chai-as-promised');5chai.use(chaiAsPromised);6const assert = chai.assert;7const should = chai.should();8const expect = chai.expect;9const path = require('path');10const fs = require('fs');11const { exec } = require('child_process');12const serverConfig = {13};14const desired = {15 app: path.resolve(__dirname, '../apps/ApiDemos-debug.apk')16};17const driver = wd.promiseChainRemote(serverConfig);18describe('Android', function () {19 this.timeout(300000);20 before(async function () {21 await driver.init(desired);22 await driver.sleep(3000);23 });24 after(async function () {25 await driver.quit();26 });27 it('should find an element', async function () {28 let screenshot = await driver.getScreenshotForImageFind();29 fs.writeFileSync('screenshot.png', screenshot, 'base64');30 let el = await driver.elementByImage('screenshot.png');31 await driver.clickElement(el);32 await driver.back();33 });34});

Full Screen

Using AI Code Generation

copy

Full Screen

1var webdriverio = require('webdriverio');2var assert = require('assert');3var options = {4 desiredCapabilities: {5 }6};7var client = webdriverio.remote(options);8 .init()9 .saveScreenshot('screen.png')10 .then(function() {11 return client.getScreenshotForImageFind('screen.png', 'ApiDemos.png', 0.8, 0.5);12 })13 .then(function(res) {14 console.log(res);15 assert(res);16 })17 .end();

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Appium Base Driver automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful