How to use d.getScreenshotForImageFind method in Appium Base Driver

Best JavaScript code snippet using appium-base-driver

Run Appium Base Driver automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

finder-specs.js

Source: finder-specs.js Github

copy
1import _ from 'lodash';
2import { imageUtil } from '@appium/support';
3import BaseDriver from '@appium/base-driver';
4import ImageElementPlugin, { IMAGE_STRATEGY } from '../../index';
5import ImageElementFinder from '../../lib/finder';
6import ImageElement from '../../lib/image-element';
7import sinon from 'sinon';
8import { TINY_PNG, TINY_PNG_DIMS } from '../fixtures';
9import chai from 'chai';
10import chaiAsPromised from 'chai-as-promised';
11
12const compareModule = require('../../lib/compare');
13chai.use(chaiAsPromised);
14const should = chai.should();
15
16const plugin = new ImageElementPlugin();
17
18class PluginDriver extends BaseDriver {
19  async getWindowSize () {}
20  async getScreenshot () {}
21  findElement (strategy, selector) {
22    return plugin.findElement(_.noop, this, strategy, selector);
23  }
24  findElements (strategy, selector) {
25    return plugin.findElements(_.noop, this, strategy, selector);
26  }
27
28}
29
30describe('finding elements by image', function () {
31  describe('findElement', function () {
32    it('should use a different special method to find element by image', async function () {
33      const d = new PluginDriver();
34      sinon.stub(plugin.finder, 'findByImage').returns(true);
35      sinon.stub(d, 'findElOrElsWithProcessing').returns(false);
36      await d.findElement(IMAGE_STRATEGY, 'foo').should.eventually.be.true;
37      await d.findElements(IMAGE_STRATEGY, 'foo').should.eventually.be.true;
38    });
39    it('should not be able to find image element from any other element', async function () {
40      const d = new PluginDriver();
41      await d.findElementFromElement(IMAGE_STRATEGY, 'foo', 'elId')
42        .should.eventually.be.rejectedWith(/Locator Strategy.+is not supported/);
43      await d.findElementsFromElement(IMAGE_STRATEGY, 'foo', 'elId')
44        .should.eventually.be.rejectedWith(/Locator Strategy.+is not supported/);
45    });
46  });
47
48  describe('findByImage', function () {
49    const rect = {x: 10, y: 20, width: 30, height: 40};
50    const score = 0.9;
51    const size = {width: 100, height: 200};
52    const screenshot = 'iVBORfoo';
53    const template = 'iVBORbar';
54    let compareStub;
55    let d = new PluginDriver();
56    let f = new ImageElementFinder(d);
57
58    function basicStub (driver, finder) {
59      const sizeStub = sinon.stub(driver, 'getWindowSize').returns(size);
60      const screenStub = sinon.stub(finder, 'getScreenshotForImageFind').returns(screenshot);
61      return {sizeStub, screenStub};
62    }
63
64    function basicImgElVerify (imgElProto, finder) {
65      const imgElId = imgElProto.ELEMENT;
66      finder.imgElCache.has(imgElId).should.be.true;
67      const imgEl = finder.imgElCache.get(imgElId);
68      (imgEl instanceof ImageElement).should.be.true;
69      imgEl.rect.should.eql(rect);
70      imgEl.score.should.eql(score);
71      return imgEl;
72    }
73
74    beforeEach(function () {
75      compareStub = sinon.stub(compareModule, 'compareImages').returns({rect, score});
76      d = new PluginDriver();
77      f = new ImageElementFinder(d);
78      basicStub(d, f);
79    });
80
81    afterEach(function () {
82      compareStub.restore();
83    });
84
85    it('should find an image element happypath', async function () {
86      const imgElProto = await f.findByImage(template, {multiple: false});
87      basicImgElVerify(imgElProto, f);
88    });
89    it('should find image elements happypath', async function () {
90      compareStub.restore();
91      compareStub = sinon.stub(compareModule, 'compareImages').returns([{rect, score}]);
92      const els = await f.findByImage(template, {multiple: true});
93      els.should.have.length(1);
94      basicImgElVerify(els[0], f);
95    });
96    it('should fail if driver does not support getWindowSize', async function () {
97      d.getWindowSize = null;
98      await f.findByImage(template, {multiple: false})
99        .should.eventually.be.rejectedWith(/driver does not support/);
100    });
101    it('should fix template size if requested', async function () {
102      const newTemplate = 'iVBORbaz';
103      await d.settings.update({fixImageTemplateSize: true});
104      sinon.stub(f, 'ensureTemplateSize').returns(newTemplate);
105      const imgElProto = await f.findByImage(template, {multiple: false});
106      const imgEl = basicImgElVerify(imgElProto, f);
107      imgEl.template.should.eql(newTemplate);
108      _.last(compareStub.args)[2].should.eql(newTemplate);
109    });
110
111    it('should fix template size scale if requested', async function () {
112      const newTemplate = 'iVBORbaz';
113      await d.settings.update({fixImageTemplateScale: true});
114      sinon.stub(f, 'fixImageTemplateScale').returns(newTemplate);
115      const imgElProto = await f.findByImage(template, {multiple: false});
116      const imgEl = basicImgElVerify(imgElProto, f);
117      imgEl.template.should.eql(newTemplate);
118      _.last(compareStub.args)[2].should.eql(newTemplate);
119    });
120    it('should not fix template size scale if it is not requested', async function () {
121      const newTemplate = 'iVBORbaz';
122      await d.settings.update({});
123      sinon.stub(f, 'fixImageTemplateScale').returns(newTemplate);
124      f.fixImageTemplateScale.callCount.should.eql(0);
125    });
126
127    it('should throw an error if template match fails', async function () {
128      compareStub.throws(new Error('Cannot find any occurrences'));
129      await f.findByImage(template, {multiple: false})
130        .should.eventually.be.rejectedWith(/element could not be located/);
131    });
132    it('should return empty array for multiple elements if template match fails', async function () {
133      compareStub.throws(new Error('Cannot find any occurrences'));
134      await f.findByImage(template, {multiple: true}).should.eventually.eql([]);
135    });
136    it('should respect implicit wait', async function () {
137      d.setImplicitWait(10);
138      compareStub.resetHistory();
139      compareStub.onCall(0).throws(new Error('Cannot find any occurrences'));
140      compareStub.returns({rect, score});
141      const imgElProto = await f.findByImage(template, {multiple: false});
142      basicImgElVerify(imgElProto, f);
143      compareStub.callCount.should.eql(2);
144    });
145    it('should not add element to cache and return it directly when checking staleness', async function () {
146      const imgEl = await f.findByImage(template, {multiple: false, shouldCheckStaleness: true});
147      (imgEl instanceof ImageElement).should.be.true;
148      f.imgElCache.has(imgEl.id).should.be.false;
149      imgEl.rect.should.eql(rect);
150    });
151  });
152
153  describe('fixImageTemplateScale', function () {
154    const d = new PluginDriver();
155    const f = new ImageElementFinder(d);
156    const basicTemplate = 'iVBORbaz';
157
158    it('should not fix template size scale if no scale value', async function () {
159      await f.fixImageTemplateScale(basicTemplate, {fixImageTemplateScale: true})
160        .should.eventually.eql(basicTemplate);
161    });
162
163    it('should not fix template size scale if it is null', async function () {
164      await f.fixImageTemplateScale(basicTemplate, null)
165        .should.eventually.eql(basicTemplate);
166    });
167
168    it('should not fix template size scale if it is not number', async function () {
169      await f.fixImageTemplateScale(basicTemplate, 'wrong-scale')
170        .should.eventually.eql(basicTemplate);
171    });
172
173    it('should fix template size scale', async function () {
174      const actual = 'iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAYAAADgzO9IAAAAWElEQVR4AU3BQRWAQAhAwa/PGBsEgrC16AFBKEIPXW7OXO+Rmey9iQjMjHFzrLUwM7qbqmLcHKpKRFBVuDvj4agq3B1VRUQYT2bS3QwRQVUZF/CaGRHB3wc1vSZbHO5+BgAAAABJRU5ErkJggg==';
175      await f.fixImageTemplateScale(TINY_PNG, {
176        fixImageTemplateScale: true, xScale: 1.5, yScale: 1.5
177      }).should.eventually.eql(actual);
178    });
179
180    it('should not fix template size scale because of fixImageTemplateScale being false', async function () {
181      await f.fixImageTemplateScale(TINY_PNG, {
182        fixImageTemplateScale: false, xScale: 1.5, yScale: 1.5
183      }).should.eventually.eql(TINY_PNG);
184    });
185
186    it('should fix template size scale with default scale', async function () {
187      const actual = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABwUlEQVR4AaXBPUsrQQCG0SeX+cBdkTjwTpG1NPgLpjY/fW1stt4UYmm2cJqwMCsaw70uJJ3CBc9Z/P3Cl+12S9u2tG1L27bEGLm/v2ez2bDZbJDEd/7wS4YT7z3X19fc3Nxwd3dHXdd47xnHkefnZ8ZxpKoq6rqmqiqMMcwMJ1VV0TQN0zThnOPj44O6rsk503UdkmiahqZpWK1WGGOYGU7quqZpGqy1SCLnTM6Z19dXcs5IYpomrLVI4uLigpnhpKoqVqsVkjgcDjw9PdF1HTlnuq5DEs45JHE4HDgznByPR97e3pimiVIK4zhyPB7x3hNCIITA5eUl3nsWiwVnhpNSCsMwsNvtGIaB/X5PKQVJpJSQxHq9RhLOOc4MJ9M0sdvt2G639H3PTBIxRiQhCUnEGLHWcmY4KaUwDAN93/P4+MhyuSSlhCRSSkjCOYe1FmstZ6bve2YvLy/s93tmy+USSUhCEpIIIfAd8/DwwOz9/Z1SCpJIKSGJ9XqNJJxz/MS0bcvs6uoKScQYkYQkJBFjxFrLT0zbtsxub29JKSGJlBKScM5hrcVay09MzplZjJHPz0+894QQCCHwP/7wS/8A4e6nAg+R8LwAAAAASUVORK5CYII=';
188      await f.fixImageTemplateScale(TINY_PNG, {
189        defaultImageTemplateScale: 4.0
190      }).should.eventually.eql(actual);
191    });
192
193    it('should fix template size scale with default scale and image scale', async function () {
194      const actual = 'iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAACaUlEQVR4AbXBMWvrWBSF0c9BsFPtW91UR1U6+///FKlKKt8qqnyqnMozggkI8xgMj6x1uv+L/6zryrIsrOvKsiys68qyLFwuF87nM5fLhfP5zOVy4Xw+84wXftkLv2ziQBK26b0TEVQVu4jANrvM5Hq9spOEJCQhCUlI4mjiQBK26b1TVewkYRvb7DKTMQaZiW1s01rDNraRxNHEgSRaa1QVO0m01jjKTDKTXe+d3jtVxU4SjyYOJGGbnSRs03snM8lMMpPb7UZmkplEBFXFThK2eTRxIAnbSMI2VcX39zdjDMYYZCaZyRiDMQZVxU4StqkqHk0cSEISf5KZ7DKTMQbLsrCTRGuN3jtVxaOJg6qiqqgqqoqqoqoYY5CZ7GwTEdzvd97f34kIeu/YRhKPJg6qiswkM7ndbmQmmUlmkpnsbBMR2CYimOeZ3ju2kcSjiYOqIjP5+vpi2za2bWPbNo5aa7TW2PXe6b3Te6e1hiQeTRxUFbfbjW3bGGNwvV4ZY2Ab27TWsI1tbGMb27TWsI0kHk0cVBWZybZtXK9XPj8/+fj4YJ5nIoLWGraJCOZ5RhKSkIQkJPFo4qCqyEy2bWOMwefnJ+u6cjqdsM3ONvM8cz6feca0ris/rtcrmcnONhHB/X7n/f2diKD3jm0k8axpWRZ+ZCaZyc42EYFtIoJ5num9YxtJPGta15U/sY1tdm9vb/Te6b1jG0k8a1qWhR+2sU1rjdYatrGNbWxjm9YaknjWtK4rPyKCiKC1hm0igojg9fUVSUhCEpJ41rQsC0e22dkmIrhcLvyNF/7H6XTib73wy174Zf8AJEsePtlPj10AAAAASUVORK5CYII=';
195      await f.fixImageTemplateScale(TINY_PNG, {
196        defaultImageTemplateScale: 4.0,
197        fixImageTemplateScale: true,
198        xScale: 1.5, yScale: 1.5
199      }).should.eventually.eql(actual);
200    });
201
202    it('should not fix template size scale with default scale and image scale', async function () {
203      const actual = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABwUlEQVR4AaXBPUsrQQCG0SeX+cBdkTjwTpG1NPgLpjY/fW1stt4UYmm2cJqwMCsaw70uJJ3CBc9Z/P3Cl+12S9u2tG1L27bEGLm/v2ez2bDZbJDEd/7wS4YT7z3X19fc3Nxwd3dHXdd47xnHkefnZ8ZxpKoq6rqmqiqMMcwMJ1VV0TQN0zThnOPj44O6rsk503UdkmiahqZpWK1WGGOYGU7quqZpGqy1SCLnTM6Z19dXcs5IYpomrLVI4uLigpnhpKoqVqsVkjgcDjw9PdF1HTlnuq5DEs45JHE4HDgznByPR97e3pimiVIK4zhyPB7x3hNCIITA5eUl3nsWiwVnhpNSCsMwsNvtGIaB/X5PKQVJpJSQxHq9RhLOOc4MJ9M0sdvt2G639H3PTBIxRiQhCUnEGLHWcmY4KaUwDAN93/P4+MhyuSSlhCRSSkjCOYe1FmstZ6bve2YvLy/s93tmy+USSUhCEpIIIfAd8/DwwOz9/Z1SCpJIKSGJ9XqNJJxz/MS0bcvs6uoKScQYkYQkJBFjxFrLT0zbtsxub29JKSGJlBKScM5hrcVay09MzplZjJHPz0+894QQCCHwP/7wS/8A4e6nAg+R8LwAAAAASUVORK5CYII=';
204      await f.fixImageTemplateScale(TINY_PNG, {
205        defaultImageTemplateScale: 4.0,
206        fixImageTemplateScale: false,
207        xScale: 1.5, yScale: 1.5
208      }).should.eventually.eql(actual);
209    });
210
211    it('should not fix template size scale because of ignoreDefaultImageTemplateScale', async function () {
212      await f.fixImageTemplateScale(TINY_PNG, {
213        defaultImageTemplateScale: 4.0,
214        ignoreDefaultImageTemplateScale: true,
215      }).should.eventually.eql(TINY_PNG);
216    });
217
218    it('should ignore defaultImageTemplateScale to fix template size scale because of ignoreDefaultImageTemplateScale', async function () {
219      const actual = 'iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAYAAADgzO9IAAAAWElEQVR4AU3BQRWAQAhAwa/PGBsEgrC16AFBKEIPXW7OXO+Rmey9iQjMjHFzrLUwM7qbqmLcHKpKRFBVuDvj4agq3B1VRUQYT2bS3QwRQVUZF/CaGRHB3wc1vSZbHO5+BgAAAABJRU5ErkJggg==';
220      await f.fixImageTemplateScale(TINY_PNG, {
221        defaultImageTemplateScale: 4.0,
222        ignoreDefaultImageTemplateScale: true,
223        fixImageTemplateScale: true,
224        xScale: 1.5, yScale: 1.5
225      }).should.eventually.eql(actual);
226    });
227  });
228
229  describe('ensureTemplateSize', function () {
230    const d = new PluginDriver();
231    const f = new ImageElementFinder(d);
232
233    it('should not resize the template if it is smaller than the screen', async function () {
234      const screen = TINY_PNG_DIMS.map((n) => n * 2);
235      await f.ensureTemplateSize(TINY_PNG, ...screen)
236        .should.eventually.eql(TINY_PNG);
237    });
238    it('should not resize the template if it is the same size as the screen', async function () {
239      await f.ensureTemplateSize(TINY_PNG, ...TINY_PNG_DIMS)
240        .should.eventually.eql(TINY_PNG);
241    });
242    it('should resize the template if it is bigger than the screen', async function () {
243      const screen = TINY_PNG_DIMS.map((n) => n / 2);
244      const newTemplate = await f.ensureTemplateSize(TINY_PNG, ...screen);
245      newTemplate.should.not.eql(TINY_PNG);
246      newTemplate.length.should.be.below(TINY_PNG.length);
247    });
248  });
249
250  describe('getScreenshotForImageFind', function () {
251    let d;
252    let f;
253
254    beforeEach(function () {
255      d = new PluginDriver();
256      f = new ImageElementFinder(d);
257      sinon.stub(d, 'getScreenshot').returns(TINY_PNG);
258    });
259
260    it('should fail if driver does not support getScreenshot', async function () {
261      const d = new BaseDriver();
262      const f = new ImageElementFinder(d);
263      await f.getScreenshotForImageFind()
264        .should.eventually.be.rejectedWith(/driver does not support/);
265    });
266    it('should not adjust or verify screenshot if asked not to by settings', async function () {
267      await d.settings.update({fixImageFindScreenshotDims: false});
268      const screen = TINY_PNG_DIMS.map((n) => n + 1);
269      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);
270      b64Screenshot.should.eql(TINY_PNG);
271      should.equal(scale, undefined);
272    });
273    it('should return screenshot without adjustment if it matches screen size', async function () {
274      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...TINY_PNG_DIMS);
275      b64Screenshot.should.eql(TINY_PNG);
276      should.equal(scale, undefined);
277    });
278    it('should return scaled screenshot with same aspect ratio if matching screen aspect ratio', async function () {
279      const screen = TINY_PNG_DIMS.map((n) => n * 1.5);
280      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);
281      b64Screenshot.should.not.eql(TINY_PNG);
282      const screenshotObj = await imageUtil.getJimpImage(b64Screenshot);
283      screenshotObj.bitmap.width.should.eql(screen[0]);
284      screenshotObj.bitmap.height.should.eql(screen[1]);
285      scale.should.eql({ xScale: 1.5, yScale: 1.5 });
286    });
287    it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio', async function () {
288      // try first with portrait screen, screen = 8 x 12
289      let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];
290      let expectedScale = { xScale: 2.67, yScale: 4 };
291
292      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);
293      b64Screenshot.should.not.eql(TINY_PNG);
294      let screenshotObj = await imageUtil.getJimpImage(b64Screenshot);
295      screenshotObj.bitmap.width.should.eql(screen[0]);
296      screenshotObj.bitmap.height.should.eql(screen[1]);
297      scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());
298      scale.yScale.should.eql(expectedScale.yScale);
299
300      // then with landscape screen, screen = 12 x 8
301      screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];
302      expectedScale = { xScale: 4, yScale: 2.67 };
303
304      const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(...screen);
305      newScreen.should.not.eql(TINY_PNG);
306      screenshotObj = await imageUtil.getJimpImage(newScreen);
307      screenshotObj.bitmap.width.should.eql(screen[0]);
308      screenshotObj.bitmap.height.should.eql(screen[1]);
309      newScale.xScale.should.eql(expectedScale.xScale);
310      newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());
311    });
312
313    it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio with fixImageTemplateScale', async function () {
314      // try first with portrait screen, screen = 8 x 12
315      let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];
316      let expectedScale = { xScale: 2.67, yScale: 4 };
317
318      const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);
319      b64Screenshot.should.not.eql(TINY_PNG);
320      let screenshotObj = await imageUtil.getJimpImage(b64Screenshot);
321      screenshotObj.bitmap.width.should.eql(screen[0]);
322      screenshotObj.bitmap.height.should.eql(screen[1]);
323      scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());
324      scale.yScale.should.eql(expectedScale.yScale);
325      // 8 x 12 stretched TINY_PNG
326      await f.fixImageTemplateScale(b64Screenshot, {fixImageTemplateScale: true, scale})
327        .should.eventually.eql('iVBORw0KGgoAAAANSUhEUgAAAAgAAAAMCAYAAABfnvydAAAAJ0lEQVR4AYXBAQEAIACDMKR/p0fTBrKdbZcPCRIkSJAgQYIECRIkPAzBA1TpeNwZAAAAAElFTkSuQmCC');
328
329      // then with landscape screen, screen = 12 x 8
330      screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];
331      expectedScale = { xScale: 4, yScale: 2.67 };
332
333      const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(...screen);
334      newScreen.should.not.eql(TINY_PNG);
335      screenshotObj = await imageUtil.getJimpImage(newScreen);
336      screenshotObj.bitmap.width.should.eql(screen[0]);
337      screenshotObj.bitmap.height.should.eql(screen[1]);
338      newScale.xScale.should.eql(expectedScale.xScale);
339      newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());
340      // 12 x 8 stretched TINY_PNG
341      await f.fixImageTemplateScale(newScreen, {fixImageTemplateScale: true, scale})
342        .should.eventually.eql('iVBORw0KGgoAAAANSUhEUgAAAAwAAAAICAYAAADN5B7xAAAAI0lEQVR4AZXBAQEAMAyDMI5/T5W2ayB5245AIokkkkgiiST6+W4DTLyo5PUAAAAASUVORK5CYII=');
343    });
344  });
345});
346
Full Screen

find.js

Source: find.js Github

copy
1import log from '../logger';
2import { logger, imageUtil } from 'appium-support';
3import _ from 'lodash';
4import { errors } from '../../..';
5import { MATCH_TEMPLATE_MODE } from './images';
6import { ImageElement, DEFAULT_TEMPLATE_IMAGE_SCALE } from '../image-element';
7
8
9const commands = {}, helpers = {}, extensions = {};
10
11const IMAGE_STRATEGY = '-image';
12const CUSTOM_STRATEGY = '-custom';
13
14// Used to compare ratio and screen width
15// Pixel is basically under 1080 for example. 100K is probably enough fo a while.
16const FLOAT_PRECISION = 100000;
17
18// Override the following function for your own driver, and the rest is taken
19// care of!
20
21// helpers.findElOrEls = async function (strategy, selector, mult, context) {}
22//   strategy: locator strategy
23//   selector: the actual selector for finding an element
24//   mult: multiple elements or just one?
25//   context: finding an element from the root context? or starting from another element
26//
27// Returns an object which adheres to the way the JSON Wire Protocol represents elements:
28// { ELEMENT: # }    eg: { ELEMENT: 3 }  or { ELEMENT: 1.023 }
29
30helpers.findElOrElsWithProcessing = async function findElOrElsWithProcessing (strategy, selector, mult, context) {
31  this.validateLocatorStrategy(strategy);
32  try {
33    return await this.findElOrEls(strategy, selector, mult, context);
34  } catch (err) {
35    if (this.opts.printPageSourceOnFindFailure) {
36      const src = await this.getPageSource();
37      log.debug(`Error finding element${mult ? 's' : ''}: ${err.message}`);
38      log.debug(`Page source requested through 'printPageSourceOnFindFailure':`);
39      log.debug(src);
40    }
41    // still want the error to occur
42    throw err;
43  }
44};
45
46commands.findElement = async function findElement (strategy, selector) {
47  if (strategy === IMAGE_STRATEGY) {
48    return await this.findByImage(selector, {multiple: false});
49  } else if (strategy === CUSTOM_STRATEGY) {
50    return await this.findByCustom(selector, false);
51  }
52
53  return await this.findElOrElsWithProcessing(strategy, selector, false);
54};
55
56commands.findElements = async function findElements (strategy, selector) {
57  if (strategy === IMAGE_STRATEGY) {
58    return await this.findByImage(selector, {multiple: true});
59  } else if (strategy === CUSTOM_STRATEGY) {
60    return await this.findByCustom(selector, true);
61  }
62
63  return await this.findElOrElsWithProcessing(strategy, selector, true);
64};
65
66commands.findElementFromElement = async function findElementFromElement (strategy, selector, elementId) {
67  return await this.findElOrElsWithProcessing(strategy, selector, false, elementId);
68};
69
70commands.findElementsFromElement = async function findElementsFromElement (strategy, selector, elementId) {
71  return await this.findElOrElsWithProcessing(strategy, selector, true, elementId);
72};
73
74/**
75 * Find an element using a custom plugin specified by the customFindModules cap.
76 *
77 * @param {string} selector - the selector which the plugin will use to find
78 * elements
79 * @param {boolean} multiple - whether we want one element or multiple
80 *
81 * @returns {WebElement} - WebDriver element or list of elements
82 */
83commands.findByCustom = async function findByCustom (selector, multiple) {
84  const plugins = this.opts.customFindModules;
85
86  // first ensure the user has registered one or more find plugins
87  if (!plugins) {
88    // TODO this info should go in docs instead; update when docs for this
89    // feature exist
90    throw new Error('Finding an element using a plugin is currently an ' +
91      'incubating feature. To use it you must manually install one or more ' +
92      'plugin modules in a way that they can be required by Appium, for ' +
93      'example installing them from the Appium directory, installing them ' +
94      'globally, or installing them elsewhere and passing an absolute path as ' +
95      'the capability. Then construct an object where the key is the shortcut ' +
96      'name for this plugin and the value is the module name or absolute path, ' +
97      'for example: {"p1": "my-find-plugin"}, and pass this in as the ' +
98      "'customFindModules' capability.");
99  }
100
101  // then do some basic checking of the type of the capability
102  if (!_.isPlainObject(plugins)) {
103    throw new Error("Invalid format for the 'customFindModules' capability. " +
104      'It should be an object with keys corresponding to the short names and ' +
105      'values corresponding to the full names of the element finding plugins');
106  }
107
108  // get the name of the particular plugin used for this invocation of find,
109  // and separate it from the selector we will pass to the plugin
110  let [plugin, realSelector] = selector.split(':');
111
112  // if the user didn't specify a plugin for this find invocation, and we had
113  // multiple plugins registered, that's a problem
114  if (_.size(plugins) > 1 && !realSelector) {
115    throw new Error(`Multiple element finding plugins were registered ` +
116      `(${_.keys(plugins)}), but your selector did not indicate which plugin ` +
117      `to use. Ensure you put the short name of the plugin followed by ':' as ` +
118      `the initial part of the selector string.`);
119  }
120
121  // but if they did not specify a plugin and we only have one plugin, just use
122  // that one
123  if (_.size(plugins) === 1 && !realSelector) {
124    realSelector = plugin;
125    plugin = _.keys(plugins)[0];
126  }
127
128  if (!plugins[plugin]) {
129    throw new Error(`Selector specified use of element finding plugin ` +
130      `'${plugin}' but it was not registered in the 'customFindModules' ` +
131      `capability.`);
132  }
133
134  let finder;
135  try {
136    log.debug(`Find plugin '${plugin}' requested; will attempt to use it ` +
137      `from '${plugins[plugin]}'`);
138    finder = require(plugins[plugin]);
139  } catch (err) {
140    throw new Error(`Could not load your custom find module '${plugin}'. Did ` +
141      `you put it somewhere Appium can 'require' it? Original error: ${err}`);
142  }
143
144  if (!finder || !_.isFunction(finder.find)) {
145    throw new Error('Your custom find module did not appear to be constructed ' +
146        'correctly. It needs to export an object with a `find` method.');
147  }
148
149  const customFinderLog = logger.getLogger(plugin);
150
151  let elements;
152  const condition = async () => {
153    // get a list of matched elements from the custom finder, which can
154    // potentially use the entire suite of methods the current driver provides.
155    // the finder should always return a list of elements, but may use the
156    // knowledge of whether we are looking for one or many to perform internal
157    // optimizations
158    elements = await finder.find(this, customFinderLog, realSelector, multiple);
159
160    // if we're looking for multiple elements, or if we're looking for only
161    // one and found it, we're done
162    if (!_.isEmpty(elements) || multiple) {
163      return true;
164    }
165
166    // otherwise we should retry, so return false to trigger the retry loop
167    return false;
168  };
169
170  try {
171    // make sure we respect implicit wait
172    await this.implicitWaitForCondition(condition);
173  } catch (err) {
174    if (err.message.match(/Condition unmet/)) {
175      throw new errors.NoSuchElementError();
176    }
177    throw err;
178  }
179
180  return multiple ? elements : elements[0];
181};
182
183/**
184 * @typedef {Object} FindByImageOptions
185 * @property {boolean} [shouldCheckStaleness=false] - whether this call to find an
186 * image is merely to check staleness. If so we can bypass a lot of logic
187 * @property {boolean} [multiple=false] - Whether we are finding one element or
188 * multiple
189 * @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we
190 * ignore defaultImageTemplateScale. It can be used when you would like to
191 * scale b64Template with defaultImageTemplateScale setting.
192 */
193
194/**
195 * Find a screen rect represented by an ImageElement corresponding to an image
196 * template sent in by the client
197 *
198 * @param {string} b64Template - base64-encoded image used as a template to be
199 * matched in the screenshot
200 * @param {FindByImageOptions} - additional options
201 *
202 * @returns {WebElement} - WebDriver element with a special id prefix
203 */
204helpers.findByImage = async function findByImage (b64Template, {
205  shouldCheckStaleness = false,
206  multiple = false,
207  ignoreDefaultImageTemplateScale = false,
208}) {
209  const {
210    imageMatchThreshold: threshold,
211    imageMatchMethod,
212    fixImageTemplateSize,
213    fixImageTemplateScale,
214    defaultImageTemplateScale,
215    getMatchedImageResult: visualize
216  } = this.settings.getSettings();
217
218  log.info(`Finding image element with match threshold ${threshold}`);
219  if (!this.getWindowSize) {
220    throw new Error("This driver does not support the required 'getWindowSize' command");
221  }
222  const {width: screenWidth, height: screenHeight} = await this.getWindowSize();
223
224  // someone might have sent in a template that's larger than the screen
225  // dimensions. If so let's check and cut it down to size since the algorithm
226  // will not work unless we do. But because it requires some potentially
227  // expensive commands, only do this if the user has requested it in settings.
228  if (fixImageTemplateSize) {
229    b64Template = await this.ensureTemplateSize(b64Template, screenWidth,
230      screenHeight);
231  }
232
233  const results = [];
234  const condition = async () => {
235    try {
236      const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);
237
238      b64Template = await this.fixImageTemplateScale(b64Template, {
239        defaultImageTemplateScale, ignoreDefaultImageTemplateScale,
240        fixImageTemplateScale, ...scale
241      });
242
243      const comparisonOpts = {
244        threshold,
245        visualize,
246        multiple,
247      };
248      if (imageMatchMethod) {
249        comparisonOpts.method = imageMatchMethod;
250      }
251      if (multiple) {
252        results.push(...(await this.compareImages(MATCH_TEMPLATE_MODE,
253                                                  b64Screenshot,
254                                                  b64Template,
255                                                  comparisonOpts)));
256      } else {
257        results.push(await this.compareImages(MATCH_TEMPLATE_MODE,
258                                              b64Screenshot,
259                                              b64Template,
260                                              comparisonOpts));
261      }
262      return true;
263
264    } catch (err) {
265      // if compareImages fails, we'll get a specific error, but we should
266      // retry, so trap that and just return false to trigger the next round of
267      // implicitly waiting. For other errors, throw them to get out of the
268      // implicit wait loop
269      if (err.message.match(/Cannot find any occurrences/)) {
270        return false;
271      }
272      throw err;
273    }
274  };
275
276  try {
277    await this.implicitWaitForCondition(condition);
278  } catch (err) {
279    // this `implicitWaitForCondition` method will throw a 'Condition unmet'
280    // error if an element is not found eventually. In that case, we will
281    // handle the element not found response below. In the case where get some
282    // _other_ kind of error, it means something blew up totally apart from the
283    // implicit wait timeout. We should not mask that error and instead throw
284    // it straightaway
285    if (!err.message.match(/Condition unmet/)) {
286      throw err;
287    }
288  }
289
290  if (_.isEmpty(results)) {
291    if (multiple) {
292      return [];
293    }
294    throw new errors.NoSuchElementError();
295  }
296
297  const elements = results.map(({rect, score, visualization}) => {
298    log.info(`Image template matched: ${JSON.stringify(rect)}`);
299    return new ImageElement(b64Template, rect, score, visualization);
300  });
301
302  // if we're just checking staleness, return straightaway so we don't add
303  // a new element to the cache. shouldCheckStaleness does not support multiple
304  // elements, since it is a purely internal mechanism
305  if (shouldCheckStaleness) {
306    return elements[0];
307  }
308
309  const registeredElements = elements.map((imgEl) => this.registerImageElement(imgEl));
310
311  return multiple ? registeredElements : registeredElements[0];
312};
313
314/**
315 * Ensure that the image template sent in for a find is of a suitable size
316 *
317 * @param {string} b64Template - base64-encoded image
318 * @param {int} screenWidth - width of screen
319 * @param {int} screenHeight - height of screen
320 *
321 * @returns {string} base64-encoded image, potentially resized
322 */
323helpers.ensureTemplateSize = async function ensureTemplateSize (b64Template, screenWidth, screenHeight) {
324  let imgObj = await imageUtil.getJimpImage(b64Template);
325  let {width: tplWidth, height: tplHeight} = imgObj.bitmap;
326
327  log.info(`Template image is ${tplWidth}x${tplHeight}. Screen size is ${screenWidth}x${screenHeight}`);
328  // if the template fits inside the screen dimensions, we're good
329  if (tplWidth <= screenWidth && tplHeight <= screenHeight) {
330    return b64Template;
331  }
332
333  log.info(`Scaling template image from ${tplWidth}x${tplHeight} to match ` +
334           `screen at ${screenWidth}x${screenHeight}`);
335  // otherwise, scale it to fit inside the screen dimensions
336  imgObj = imgObj.scaleToFit(screenWidth, screenHeight);
337  return (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');
338};
339
340/**
341 * @typedef {Object} Screenshot
342 * @property {string} b64Screenshot - base64 based screenshot string
343 */
344/**
345 * @typedef {Object} ScreenshotScale
346 * @property {float} xScale - Scale ratio for width
347 * @property {float} yScale - Scale ratio for height
348 */
349/**
350 * Get the screenshot image that will be used for find by element, potentially
351 * altering it in various ways based on user-requested settings
352 *
353 * @param {int} screenWidth - width of screen
354 * @param {int} screenHeight - height of screen
355 *
356 * @returns {Screenshot, ?ScreenshotScale} base64-encoded screenshot and ScreenshotScale
357 */
358helpers.getScreenshotForImageFind = async function getScreenshotForImageFind (screenWidth, screenHeight) {
359  if (!this.getScreenshot) {
360    throw new Error("This driver does not support the required 'getScreenshot' command");
361  }
362
363  let b64Screenshot = await this.getScreenshot();
364
365  // if the user has requested not to correct for aspect or size differences
366  // between the screenshot and the screen, just return the screenshot now
367  if (!this.settings.getSettings().fixImageFindScreenshotDims) {
368    log.info(`Not verifying screenshot dimensions match screen`);
369    return {b64Screenshot};
370  }
371
372  if (screenWidth < 1 || screenHeight < 1) {
373    log.warn(`The retrieved screen size ${screenWidth}x${screenHeight} does ` +
374      `not seem to be valid. No changes will be applied to the screenshot`);
375    return {b64Screenshot};
376  }
377
378  // otherwise, do some verification on the screenshot to make sure it matches
379  // the screen size and aspect ratio
380  log.info('Verifying screenshot size and aspect ratio');
381
382  let imgObj = await imageUtil.getJimpImage(b64Screenshot);
383  let {width: shotWidth, height: shotHeight} = imgObj.bitmap;
384
385  if (shotWidth < 1 || shotHeight < 1) {
386    log.warn(`The retrieved screenshot size ${shotWidth}x${shotHeight} does ` +
387      `not seem to be valid. No changes will be applied to the screenshot`);
388    return {b64Screenshot};
389  }
390
391  if (screenWidth === shotWidth && screenHeight === shotHeight) {
392    // the height and width of the screenshot and the device screen match, which
393    // means we should be safe when doing template matches
394    log.info('Screenshot size matched screen size');
395    return {b64Screenshot};
396  }
397
398  // otherwise, if they don't match, it could spell problems for the accuracy
399  // of coordinates returned by the image match algorithm, since we match based
400  // on the screenshot coordinates not the device coordinates themselves. There
401  // are two potential types of mismatch: aspect ratio mismatch and scale
402  // mismatch. We need to detect and fix both
403
404  const scale = {xScale: 1.0, yScale: 1.0};
405
406  const screenAR = screenWidth / screenHeight;
407  const shotAR = shotWidth / shotHeight;
408  if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {
409    log.info(`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +
410      `screen aspect ratio '${screenAR}' (${screenWidth}x${screenHeight})`);
411  } else {
412    log.warn(`When trying to find an element, determined that the screen ` +
413             `aspect ratio and screenshot aspect ratio are different. Screen ` +
414             `is ${screenWidth}x${screenHeight} whereas screenshot is ` +
415             `${shotWidth}x${shotHeight}.`);
416
417    // In the case where the x-scale and y-scale are different, we need to decide
418    // which one to respect, otherwise the screenshot and template will end up
419    // being resized in a way that changes its aspect ratio (distorts it). For example, let's say:
420    // this.getScreenshot(shotWidth, shotHeight) is 540x397,
421    // this.getDeviceSize(screenWidth, screenHeight) is 1080x1920.
422    // The ratio would then be {xScale: 0.5, yScale: 0.2}.
423    // In this case, we must should `yScale: 0.2` as scaleFactor, because
424    // if we select the xScale, the height will be bigger than real screenshot size
425    // which is used to image comparison by OpenCV as a base image.
426    // All of this is primarily useful when the screenshot is a horizontal slice taken out of the
427    // screen (for example not including top/bottom nav bars)
428    const xScale = (1.0 * shotWidth) / screenWidth;
429    const yScale = (1.0 * shotHeight) / screenHeight;
430    const scaleFactor = xScale >= yScale ? yScale : xScale;
431
432    log.warn(`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` +
433             `screen aspect ratio so that image element coordinates have a ` +
434             `greater chance of being correct.`);
435    imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor);
436
437    scale.xScale *= scaleFactor;
438    scale.yScale *= scaleFactor;
439
440    shotWidth = imgObj.bitmap.width;
441    shotHeight = imgObj.bitmap.height;
442  }
443
444  // Resize based on the screen dimensions only if both width and height are mismatched
445  // since except for that, it might be a situation which is different window rect and
446  // screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and
447  // `"deviceScreenSize"=>"1080x1920"`
448  if (screenWidth !== shotWidth && screenHeight !== shotHeight) {
449    log.info(`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +
450             `screen at ${screenWidth}x${screenHeight}`);
451    imgObj = imgObj.resize(screenWidth, screenHeight);
452
453    scale.xScale *= (1.0 * screenWidth) / shotWidth;
454    scale.yScale *= (1.0 * screenHeight) / shotHeight;
455  }
456
457  b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');
458  return {b64Screenshot, scale};
459};
460
461/**
462 * @typedef {Object} ImageTemplateSettings
463 * @property {boolean} fixImageTemplateScale - fixImageTemplateScale in device-settings
464 * @property {float} defaultImageTemplateScale - defaultImageTemplateScale in device-settings
465 * @property {boolean} ignoreDefaultImageTemplateScale - Ignore defaultImageTemplateScale if it has true.
466 * If b64Template has been scaled to defaultImageTemplateScale or should ignore the scale,
467 * this parameter should be true. e.g. click in image-element module
468 * @property {float} xScale - Scale ratio for width
469 * @property {float} yScale - Scale ratio for height
470
471 */
472/**
473 * Get a image that will be used for template maching.
474 * Returns scaled image if scale ratio is provided.
475 *
476 *
477 * @param {string} b64Template - base64-encoded image used as a template to be
478 * matched in the screenshot
479 * @param {ImageTemplateSettings} opts - Image template scale related options
480 *
481 * @returns {string} base64-encoded scaled template screenshot
482 */
483const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;
484helpers.fixImageTemplateScale = async function fixImageTemplateScale (b64Template, opts = {}) {
485  if (!opts) {
486    return b64Template;
487  }
488
489  let {
490    fixImageTemplateScale = false,
491    defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,
492    ignoreDefaultImageTemplateScale = false,
493    xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
494    yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE
495  } = opts;
496
497  if (ignoreDefaultImageTemplateScale) {
498    defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;
499  }
500
501  // Default
502  if (defaultImageTemplateScale === DEFAULT_TEMPLATE_IMAGE_SCALE && !fixImageTemplateScale) {
503    return b64Template;
504  }
505
506  // Calculate xScale and yScale Appium should scale
507  if (fixImageTemplateScale) {
508    xScale *= defaultImageTemplateScale;
509    yScale *= defaultImageTemplateScale;
510  } else {
511    xScale = yScale = 1 * defaultImageTemplateScale;
512  }
513
514  // xScale and yScale can be NaN if defaultImageTemplateScale is string, for example
515  if (!parseFloat(xScale) || !parseFloat(yScale)) {
516    return b64Template;
517  }
518
519  // Return if the scale is default, 1, value
520  if (Math.round(xScale * FLOAT_PRECISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)
521      && Math.round(yScale * FLOAT_PRECISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION))) {
522    return b64Template;
523  }
524
525  let imgTempObj = await imageUtil.getJimpImage(b64Template);
526  let {width: baseTempWidth, height: baseTempHeigh} = imgTempObj.bitmap;
527
528  const scaledWidth = baseTempWidth * xScale;
529  const scaledHeight = baseTempHeigh * yScale;
530  log.info(`Scaling template image from ${baseTempWidth}x${baseTempHeigh}` +
531            ` to ${scaledWidth}x${scaledHeight}`);
532  log.info(`The ratio is ${xScale} and ${yScale}`);
533  imgTempObj = await imgTempObj.resize(scaledWidth, scaledHeight);
534  return (await imgTempObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');
535};
536
537Object.assign(extensions, commands, helpers);
538export { commands, helpers, IMAGE_STRATEGY, CUSTOM_STRATEGY };
539export default extensions;
540
Full Screen

Accelerate Your Automation Test Cycles With LambdaTest

Leverage LambdaTest’s cloud-based platform to execute your automation tests in parallel and trim down your test execution time significantly. Your first 100 automation testing minutes are on us.

Try LambdaTest

Run JavaScript Tests on LambdaTest Cloud Grid

Execute automation tests with Appium Base Driver on a cloud-based Grid of 3000+ real browsers and operating systems for both web and mobile applications.

Test now for Free
LambdaTestX

We use cookies to give you the best experience. Cookies help to provide a more personalized experience and relevant advertising for you, and web analytics for us. Learn More in our Cookies policy, Privacy & Terms of service

Allow Cookie
Sarah

I hope you find the best code examples for your project.

If you want to accelerate automated browser testing, try LambdaTest. Your first 100 automation testing minutes are FREE.

Sarah Elson (Product & Growth Lead)