const puppeteer = require('puppeteer');
const db = require('./models');
const crawler = async () => {
await db.sequelize.sync();
try {
let browser = await puppeteer.launch({
headless: false,
args: ['--window-size=1920,1080', '--disable-notifications'],
});
let page = await browser.newPage();
await page.setViewport({
width: 1080,
height: 1080,
});
await page.goto('http://spys.one/free-proxy-list/KR/');
const proxies = await page.evaluate(() => {
const ips = Array.from(document.querySelectorAll('tr > td:first-of-type > .spy14')).map(v => v.textContent.replace(/document\.write\(.+\)/, ''));
const types = Array.from(document.querySelectorAll('tr > td:nth-of-type(2)'))
.slice(5)
.map(v => v.textContent);
const latencies = Array.from(document.querySelectorAll('tr > td:nth-of-type(6) .spy1')).map(v => v.textContent);
return ips.map((v, i) => {
return {
ip: v,
type: types[i],
latency: latencies[i],
};
});
});
const filtered = proxies.filter(v => v.type.startsWith('HTTP')).sort((p, c) => p.latency - c.latency);
await Promise.all(
filtered.map(async v => {
return db.Proxy.upsert({
ip: v.ip,
type: v.type,
latency: v.latency,
});
}),
);
await page.close();
await browser.close();
const fastestProxies = await db.Proxy.findAll({
order: [['latency', 'ASC']],
});
let browsers = [];
for (let i = 0; i < 10; i++) {
const browser = await puppeteer.launch({
headless: false,
args: ['--window-size=1920,1080', '--disable-notifications', `--proxy-server=${fastestProxies[i].ip}`],
});
const page = await browser.newPage();
page.goto('http://www.haiip.net/service/myip/cur_ip_chk.php');
}
console.log(browsers);
// const context = await browser.createIncognitoBrowserContext();
// const context2 = await browser.createIncognitoBrowserContext();
// const context3 = await browser.createIncognitoBrowserContext();
// console.log(await browser.browserContexts());
// page = await browser.newPage();
// await page.goto('https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query=%EB%82%B4+%EC%95%84%EC%9D%B4%ED%94%BC');
// await page.waitFor(10000);
// await page.close();
// await browser.close();
await db.sequelize.close();
} catch (e) {
console.error(e);
}
};
crawler();
const puppeteer = require('puppeteer');
const db = require('./models');
const crawler = async () => {
await db.sequelize.sync();
try {
let browser = await puppeteer.launch({
headless: false,
args: ['--window-size=1920,1080', '--disable-notifications'],
});
let page = await browser.newPage();
await page.setViewport({
width: 1080,
height: 1080,
});
await page.goto('http://spys.one/free-proxy-list/KR/');
const proxies = await page.evaluate(() => {
const ips = Array.from(document.querySelectorAll('tr > td:first-of-type > .spy14')).map((v) => v.textContent.replace(/document\.write\(.+\)/, ''));
const types = Array.from(document.querySelectorAll('tr > td:nth-of-type(2)')).slice(5).map((v) => v.textContent);
const latencies = Array.from(document.querySelectorAll('tr > td:nth-of-type(6) .spy1')).map((v) => v.textContent);
return ips.map((v, i) => {
return {
ip: v,
type: types[i],
latency: latencies[i],
}
});
});
const filtered = proxies.filter((v) => v.type.startsWith('HTTP')).sort((p, c) => p.latency - c.latency);
await Promise.all(filtered.map(async (v) => {
return db.Proxy.upsert({
ip: v.ip,
type: v.type,
latency: v.latency,
});
}));
await page.close();
await browser.close();
const fastestProxies = await db.Proxy.findAll({
order: [['latency', 'ASC']],
});
browser = await puppeteer.launch({
headless: false,
args: ['--window-size=1920,1080', '--disable-notifications', `--proxy-server=${fastestProxies[0].ip}`],
});
const browser2 = await puppeteer.launch({
headless: false,
args: ['--window-size=1920,1080', '--disable-notifications', `--proxy-server=${fastestProxies[1].ip}`],
});
const browser3 = await puppeteer.launch({
headless: false,
args: ['--window-size=1920,1080', '--disable-notifications', `--proxy-server=${fastestProxies[2].ip}`],
});
// const context = await browser.createIncognitoBrowserContext();
// const context2 = await browser.createIncognitoBrowserContext();
// const context3 = await browser.createIncognitoBrowserContext();
// console.log(await browser.browserContexts());
const page1 = await browser.newPage();
const page2 = await browser2.newPage();
const page3 = await browser3.newPage();
await page1.goto('https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query=%EB%82%B4+%EC%95%84%EC%9D%B4%ED%94%BC');
await page2.goto('https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query=%EB%82%B4+%EC%95%84%EC%9D%B4%ED%94%BC');
await page3.goto('https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query=%EB%82%B4+%EC%95%84%EC%9D%B4%ED%94%BC');
// page = await browser.newPage();
// await page.goto('https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query=%EB%82%B4+%EC%95%84%EC%9D%B4%ED%94%BC');
// await page.waitFor(10000);
// await page.close();
// await browser.close();
await db.sequelize.close();
} catch (e) {
console.error(e);
}
};
crawler();
/**
* Copyright 2018 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const utils = require('./utils');
const puppeteer = utils.requireRoot('index');
const {TimeoutError} = utils.requireRoot('Errors');
module.exports.addTests = function({testRunner, expect}) {
const {describe, xdescribe, fdescribe} = testRunner;
const {it, fit, xit} = testRunner;
const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;
describe('BrowserContext', function() {
it('should have default context', async function({browser, server}) {
expect(browser.browserContexts().length).toBe(1);
const defaultContext = browser.browserContexts()[0];
expect(defaultContext.isIncognito()).toBe(false);
let error = null;
await defaultContext.close().catch(e => error = e);
expect(browser.defaultBrowserContext()).toBe(defaultContext);
expect(error.message).toContain('cannot be closed');
});
it('should create new incognito context', async function({browser, server}) {
expect(browser.browserContexts().length).toBe(1);
const context = await browser.createIncognitoBrowserContext();
expect(context.isIncognito()).toBe(true);
expect(browser.browserContexts().length).toBe(2);
expect(browser.browserContexts().indexOf(context) !== -1).toBe(true);
await context.close();
expect(browser.browserContexts().length).toBe(1);
});
it('should close all belonging targets once closing context', async function({browser, server}) {
expect((await browser.pages()).length).toBe(1);
const context = await browser.createIncognitoBrowserContext();
await context.newPage();
expect((await browser.pages()).length).toBe(2);
expect((await context.pages()).length).toBe(1);
await context.close();
expect((await browser.pages()).length).toBe(1);
});
it('window.open should use parent tab context', async function({browser, server}) {
const context = await browser.createIncognitoBrowserContext();
const page = await context.newPage();
await page.goto(server.EMPTY_PAGE);
const [popupTarget] = await Promise.all([
utils.waitEvent(browser, 'targetcreated'),
page.evaluate(url => window.open(url), server.EMPTY_PAGE)
]);
expect(popupTarget.browserContext()).toBe(context);
await context.close();
});
it('should fire target events', async function({browser, server}) {
const context = await browser.createIncognitoBrowserContext();
const events = [];
context.on('targetcreated', target => events.push('CREATED: ' + target.url()));
context.on('targetchanged', target => events.push('CHANGED: ' + target.url()));
context.on('targetdestroyed', target => events.push('DESTROYED: ' + target.url()));
const page = await context.newPage();
await page.goto(server.EMPTY_PAGE);
await page.close();
expect(events).toEqual([
'CREATED: about:blank',
`CHANGED: ${server.EMPTY_PAGE}`,
`DESTROYED: ${server.EMPTY_PAGE}`
]);
await context.close();
});
it('should wait for a target', async function({browser, server}) {
const context = await browser.createIncognitoBrowserContext();
let resolved = false;
const targetPromise = context.waitForTarget(target => target.url() === server.EMPTY_PAGE);
targetPromise.then(() => resolved = true);
const page = await context.newPage();
expect(resolved).toBe(false);
await page.goto(server.EMPTY_PAGE);
const target = await targetPromise;
expect(await target.page()).toBe(page);
await context.close();
});
it('should timeout waiting for a non-existent target', async function({browser, server}) {
const context = await browser.createIncognitoBrowserContext();
const error = await context.waitForTarget(target => target.url() === server.EMPTY_PAGE, {timeout: 1}).catch(e => e);
expect(error).toBeInstanceOf(TimeoutError);
await context.close();
});
it('should isolate localStorage and cookies', async function({browser, server}) {
// Create two incognito contexts.
const context1 = await browser.createIncognitoBrowserContext();
const context2 = await browser.createIncognitoBrowserContext();
expect(context1.targets().length).toBe(0);
expect(context2.targets().length).toBe(0);
// Create a page in first incognito context.
const page1 = await context1.newPage();
await page1.goto(server.EMPTY_PAGE);
await page1.evaluate(() => {
localStorage.setItem('name', 'page1');
document.cookie = 'name=page1';
});
expect(context1.targets().length).toBe(1);
expect(context2.targets().length).toBe(0);
// Create a page in second incognito context.
const page2 = await context2.newPage();
await page2.goto(server.EMPTY_PAGE);
await page2.evaluate(() => {
localStorage.setItem('name', 'page2');
document.cookie = 'name=page2';
});
expect(context1.targets().length).toBe(1);
expect(context1.targets()[0]).toBe(page1.target());
expect(context2.targets().length).toBe(1);
expect(context2.targets()[0]).toBe(page2.target());
// Make sure pages don't share localstorage or cookies.
expect(await page1.evaluate(() => localStorage.getItem('name'))).toBe('page1');
expect(await page1.evaluate(() => document.cookie)).toBe('name=page1');
expect(await page2.evaluate(() => localStorage.getItem('name'))).toBe('page2');
expect(await page2.evaluate(() => document.cookie)).toBe('name=page2');
// Cleanup contexts.
await Promise.all([
context1.close(),
context2.close()
]);
expect(browser.browserContexts().length).toBe(1);
});
it('should work across sessions', async function({browser, server}) {
expect(browser.browserContexts().length).toBe(1);
const context = await browser.createIncognitoBrowserContext();
expect(browser.browserContexts().length).toBe(2);
const remoteBrowser = await puppeteer.connect({
browserWSEndpoint: browser.wsEndpoint()
});
const contexts = remoteBrowser.browserContexts();
expect(contexts.length).toBe(2);
await remoteBrowser.disconnect();
await context.close();
});
});
};
Accelerate Your Automation Test Cycles With LambdaTest
Leverage LambdaTestās cloud-based platform to execute your automation tests in parallel and trim down your test execution time significantly. Your first 100 automation testing minutes are on us.
