Unlock 30% off on Manual Testing Annual Plans this Holiday Season. 
Use Code: LCMYEARENDEXCL30
Use Code: LCMYEARENDEXCL30Check icon
Copied to Clipboard!
How to use clearCustomQueryHandlers method in Puppeteer
Best JavaScript code snippet using puppeteer
scraper.js
Source:scraper.js 
1console.log("----------------- EXECUTING FILE: scraper.js -----------------")2const puppeteer = require("puppeteer-extra");3const { Cluster } = require('puppeteer-cluster');4const fs = require('fs-extra');5const axios = require("axios");6const links = {7 "Challenger": [8 {9 "requests": [10 {11 "indexName": "hachisearchengine",12 "params": "filters=active_sites%3AHSG&maxValuesPerFacet=1&query=&hitsPerPage=1000&highlightPreTag=__ais-highlight__&highlightPostTag=__%2Fais-highlight__&page=0&tagFilters=&facetFilters=%5B%5B%22boutiquecates.subcategory%3APc%20%26%20Notebooks%20%3E%20Notebooks%20%26%20Desktops%20%3E%20Monitors%22%5D%5D"13 }, // For normal monitors14 {15 "indexName": "hachisearchengine",16 "params": "filters=active_sites%3AHSG&maxValuesPerFacet=1&query=&hitsPerPage=1000&highlightPreTag=__ais-highlight__&highlightPostTag=__%2Fais-highlight__&page=0&tagFilters=&facetFilters=%5B%5B%22boutiquecates.subcategory%3AGaming%20%3E%20Pc%20Gaming%20%3E%20Gaming%20Monitor%22%5D%5D"17 }, // for Gaming monitors18 ]19 }20 ],21 "Best Denki": ['https://www.bestdenki.com.sg/it-mobile/computer/monitors.html'],22 "Courts": ['https://www.courts.com.sg/computing-mobile/monitors-projectors/monitors?product_list_limit=32'],23 "Harvey Norman": ['https://www.harveynorman.com.sg/computing/computers-en/monitors-en/'],24 "Gain City": ['https://www.gaincity.com/catalog/category/166/monitors'],25 "ACER Store": ['https://store.acer.com/en-sg/monitors?product_list_limit=all'],26 "DELL Store": ['https://www.dell.com/en-sg/shop/monitors/ar/6481'],27 "HP Store": ['https://www.hp.com/sg-en/shop/monitors.html?product_list_limit=30'],28 "LENOVO Store": ['https://www.lenovo.com/sg/en/accessories-and-monitors/dc/monitors?sort=sortBy¤tResultsLayoutType=grid&visibleDatas=facet_Group%3AGroup%3AMonitors'],29 "LG Store": ['https://www.lg.com/sg/consumer-monitors/view-all']30};31const headless = true32const StealthPlugin = require('puppeteer-extra-plugin-stealth');33// const { clearCustomQueryHandlers } = require("puppeteer");34puppeteer.use(StealthPlugin());35const pageTimeout = 5 * 1000 * 6036const CLUSTEROPTS = {37 puppeteer,38 concurrency: Cluster.CONCURRENCY_PAGE,39 maxConcurrency: 2,40 timeout: 30 * 1000 * 60,41 sameDomainDelay: 500,42 workerCreationDelay: 500,43 monitor: true,44 puppeteerOptions: {45 headless,46 // TOGGLE ARGS FOR PUSH47 args: [48 // '--disable-setuid-sandbox',49 // '--disable-dev-shm-usage',50 // '--disable-accelerated-2d-canvas',51 // '--no-first-run',52 // '--no-zygote',53 // '--single-process', // <- this one doesn't works in Windows54 '--disable-gpu',55 // '--no-sandbox'56 ]57 }58}59const brands = []60 ; (async () => {61 const delay = ms => new Promise(resolve => setTimeout(resolve, ms))62 try {63 const browser = await puppeteer.launch({ headless, args: ['--no-sandbox'] });64 const dictionary = await fs.readJSON(`${process.cwd()}/data/dictionary.json`)65 const extras = await fs.readJSON(`${process.cwd()}/data/extras.json`)66 await fs.ensureDir(`${process.cwd()}/data/raw/monitors`)67 await fs.ensureFile(`${process.cwd()}/data/raw/monitors/harvey.json`)68 await fs.ensureFile(`${process.cwd()}/data/raw/monitors/challenger.json`)69 await fs.ensureFile(`${process.cwd()}/data/raw/monitors/gain.json`)70 await fs.ensureFile(`${process.cwd()}/data/raw/monitors/courts.json`)71 await fs.ensureFile(`${process.cwd()}/data/raw/monitors/best.json`)72 // await fs.ensureFile(`${process.cwd()}/data/raw/monitors/lg.json`) 73 await fs.ensureFile(`${process.cwd()}/data/raw/monitors/acer.json`)74 // await fs.ensureFile(`${process.cwd()}/data/raw/monitors/asus.json`) // No monitors75 await fs.ensureFile(`${process.cwd()}/data/raw/monitors/dell.json`)76 await fs.ensureFile(`${process.cwd()}/data/raw/monitors/hp.json`)77 await fs.ensureFile(`${process.cwd()}/data/raw/monitors/lenovo.json`)78 await gain()79 await acer()80 await dell() 81 await hp()82 await lenovo()83 await harvey()84 await challenger()85 await courts() // Illegal86 await best()87 /* SECTION FOR ACER */88 async function acer() {89 console.log("PUPPETEER: Scraping Acer Store")90 const acerPage = await browser.newPage()91 await acerPage.goto(links["ACER Store"][0], { waitUntil: 'networkidle0' })92 await acerPage.exposeFunction("cleaner", cleaner)93 let acerData = await acerPage.evaluate(async () => {94 try {95 let items = document.querySelectorAll(".product-item")96 let products = []97 for (let i = 0; i < items.length; i++) {98 let item = items[i]99 let childName = item.querySelector(".product-item-link").innerText100 let price = item.querySelector('.price').innerText101 let brand = "ACER"102 let link = item.querySelector(".product-item-link").getAttribute("href")103 let model_ID = await cleaner(childName.includes("|") ? childName.split("|")[1].trim() : childName.trim())104 let image_url = item.querySelector(".product-image-photo").src105 let instock = item.querySelector(".stock.available") ? "" : "c-false"106 products.push({107 name: childName, price, brand, link, model_ID, image_url, instock,108 location: "ACER Store"109 })110 }111 return products112 } catch (e) {113 console.log(e)114 return e115 }116 })117 await acerPage.close()118 // Write to file119 fs.writeFile(`${process.cwd()}/data/raw/monitors/acer.json`, JSON.stringify(acerData), (err, file) => {120 if (err) console.log(err)121 })122 console.log("PUPPETEER: Completed scraping Acer Store")123 return true124 }125 /* SECTION FOR DELL Store (inoperable) */126 async function dell() {127 try {128 console.log("PUPPETEER: Scraping DELL Store")129 const dellPage = await browser.newPage()130 await dellPage.goto(links['DELL Store'][0], { waitUntil: 'domcontentloaded' })131 let pages = await dellPage.$eval(("#middle-content > div > div.accessories-result-product-stack > div.pageinfo-control.hide-xs.hide-sm.hide-md > div:nth-child(1) > div > span.resultcount"), e => Math.ceil(Number(e.innerText) / 12))132 console.log("Found " + pages + " for Dell")133 await dellPage.close()134 const cluster = await Cluster.launch(CLUSTEROPTS);135 for (let i = 1; i <= pages; i++) {136 let url = `${links['DELL Store'][0]}?page=${i}`137 console.log(url)138 cluster.queue({139 url, i140 })141 }142 const DELLProducts = []143 await cluster.task(async ({ page, data }) => {144 let url = data.url145 let i = data.i146 await page.exposeFunction('cleaner', cleaner)147 console.log("Scraping page " + i + " of " + pages)148 await page.goto(url, { waitUntil: 'networkidle0', timeout: pageTimeout })149 let products = await page.evaluate(async () => {150 try {151 let items = document.querySelectorAll(".stack-accessories.ps-stack")152 let products = []153 for (let i = 0; i < items.length; i++) {154 let item = items[i]155 let childName = item.querySelector(".ps-title > a").innerText156 let price = item.querySelector('.smart-popover-btn').innerText157 let brand = "DELL"158 let link = item.querySelector(".ps-title > a").getAttribute("href")159 if (!link.match(/^https?:\/\//g)) link = "https:" + link160 let model_ID = await cleaner(childName)161 // let model_ID = await cleaner(childName.includes("|") ? childName.split("|")[1].trim() : childName.trim())162 let image_url = item.querySelector("img").src163 if (!image_url.match(/^https?:\/\//g)) image_url = "https:" + image_url164 if (image_url.match('data:image')) {165 debugger;166 console.log(item) // TODO: WTF IS THIS? Do we ignore the image, and thus use the placeholder image?167 }168 let instock = item.querySelector(".stock-level-wrapper") ? "c-false" : "" // If present, out of stock169 products.push({170 name: childName, price, brand, link, model_ID, image_url, instock,171 location: "DELL Store"172 })173 }174 return products175 } catch (e) {176 console.log(e)177 return e178 }179 })180 DELLProducts.push(...products)181 })182 await cluster.idle()183 await cluster.close()184 // Write to file185 fs.writeFile(`${process.cwd()}/data/raw/monitors/dell.json`, JSON.stringify(DELLProducts), (err, file) => {186 if (err) console.log(err)187 })188 console.log("PUPPETEER: Completed scraping Dell Store")189 return true190 } catch (e) {191 console.log(e)192 }193 }194 /* SECTION FOR HP Store */195 async function hp() {196 try {197 console.log("PUPPETEER: Scraping HP Store")198 const hpPage = await browser.newPage()199 await hpPage.goto(links['HP Store'][0], { waitUntil: "networkidle2", timeout: pageTimeout })200 let pages = 1;201 try { 202 pages = await hpPage.$eval(("#category\\.product\\.list > div:nth-child(3) > div.pages > ul"), e => e.childElementCount - 1)203 } catch (e) {204 // no element, means that there's only one page205 }206 console.log("FOUND " + pages + " pages")207 await hpPage.close()208 const cluster = await Cluster.launch(CLUSTEROPTS);209 cluster.on('taskerror', (err, data, willRetry) => {210 if (willRetry) {211 console.warn(`Encountered an error while crawling ${data}. ${err.message}\nThis job will be retried`);212 } else {213 console.error(`Failed to crawl ${data}: ${err.message}`);214 }215 });216 for (let i = 1; i <= pages; i++) {217 let url = `${links['HP Store'][0]}&p=${i}`218 cluster.queue({219 url, i220 })221 }222 const HPProducts = []223 await cluster.task(async ({ page, data }) => {224 try {225 let url = data.url226 let i = data.i227 await page.exposeFunction('cleaner', cleaner)228 console.log("Scraping page " + i + " of " + pages)229 await page.goto(url, { waitUntil: 'networkidle2', timeout: pageTimeout })230 let products = await page.evaluate(async () => {231 try {232 let items = document.querySelectorAll(".product-item")233 let products = []234 for (let j = 0; j < items.length; j++) {235 let item = items[j]236 let childName = item.querySelector(".product-item-link").innerText237 let price = item.querySelector('.price').innerText238 let brand = "HP"239 let link = item.querySelector(".product-item-link").getAttribute("href")240 let model_ID = await cleaner(childName)241 let image_url = item.querySelector(".product-image-photo").dataset.src ? item.querySelector(".product-image-photo").dataset.src : item.querySelector(".product-image-photo").src242 let instock = item.querySelector(".out-stock-messaging") ? "c-false" : ""243 products.push({244 name: childName, price, brand, link, model_ID, image_url, instock,245 location: "HP Store"246 })247 }248 return products249 } catch (e) {250 console.log(e)251 return e252 }253 })254 console.log("Finished scraping page")255 HPProducts.push(...products)256 return true257 } catch (e) {258 console.log(e)259 return false260 }261 })262 await cluster.idle()263 await cluster.close()264 // console.log(ASUSProducts)265 // Write to file266 fs.writeFile(`${process.cwd()}/data/raw/monitors/hp.json`, JSON.stringify(HPProducts), (err, file) => {267 if (err) console.log(err)268 })269 console.log("PUPPETEER: Completed scraping HP Store")270 return true271 } catch (e) {272 console.log(e)273 }274 }275 /* SECTION FOR LENOVO */276 async function lenovo() {277 try {278 console.log("PUPPETEER: Scraping Lenovo Store")279 const lenovoPage = await browser.newPage()280 await lenovoPage.goto(links['LENOVO Store'][0], { waitUntil: 'networkidle2' })281 let loadTimes = await lenovoPage.$eval(("#facetTop-count"), e => Math.ceil(Number(e.innerText.split(" ")[0]) / 20) - 1)282 for (let i = 0; i < loadTimes; i++) {283 console.log("Clicking load more button (" + (i + 1) + "/" + loadTimes + ")")284 await lenovoPage.$eval((".loadmore-button"), e => e.click())285 // await lenovoPage.waitForTimeout({waitUntil: "networkidle2"})286 await timeout(1000)287 let loading = true288 while (loading) {289 // Loading modal with class .searchLoader appears when loading (display: block)290 loading = await lenovoPage.$eval((".searchLoader"), e => {291 return e.style.display == "none" ? false : true292 })293 console.log("Waiting for loading...")294 await timeout(1000)295 }296 }297 console.log("All items loaded")298 // await lenovoPage.exposeFunction("cleaner", cleaner)299 let products = await lenovoPage.evaluate(() => {300 try {301 let items = document.querySelectorAll(".facetResults-item ")302 let products = []303 for (let i = 0; i < items.length; i++) {304 let item = items[i]305 // Skip this if the price is NOT in the correct format306 if (!item.querySelector(".xblodSize")) continue307 let childName = item.querySelector('div.item-descriptions > a').innerText.replace(/\\n/g, "")308 let price = item.querySelector(".xblodSize").innerText309 let brand = "LENOVO"310 let link = item.querySelector("div.item-descriptions > a").getAttribute("href")311 link = 'https://www.lenovo.com' + link312 let model_ID = item.dataset.productcode313 let image_url = item.querySelector("div.image-container > a > img").dataset.original314 if (!image_url.match(/^https?:\/\//g)) image_url = "https:" + image_url315 if (image_url == "https:") image_url = "/images/missing.jpg"316 let instock = item.dataset.productstatus != 'available' ? "c-false" : ""317 let customizable = item.querySelector("button[type='submit']").innerText.toUpperCase().includes("CUSTOMIZE") ? true : false318 products.push({319 name: childName, price, brand, link, model_ID, image_url, instock, customizable,320 location: "LENOVO Store"321 })322 }323 console.log(products)324 return products325 } catch (e) {326 console.log(e)327 return e328 }329 })330 await lenovoPage.close()331 console.log(products)332 // Write to file333 fs.writeFile(`${process.cwd()}/data/raw/monitors/lenovo.json`, JSON.stringify(products), (err, file) => {334 if (err) console.log(err)335 })336 console.log("PUPPETEER: Completed scraping LENOVO Store")337 return true338 } catch (e) {339 }340 }341 /* SECTION FOR RAZER */342 async function razer() {343 try {344 console.log("PUPPETEER: Scraping RAZER Store")345 const razerPage = await browser.newPage()346 await razerPage.goto(links['RAZER Store'][0], { waitUntil: "networkidle2" })347 await razerPage.waitForTimeout(2500)348 // await razerPage.exposeFunction('cleaner', cleaner)349 let products = await razerPage.evaluate(async () => {350 try {351 let items = document.querySelectorAll(".grid-item")352 let products = []353 for (let i = 0; i < items.length; i++) {354 let item = items[i]355 let childName = item.querySelector(".product-item-title").innerText356 let price = item.querySelector(".final-price").innerText357 let brand = "RAZER"358 let link = item.querySelector(".product-item-title").getAttribute("href")359 link = "https://www.razer.com" + link360 let model_ID = item.querySelector(".product-item-title").dataset['productId']361 let image_url = item.querySelector(".cx-product-image > img").src362 let instock = item.querySelector(".gtm_notify_me") ? "c-false" : ""363 let customizable = false364 products.push({365 name: childName, price, brand, link, model_ID, image_url, instock, customizable,366 location: "RAZER Store"367 })368 }369 return products370 } catch (e) {371 console.log(e)372 return e373 }374 })375 await razerPage.close()376 fs.writeFile(`${process.cwd()}/data/raw/laptops/razer.json`, JSON.stringify(products), (err, file) => {377 if (err) console.log(err)378 })379 console.log("PUPPETEER: Completed scraping RAZER Store")380 return true381 } catch (e) {382 console.log(e)383 }384 }385 /* SECTION FOR GAIN CITY */386 async function gain() { // Confidence: 3387 console.log("PUPPETEER: Scraping Gain City")388 const gainPage = await browser.newPage()389 await gainPage.goto(links["Gain City"][0], { waitUntil: 'networkidle2', timeout: pageTimeout })390 await gainPage.exposeFunction("cleaner", cleaner)391 // Gain city automatically loads more items if we scroll down, so run a function to scroll all the way down until we can't anymore392 console.log("PUPPEETER: Scrolling page")393 gainPage.on('console', consoleObj => console.log(consoleObj.text()));394 await gainPage.evaluate(async () => {395 // Scroll to bottom function 396 const delay = 2000;397 const wait = (ms) => new Promise(res => setTimeout(res, ms));398 const scrollDown = async () => {399 // Scroll to footer, which is always at the bottom400 document.querySelectorAll(".product-item-info")[document.querySelectorAll(".product-item-info").length - 1].scrollIntoView({ behavior: 'smooth', block: 'end', inline: 'end' })401 }402 const run = async () => {403 let reachedEnd = false404 do {405 console.log("Checking data")406 let reachedEndElem = document.querySelector(".ias-noneleft")407 console.log(reachedEndElem ? true : false)408 if (reachedEndElem) reachedEnd = true409 console.log("Scrolling")410 await scrollDown();411 await wait(delay);412 } while (!reachedEnd);413 await wait(delay);414 return415 }416 await run()417 })418 console.log("PUPPEETER: Page completed scrolling")419 let gainData = await gainPage.evaluate(async () => {420 let items = document.querySelectorAll(".product-item-info")421 let products = []422 for (item of items) {423 let childName = item.querySelector(".product-item-link").innerText424 if (!childName) childName = "MISSING INFO"425 console.log(childName)426 let price = item.querySelector(".price").innerText427 let brand = childName.split(" ")[0]428 let model_ID = await cleaner(item.querySelector(".product-model-number").innerText)429 console.log(childName, " - ", model_ID)430 let link = item.querySelector(".product-item-link").getAttribute("href").match(/((http|ftp|https):\/\/)?([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])?/i)[0]431 if (!link.match(/^https?:\/\//g)) link = "https://" + link432 let image_url = item.querySelector(".product-image-photo").src433 if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url434 let instock = item.querySelector(".stock.unavailable") ? "c-false" : ""435 products.push({436 name: childName, price, brand, link, model_ID, image_url, instock,437 location: "Gain City"438 })439 }440 return products441 })442 await gainPage.close()443 // Write to file444 fs.writeFile(`${process.cwd()}/data/raw/monitors/gain.json`, JSON.stringify(gainData), (err, file) => {445 if (err) console.log(err)446 })447 console.log("PUPPETEER: Completed scraping GAIN CITY")448 return true449 }450 /* SECTION FOR HARVEY NORMAN */451 async function harvey() {452 try {453 console.log("PUPPETEER: Scraping Harvey Norman")454 // Create new tab455 const harveyPage = await browser.newPage()456 // Go to harvey norman laptop page457 await harveyPage.goto(links["Harvey Norman"][0], { waitUntil: 'domcontentloaded', timeout: pageTimeout })458 // Get the number of pages (Total / 20)459 let harveyPages = await harveyPage.evaluate(() => Math.ceil(Number(document.querySelector("#pagination_contents > div.toolbar > div > div:nth-child(2) > div > div.pagination-amount.col-xs-4").innerText.split(" ")[0]) / 20))460 console.log("PUPPEETER: Found pages: " + harveyPages)461 await harveyPage.close()462 const cluster = await Cluster.launch(CLUSTEROPTS);463 cluster.on('taskerror', (err, data, willRetry) => {464 if (willRetry) {465 console.warn(`Encountered an error while crawling ${data}. ${err.message}\nThis job will be retried`);466 } else {467 console.error(`Failed to crawl ${data}: ${err.message}`);468 }469 });470 console.log("After launching cluster")471 await cluster.task(async ({ page, data }) => {472 try {473 let url = data.url474 let i = data.i475 console.log("PUPPEETER-CLUSTER: Scraping page " + i + " of " + harveyPages)476 await page.goto(url, { waitUntil: 'networkidle2', timeout: pageTimeout })477 let products = await page.evaluate(() => {478 let items = document.querySelector(".col-xs-12.col-sm-9.col-md-9.omega").querySelectorAll('form')479 let products = []480 items.forEach(item => {481 let childName = item.querySelector(".product-info > a").getAttribute("title").trim().toUpperCase()482 if (!childName) childName = "MISSING INFO"483 let price = item.querySelector(".price").innerText.trim().toUpperCase()484 let brand = childName.split(" ")[0]485 let link = item.querySelector(".product-info > a").getAttribute("href").match(/((http|ftp|https):\/\/)?([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])?/i)[0]486 if (!link.match(/^https?:\/\//g)) link = "https://" + link487 let image_url = item.querySelector(".product-image > a > img").src488 if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url489 let instock = item.querySelector(".out-of-stock-block") ? "c-false" : ""490 products.push({491 name: childName, brand, price, link, image_url, instock,492 location: "Harvey Norman"493 })494 })495 return products496 })497 for (let j = 0; j < products.length; j++) {498 console.log("PUPPEETER-CLUSTER: Scraping model " + (j + 1) + " of " + products.length + " (Page " + i + ")")499 let product = products[j]500 // Query the product page to get the model data501 await page.goto(product.link, { waitUntil: "networkidle2", timeout: pageTimeout })502 let model_ID = await page.$eval("#content_features > div > table:nth-child(3)", table => {503 let headings = Array.from(table.querySelectorAll('tr > th'))504 let index;505 headings.forEach(heading => {506 if (heading.innerText.toUpperCase() == "MODEL") index = headings.indexOf(heading) + 1507 })508 return table.querySelector(`tr:nth-child(${index}) > td`).innerText.toUpperCase().trim()509 })510 product.model_ID = cleaner(model_ID)511 }512 HARVEYPRODUCTS.push(...products)513 return true514 } catch (e) {515 console.log(e)516 return false517 }518 })519 let HARVEYPRODUCTS = []520 for (let i = 1; i <= harveyPages; i++) {521 let url = `${links["Harvey Norman"][0]}page-${i}/`522 // console.log("queueing" + url)523 console.log("QUEUEING: ", url)524 cluster.queue({ url, i })525 }526 await cluster.idle()527 await cluster.close()528 fs.writeFile(`${process.cwd()}/data/raw/monitors/harvey.json`, JSON.stringify(HARVEYPRODUCTS), (err, file) => {529 if (err) console.log(err)530 })531 console.log("PUPPEETER: Completed scraping Harvey Norman")532 return true533 } catch (e) {534 console.log(e)535 return e536 }537 }538 /* SECTION FOR CHALLENGER (using their API (AXIOS)) */539 async function challenger() { // MODEL CONFIDENCE : HIGH540 try {541 console.log("AXIOS: Scraping Challenger")542 // Get the object of every item ID in challenger's laptop database543 const products = []544 for (link of links["Challenger"]) {545 let response = await axios.post("https://6bc318ijnf-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20JavaScript%20(4.10.3)%3B%20Browser%20(lite)%3B%20instantsearch.js%20(4.25.2)%3B%20Vue%20(2.6.14)%3B%20Vue%20InstantSearch%20(3.8.1)%3B%20JS%20Helper%20(3.5.4)&x-algolia-api-key=88c8a34f2b7653f93b1ce0053dbc64fe&x-algolia-application-id=6BC318IJNF",546 link,547 {548 "headers": {549 "content-type": "application/json"550 }551 }552 )553 let results = response.data.results554 // console.log(results)555 for (let r = 0; r < results.length; r++) {556 let hits = results[r].hits557 for (let i = 0; i < hits.length; i++) {558 let item_ID = hits[i].item_id559 let productResponse = await axios.get(`https://www.hachi.tech/product/${item_ID}/details`)560 let data = productResponse.data.data561 console.log(data)562 if (!data) continue563 let name = data.short_desc.toUpperCase().trim()564 let price = `$${data.prices.regular_price}`565 let brand = data.settings.dimensions.brand_id.toUpperCase().trim()566 let link = `https://www.hachi.tech/product/${item_ID}`567 let model_ID = data.settings.dimensions.model_id.toUpperCase().trim()568 // EXCEPTION: ACER PRODUCTS - CHALLENGER PLACES THE ID WRONGLY, SO WE NEED TO USE THE PRODUCT NAME INSTEAD569 if (brand == "ACER") model_ID = cleaner(name)570 let image_url = data.image_name571 if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url572 await timeout(500)573 let stockStatus = await axios.get(`https://www.hachi.tech/product/${item_ID}/inventory`)574 let stock = stockStatus.data.data575 let instock = "c-true"576 if (!stock['delv_options'].length && !stock['scl_options'].length) instock = "c-false"577 products.push({578 name,579 price,580 brand,581 model_ID,582 link,583 location: "Challenger",584 image_url,585 instock586 })587 console.log(`AXIOS: Scraping model ${i + 1} of ${hits.length} (Page ${r + 1}/${results.length})`)588 await delay(500)589 }590 }591 }592 console.log("AXIOS: Completed scraping Challenger")593 fs.writeFile(`${process.cwd()}/data/raw/monitors/challenger.json`, JSON.stringify(products), (err, file) => {594 if (err) console.log(err)595 })596 return true597 } catch (e) {598 console.log(e)599 throw e600 }601 }602 /* SECTION FOR BEST DENKI */603 async function best() {604 console.log("PUPPETEER-CLUSTER: Scraping Best Denki")605 let BESTPRODUCTS = []606 const bestPage = await browser.newPage()607 const cluster1 = await Cluster.launch(CLUSTEROPTS);608 await cluster1.task(async ({ page, data }) => {609 try {610 let url = data.url611 let i = data.i612 console.log("PUPPETEER-CLUSTER (1): Scraping page " + (i + 1) + " of " + pages)613 await page.goto(url, { waitUntil: 'networkidle2', timeout: pageTimeout })614 await page.exposeFunction("cleaner", cleaner)615 let products = await page.evaluate(async () => {616 try {617 let items = document.querySelectorAll(".product-item")618 let products = []619 for (let i = 0; i < items.length; i++) {620 let item = items[i]621 let instock = item.querySelector("button[type='submit']") ? "" : "c-false"622 // TODO: Best Denki lists everything (6XX items listed) but only a few are in stock (1XX in stock)623 // So, we ignore if it's out of stock624 // Special case625 if (instock == "c-false") continue626 var childName = item.querySelector(".product-item-link").innerText.trim()627 var price = item.querySelector(".custominhouseprice > .price").innerText.trim()628 let brand = item.querySelector(".product-item-brand").childName.split(" ")[0].toUpperCase()629 let link = item.querySelector(".product-item-link").getAttribute("href").match(/((http|ftp|https):\/\/)?([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])?/i)[0]630 if (!link.match(/^https?:\/\//g)) link = "https://" + link631 let model_ID = await cleaner(childName) // the model is the ending of the url632 let image_url = item.querySelector('.product-image-photo').dataset.src633 if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url634 products.push({635 name: childName, price, brand, link, image_url, model_ID, instock,636 location: "Best Denki"637 })638 }639 return products640 } catch (e) {641 console.log(e)642 return e643 }644 })645 BESTPRODUCTS.push(...products)646 } catch (e) {647 console.log(e)648 return e649 }650 })651 let pages = 0652 for (let i = 1; i < 10000; i++) {653 let url = `${links["Best Denki"][0]}?p=${i}`654 await bestPage.goto(url, { waitUntil: 'domcontentloaded' })655 let ended = await bestPage.$('#maincontent > div.row > div.column.main.col-lg-9.col-md-9.col-sm-12.col-xs-12.pull-right > div.message.info.empty')656 if (ended) {657 pages = i - 1658 break;659 }660 }661 for (let i = 1; i < pages; i++) {662 let url = `${links["Best Denki"][0]}?p=${i}`663 cluster1.queue({664 url, i665 })666 }667 console.log("PUPPETEER: Found pages: " + pages)668 await bestPage.close()669 await cluster1.idle()670 await cluster1.close()671 fs.writeFile(`${process.cwd()}/data/raw/monitors/best.json`, JSON.stringify(BESTPRODUCTS), (err, file) => {672 if (err) console.log(err)673 })674 console.log("PUPPETEER-CLUSTER: Completed scraping Best Denki")675 return true676 }677 /* SECTION FOR COURTS */678 async function courts() {679 console.log("PUPPETEER-CLUSTER: Scraping Courts")680 const courtsPage = await browser.newPage()681 await courtsPage.goto(links["Courts"][0], { waitUntil: "networkidle2", timeout: pageTimeout })682 // Find out how many pages there are683 // Total products divided by # per page (32)684 let courtsPages = await courtsPage.evaluate(() => Math.ceil(Number(document.querySelector('.product-count').innerText.split(" ")[0]) / 32))685 console.log("PUPPETEER: Found pages: " + courtsPages)686 await courtsPage.close()687 const cluster = await Cluster.launch(CLUSTEROPTS);688 await cluster.task(async ({ page, data }) => {689 try {690 page.on('console', async msg => {691 const args = await msg.args()692 args.forEach(async (arg) => {693 const val = await arg.jsonValue()694 // value is serializable695 if (JSON.stringify(val) !== JSON.stringify({})) console.log(val)696 // value is unserializable (or an empty oject)697 else {698 const { type, subtype, description } = arg._remoteObject699 console.log(`type: ${type}, subtype: ${subtype}, description:\n ${description}`)700 }701 })702 });703 let url = data.url704 let i = data.i705 console.log("PUPPETEER-CLUSTER: Scraping page " + i + " of " + courtsPages)706 await page.goto(url, { waitUntil: 'networkidle2', timeout: pageTimeout })707 console.log("Page Loaded!")708 await page.exposeFunction("cleaner", cleaner)709 console.log("Function exposed!")710 let products = await page.evaluate(async () => {711 try {712 let items = Array.from(document.querySelector(".columns").querySelector(".product-items").querySelectorAll(".product-item-info"))713 let products = []714 for (item of items) {715 // console.log("Here")716 let childName = item.querySelector(".product-item-name").innerText.trim().toUpperCase()717 if (!childName) childName = "MISSING INFO"718 // console.log("Here2")719 // First word is the brand720 let brand = childName.split(" ")[0]721 // console.log("Her3e")722 let price = item.querySelector(".price").innerText.trim().toUpperCase()723 // console.log("Here4")724 let link = item.querySelector(".product-item-name > a").getAttribute("href").match(/((http|ftp|https):\/\/)?([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])?/i)[0]725 if (!link.match(/^https?:\/\//g)) link = "https://" + link726 let model_ID = await cleaner(childName) ? await cleaner(childName) : childName727 // console.log("Here5")728 let image_url = item.querySelector(".product-image-photo").getAttribute("data-original")729 if (!image_url.match(/^https?:\/\//g)) image_url = "https://" + image_url730 // console.log("Here6")731 let instock = item.querySelector(".stock.unavailable") ? "c-false" : ""732 // console.log("Here7")733 products.push({734 name: childName, brand, price, link, model_ID, image_url, instock,735 location: "Courts"736 })737 // console.log("Here8")738 }739 return products740 } catch (e) {741 console.log(e)742 return e743 }744 })745 console.log(products)746 COURTSPRODUCTS.push(...products)747 } catch (e) {748 console.log(e)749 return e750 }751 })752 let COURTSPRODUCTS = []753 for (let i = 1; i <= courtsPages; i++) {754 let url = `${links["Courts"][0]}&p=${i}`755 console.log("Queuing: " + i + " of " + courtsPages)756 cluster.queue({ url, i })757 }758 await cluster.idle()759 await cluster.close()760 fs.writeFile(`${process.cwd()}/data/raw/monitors/courts.json`, JSON.stringify(COURTSPRODUCTS), (err, file) => {761 if (err) console.log(err)762 })763 console.log("PUPPETEER-CLUSTER: Completed scraping Courts")764 return true765 }766 // /* SECTION FOR LG */767 // async function lg() {768 // console.log("PUPPETEER: SCRAPING LG Store")769 // const lgPage = await browser.newPage()770 // await lgPage.goto(links["LG Store"][0], { waitUntil: "networkidle2", timeout: pageTimeout })771 // await lgPage.exposeFunction("cleaner", cleaner)772 // let products = await lgPage.evaluate(async () => {773 // let products = []774 // let items = document.querySelector()775 // let childName = 776 // products.push({777 // name: childName, brand, price, link, model_ID, image_url, instock,778 // location: "Courts"779 // })780 // return products781 // })782 // }783 await browser.close()784 function cleaner(string) {785 var original = string786 var string = string.trim().toLowerCase()787 // Replace all non-breaking space and underscores788 string = string.replace(/ |_/gi, " ")789 var finalArr = []790 // 3) Remove all special characters791 string = string.replace(/,|®|â¢|\+|\/\/|â|:/g, "")792 // 1) Remove remove all '-' that have spaces on either side - e.g. 793 // remove ' -', '- ', ' - ' but not 'adsfads-asdfasd'794 string = string.replace(/[- ][ -]/g, " ")795 // 2) Remove specifications (RAM / SSD)796 // string = string.replace(/(\s\d+[kgtp]b)+(?=\W)|^\d+[kgtb]b(?=\W)|\s\d+[kgtb]b/gi, "")797 string = string.replace(/\d+[gt]b/gi, "")798 string = string.replace(/\d\d\dSSD/gi, "")799 // GHZ800 string = string.replace(/\d.\d[gm]hz/gi, "")801 // 8-core802 string = string.replace(/\d-core/gi, "")803 // cpu name804 string = string.replace(/[ir]\d[-\s]\d\d\d\d\w?\w?[kqe]?/gi, "")805 string = string.replace(/ryzen\s\d\s\d\d\d\d(\d?|(\d\d)?)\w/gi, "")806 // GPU807 string = string.replace(/(R|G)TX\s?\d?\d\d\d(TI)?|(MX\s?\d\d\d)|RX\s?\d\d\d\d\s?(XT)?/gmi, "")808 // 4) remove pure alphanumerics?809 // 5) Remove sizing, either 810 // NN-inch, NNinch, NN-in, NNin, NN', NN"811 string = string.replace(/((\d\d\.)?\d+-?(in(ch)?|"|''?))/gi, "")812 // 1) Usually, stuff in (text)/[text]/*text*/ denotes additional information, like what challenger does.813 // Harvey norman puts the model number inside the () though, so what we do is we check each []/()/**, if 814 // inside has at least ONE word as defined in dictionary.json, then we remove the whole string, else, 815 // we only remove the brackets816 var foundBracket = string.match(/\[[^\]]*]|\([^)]*\)*|\*[^*]*\*/g)817 if (foundBracket) {818 foundBracket.forEach(match => {819 // Strip the brakcets for word comparison820 match = match.replace(/\(|\)|\[|\]|\*/g, "")821 var wordArr = match.split(" ")822 for (var i = 0; i < wordArr.length; i++) {823 var word = wordArr[i]824 var toRemove = false825 if (dictionary[word] || extras[word]) {826 // Remove this bracket827 toRemove = true828 break;829 }830 }831 if (toRemove) {832 // // This bracket (the 'match') is to be removed833 // string.replace(match, " ")834 } else {835 // Don't remove it, but strip the brackets836 finalArr.push(match)837 }838 })839 }840 // BRACKETS HAVE BEEN DEALT WITH, now REMOVE ALL THE BRACKETS841 string = string.replace(/\[[^\]]*]|\([^)]*\)*|\*[^*]*\*/g, " ")842 // console.log("*****************")843 // console.log(string )844 // console.log(original)845 // console.log("*****************")846 // remove all '-' that have spaces on either side - e.g. 847 // remove ' -', '- ', ' - ' but not 'adsfads-asdfasd'848 string = string.replace(/[- ][ -]/g, " ")849 // Final remove special chars850 string = string.replace(/,|®|â¢|\+|\/\/|â|:/g, "")851 // Remove lone numbers852 string = string.replace(/(?<=[ \t])(\d+)(?=[ \t])/g, "")853 // Split into array854 var stringArr = string.toLowerCase().split(" ")855 // for each word, check if exists in dictionary or extras856 let cleanedArr = []857 for (var i = 0; i < stringArr.length; i++) {858 var word = stringArr[i]859 var toRemove = false860 if (dictionary[word] || extras[word]) {861 // to be removed862 continue863 }864 if (!Number.isNaN(Number(word))) {865 // if the word is a number, we don't add it866 continue867 }868 // cleanedArr.push(word)869 finalArr.push(word)870 }871 return finalArr.join(" ").trim().toUpperCase()872 }873 console.log("----------------- COMPLETED EXECUTING FILE: scraper.js -----------------")874 require(`${__dirname}/sorter.js`)875 } catch (e) {876 console.log(e)877 }878 })();879function timeout(ms) {880 return new Promise(resolve => setTimeout(resolve, ms));...
puppeteer-run.js
Source:puppeteer-run.js
...44 const page3 = await browser.newPage();45 // dá»nh dạng khung cá»a sá» hiá»n thá»46 // page.setViewport({ width: 1280, height: 926 });47 try{48puppeteer.clearCustomQueryHandlers();49 await page3.goto('http://kenh14fsdfsdfds.vn/', {timeout: 5000});50 51 }catch(error){52 console.log("failed to open the page");53 }54*/55 const page = await browser.newPage();56 // dá»nh dạng khung cá»a sá» hiá»n thá»57 // page.setViewport({ width: 1280, height: 926 });58 try{59 // puppeteer.clearCustomQueryHandlers();60await page.goto('http://kenh14.vn/',{waitUntil: 'networkidle2', timeout: 0});61 // Wait 5 seconds62 //await new Promise(resolve => setTimeout(resolve, 5000));63 // console.log('Page loaded');64 65 }catch(error){66 console.log("failed to open the page");67 }68 //finally {69 // Always close the browser70 // await browser.close();71//}72 73 ...
sidebars.js
Source:sidebars.js
1module.exports = {2 docs: {3 Puppeteer: [4 "puppeteer.puppeteer",{5 Methods: [6 "puppeteer.puppeteer.clearcustomqueryhandlers",7 'puppeteer.puppeteer.connect',8 'puppeteer.puppeteer.customqueryhandlernames',9 'puppeteer.puppeteer.devices',10 'puppeteer.puppeteer.errors',11 'puppeteer.puppeteer.networkconditions',12 'puppeteer.puppeteer.registercustomqueryhandler',13 'puppeteer.puppeteer.unregistercustomqueryhandler',14 ]15 }16 ],17 "BrowserFetcher": [18 'puppeteer.browserfetcher',19 'puppeteer.browserfetcher.candownload',20 'puppeteer.browserfetcher.download',21 'puppeteer.browserfetcher.host',22 'puppeteer.browserfetcher.localrevisions',23 'puppeteer.browserfetcher.platform',24 'puppeteer.browserfetcher.product',25 'puppeteer.browserfetcher.remove',26 'puppeteer.browserfetcher.revisioninfo',27 ],28 "Browser": [29 'puppeteer.browser',30 'puppeteer.browser.browsercontexts',31 'puppeteer.browser.close',32 'puppeteer.browser.createincognitobrowsercontext',33 'puppeteer.browser.defaultbrowsercontext',34 'puppeteer.browser.disconnect',35 'puppeteer.browser.isconnected',36 'puppeteer.browser.newpage',37 'puppeteer.browser.pages',38 'puppeteer.browser.process',39 'puppeteer.browser.target',40 'puppeteer.browser.targets',41 'puppeteer.browser.useragent',42 'puppeteer.browser.waitfortarget',43 'puppeteer.browser.wsendpoint',44 ],45 "BrowserContext": [46 'puppeteer.browsercontext',47 'puppeteer.browsercontext.browser',48 'puppeteer.browsercontext.overridepermissions',49 'puppeteer.browsercontext.close',50 'puppeteer.browsercontext.isincognito',51 'puppeteer.browsercontext.newpage',52 'puppeteer.browsercontext.overridepermissions',53 'puppeteer.browsercontext.pages',54 'puppeteer.browsercontext.targets',55 'puppeteer.browsercontext.waitfortarget',56 ],57 "WebWorker": [58 'puppeteer.webworker',59 'puppeteer.webworker.evaluate',60 'puppeteer.webworker.evaluatehandle',61 'puppeteer.webworker.executioncontext',62 'puppeteer.webworker.url',63 ],64 "Accessibility": [65 'puppeteer.accessibility',66 'puppeteer.accessibility.snapshot',67 ],68 "FileChooser": [69 'puppeteer.filechooser',70 'puppeteer.filechooser.accept',71 'puppeteer.filechooser.cancel',72 'puppeteer.filechooser.ismultiple',73 ],74 "ExecutionContext": [75 'puppeteer.executioncontext',76 'puppeteer.executioncontext.evaluate',77 'puppeteer.executioncontext.evaluatehandle',78 'puppeteer.executioncontext.frame',79 'puppeteer.executioncontext.queryobjects',80 ],81 "JSHandle": [82 'puppeteer.jshandle',83 'puppeteer.jshandle.aselement',84 'puppeteer.jshandle.dispose',85 'puppeteer.jshandle.evaluate',86 'puppeteer.jshandle.evaluatehandle',87 'puppeteer.jshandle.executioncontext',88 'puppeteer.jshandle.getproperties',89 'puppeteer.jshandle.getproperty',90 'puppeteer.jshandle.jsonvalue',91 ],92 "ElementHandle": [93 'puppeteer.elementhandle',94 'puppeteer.elementhandle._',95 'puppeteer.elementhandle.__',96 'puppeteer.elementhandle.__eval',97 'puppeteer.elementhandle._eval',98 'puppeteer.elementhandle._x',99 'puppeteer.elementhandle.aselement',100 'puppeteer.elementhandle.boundingbox',101 'puppeteer.elementhandle.boxmodel',102 'puppeteer.elementhandle.click',103 'puppeteer.elementhandle.contentframe',104 'puppeteer.elementhandle.focus',105 'puppeteer.elementhandle.hover',106 'puppeteer.elementhandle.clickablepoint',107 'puppeteer.elementhandle.drag',108 'puppeteer.elementhandle.draganddrop',109 'puppeteer.elementhandle.dragenter',110 'puppeteer.elementhandle.dragover',111 'puppeteer.elementhandle.drop',112 'puppeteer.elementhandle.isintersectingviewport',113 'puppeteer.elementhandle.press',114 'puppeteer.elementhandle.screenshot',115 'puppeteer.elementhandle.select',116 'puppeteer.elementhandle.tap',117 'puppeteer.elementhandle.type',118 'puppeteer.elementhandle.uploadfile',119 ],120 "HTTPRequest": [121 'puppeteer.httprequest',122 'puppeteer.httprequest.abort',123 'puppeteer.httprequest.continue',124 'puppeteer.httprequest.failure',125 'puppeteer.httprequest.frame',126 'puppeteer.httprequest.headers',127 'puppeteer.httprequest.isnavigationrequest',128 'puppeteer.httprequest.method',129 'puppeteer.httprequest.postdata',130 'puppeteer.httprequest.redirectchain',131 'puppeteer.httprequest.resourcetype',132 'puppeteer.httprequest.respond',133 'puppeteer.httprequest.response',134 'puppeteer.httprequest.url',135 ],136 "HTTPRespose": [137 'puppeteer.httpresponse',138 'puppeteer.httpresponse.buffer',139 'puppeteer.httpresponse.frame',140 'puppeteer.httpresponse.fromcache',141 'puppeteer.httpresponse.fromserviceworker',142 'puppeteer.httpresponse.headers',143 'puppeteer.httpresponse.json',144 'puppeteer.httpresponse.ok',145 'puppeteer.httpresponse.remoteaddress',146 'puppeteer.httpresponse.request',147 'puppeteer.httpresponse.securitydetails',148 'puppeteer.httpresponse.status',149 'puppeteer.httpresponse.statustext',150 'puppeteer.httpresponse.text',151 'puppeteer.httpresponse.url',152 ],153 "SecurityDetails": [154 'puppeteer.securitydetails',155 'puppeteer.securitydetails.issuer',156 'puppeteer.securitydetails.protocol',157 'puppeteer.securitydetails.subjectalternativenames',158 'puppeteer.securitydetails.subjectname',159 'puppeteer.securitydetails.validfrom',160 'puppeteer.securitydetails.validto',161 ],162 "Target": [163 'puppeteer.target',164 'puppeteer.target.browser',165 'puppeteer.target.browsercontext',166 'puppeteer.target.createcdpsession',167 'puppeteer.target.opener',168 'puppeteer.target.page',169 'puppeteer.target.type',170 'puppeteer.target.url',171 'puppeteer.target.worker',172 ],173 "CDPSession": [174 'puppeteer.cdpsession',175 'puppeteer.cdpsession.connection',176 'puppeteer.cdpsession.detach',177 'puppeteer.cdpsession.send',178 ],179 "Coverage": [180 'puppeteer.coverage',181 'puppeteer.coverage.startcsscoverage',182 'puppeteer.coverage.startjscoverage',183 'puppeteer.coverage.stopcsscoverage',184 'puppeteer.coverage.stopjscoverage',185 ],186 "TimeOutError": [187 'puppeteer.timeouterror'188 ],189 "EventEmitter": [190 'puppeteer.eventemitter',191 'puppeteer.eventemitter.addlistener',192 'puppeteer.eventemitter.emit',193 'puppeteer.eventemitter.addlistener',194 'puppeteer.eventemitter.listenercount',195 'puppeteer.eventemitter.off',196 'puppeteer.eventemitter.on',197 'puppeteer.eventemitter.once',198 'puppeteer.eventemitter.removelistener',199 'puppeteer.eventemitter.removealllisteners',200 ],201 },...
QueryHandler.js
Source:QueryHandler.js
...139exports.customQueryHandlerNames = customQueryHandlerNames;140/**141 * @internal142 */143function clearCustomQueryHandlers() {144 customQueryHandlerNames().forEach(unregisterCustomQueryHandler);145}146exports.clearCustomQueryHandlers = clearCustomQueryHandlers;147/**148 * @internal149 */150function getQueryHandlerAndSelector(selector) {151 const hasCustomQueryHandler = /^[a-zA-Z]+\//.test(selector);152 if (!hasCustomQueryHandler)153 return { updatedSelector: selector, queryHandler: _defaultHandler };154 const index = selector.indexOf('/');155 const name = selector.slice(0, index);156 const updatedSelector = selector.slice(index + 1);157 const queryHandler = _queryHandlers.get(name);...
Puppeteer.js
Source:Puppeteer.js
...125 }126 /**127 * Clears all registered handlers.128 */129 clearCustomQueryHandlers() {130 clearCustomQueryHandlers();131 }...
index.js
Source:index.js
...101}102// async function like(id) {103// try {104// const url = 'https://padlet.com/chithuong70/uahr2skx76gww651';105// puppeteer.clearCustomQueryHandlers()106// const browser = await puppeteer.launch()107// const page = await browser.newPage()108// await page.goto(url, {109// waitUntil: 'networkidle2',110// })111// const work = await page.evaluate(async (id) => {112// a = document.querySelectorAll(`#${id.id} section .cursor-pointer`)[0];113// if (a) {114// a.click()115// b = document.querySelectorAll(`#${id.id} section .cursor-pointer span`)[0]116// return b.innerText117// }118// return a119// }, { id })120// console.log(`work[${id}] =>`, work)121// await browser.close()122// await sleep(Math.floor(Math.random() * 1000))123// } catch (error) {124// console.log("error", error)125// }126// }127// async function comment(id) {128// try {129// const url = 'https://padlet.com/chithuong70/uahr2skx76gww651';130// puppeteer.clearCustomQueryHandlers()131// const browser = await puppeteer.launch()132// const page = await browser.newPage()133// await page.goto(url)134// const work = await page.evaluate(async () => {135// a = document.querySelectorAll(`#${id} section .cursor-pointer`)[0];136// if (a) {137// return a.click()138// }139// return false140// })141// console.log("work =>", work)142// await browser.close()143// await sleep(Math.floor(Math.random() * 100))144// console.log(i)...
calenderAPI.js
Source:calenderAPI.js
1//read heavy2// start time, end time, name 3const { clearCustomQueryHandlers } = require("puppeteer");4//insert events 5// return ordered collection ordered by start time, ascending, then end time ascending , then by name 6//Questions7// When you say read heavy, you just want the whole thing right?8//can i just assume a double for now, and replace with time9class Calendar {10 eventList;11 //constructor should initialize an empty list12 constructor(){13 this.eventList = [];14 }15 //insert an event 16 //Because it is read heavy, we are going to keep the list ordered17 insert(start, end, name){18 let newEvent = new Event(start, end, name);19 20 let added = false;21 for(let i=0; i<this.eventList.length; i++){22 let currentEvent = this.eventList[i];23 //traverse the list until we find that the next event start time is greater or equal to newEvent24 if(currentEvent.startTime >= newEvent.startTime){25 26 //if not equal, add before27 if(currentEvent.startTime != newEvent.startTime){28 this.eventList.splice(i, 0, newEvent);29 added = true;30 break;31 }32 //start time is equal. compare the end time33 else if(currentEvent.endTime > newEvent.endTime){34 console.log('new event end time is smaller')35 this.eventList.splice(i, 0, newEvent);36 added = true;37 break;38 }39 else if(currentEvent.endTime == newEvent.endTime){40 //check name41 if(currentEvent.name >= newEvent.name){42 43 this.eventList.splice(i, 0, newEvent);44 added = true;45 }46 else{47 continue;48 }49 }50 //end time for event is less than new event -> go to next event51 else{52 continue;53 }54 }55 56 57 }58 if(!added){59 this.eventList.push(newEvent)60 console.log('added')61 }62 }63 view(){64 return this.eventList;65 }66 delete(name){67 for(let i =0; i< this.eventList.length; i++){68 if(this.eventList[i].name == name){69 this.eventList.splice(i,1);70 return true;71 }72 }73 return false;74 }75 eventsForPage(pageSize, pageNum){76 let eventItemStart = ((pageNum-1) * pageSize ) 77 78 if(eventItemStart >= this.eventList.length){79 return [];80 }81 else if(eventItemStart + pageSize >= this.eventList.length){82 pageSize = this.eventList.length - eventItemStart;83 }84 85 return this.eventList.splice(eventItemStart, pageSize);86 }87 88}89class Event {90 startTime;91 endTime;92 name; 93 constructor(startTime, endTime, name){94 this.startTime = startTime;95 this.endTime = endTime;96 this.name = name;97 }98}99let calendar = new Calendar();100// calendar.insert(3, 8, 'a');101// calendar.insert(3,5, 'b');102// calendar.insert(4,6, 'c')103// console.log(calendar.view());104// console.log(calendar.delete('a'));105// console.log(calendar.view());106calendar.insert(1,2,'a');107console.log(calendar.view());108calendar.insert(3,4,'b');109calendar.insert(4,5,'c');110calendar.insert(6,7,'d');111calendar.insert(8,9,'e');112console.log(calendar.view());...
payments.js
Source:payments.js
1// const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY);2const stripe = require("stripe")(3 "sk_test_51IhcBtCueTz3N4RSPKazGPYXmvuzwys5QazLWAgAWbNIqEngyHpl7bjfPBlsdUqGWhilPXrFFTYtVGJmH7YO8o7C00ZvDeQw8V"4);5const { clearCustomQueryHandlers } = require("puppeteer");6const uuid = require("uuid/v4");7const order = require("../models/order");8const user = require("../models/user");9const Payment = require("../models/payment");10const { OrderDetails } = require("./orders");11exports.orderPayment = async (req, res) => {12 const { products, token, order_id } = req.body;13 const idempotencyKey = uuid();14 // console.log(token);15 try {16 const { userid, total } = (17 await order.findById(order_id, "userid total").exec()18 )._doc;19 const { email } = (await user.findById(userid, "email").exec())._doc;20 const customer = await stripe.customers.create({21 email: token.email,22 source: token.id,23 });24 const charge = await stripe.charges.create(25 {26 amount: total * 100,27 currency: "usd",28 customer: customer.id,29 receipt_email: token.email,30 description: `order payment for amount of ${total} done by the user with id ${userid} on ${new Date()}`,31 },32 { idempotencyKey }33 );34 console.log(charge);35 const _payment = new Payment({36 payment_id: charge.id,37 amount: charge.amount / 100,38 currency: charge.currency,39 stripe_customer_id: charge.customer,40 user_id: userid,41 payment_for: "order",42 order_deposit_id: order_id,43 description: charge.description,44 status: charge.outcome.network_status,45 card: {46 brand: charge.payment_method_details.card.brand,47 country: charge.payment_method_details.card.country,48 exp_month: charge.payment_method_details.card.exp_month,49 exp_year: charge.payment_method_details.card.exp_year,50 funding: charge.payment_method_details.card.funding,51 },52 email: charge.receipt_email,53 receipt_url: charge.receipt_url,54 });55 const payment_done = await _payment.save();56 if (payment_done) {57 return res.status(200).json({58 message: "Payment has been successfull",59 });60 }61 } catch (error) {62 console.log(error);63 return res.status(400).json({64 message: "There has been an error while processing the payment",65 error,66 });67 }...
Using AI Code Generation
1const puppeteer = require('puppeteer');2(async () => {3 const browser = await puppeteer.launch();4 const page = await browser.newPage();5 await page.setRequestInterception(true);6 page.on('request', request => {7 if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))8 request.abort();9 request.continue();10 });11 await browser.close();12})();
Using AI Code Generation
1const puppeteer = require('puppeteer');2(async () => {3 const browser = await puppeteer.launch({headless: false});4 const page = await browser.newPage();5 await page.type('input[type="text"]', 'Hello World');6 await page.keyboard.press('Enter');7 await page.waitForNavigation();8 await page.waitForSelector('input[type="text"]');9 await page.type('input[type="text"]', 'Hello World');10 await page.keyboard.press('Enter');11 await page.waitForNavigation();12 await page.waitForSelector('input[type="text"]');13 await page.type('input[type="text"]', 'Hello World');14 await page.keyboard.press('Enter');15 await page.waitForNavigation();16 await page.waitForSelector('input[type="text"]');17 await page.type('input[type="text"]', 'Hello World');18 await page.keyboard.press('Enter');19 await page.waitForNavigation();20 await page.waitForSelector('input[type="text"]');21 await page.type('input[type="text"]', 'Hello World');22 await page.keyboard.press('Enter');23 await page.waitForNavigation();24 await page.waitForSelector('input[type="text"]');25 await page.type('input[type="text"]', 'Hello World');26 await page.keyboard.press('Enter');27 await page.waitForNavigation();28 await page.waitForSelector('input[type="text"]');29 await page.type('input[type="text"]', 'Hello World');30 await page.keyboard.press('Enter');31 await page.waitForNavigation();32 await page.waitForSelector('input[type="text"]');33 await page.type('input[type="text"]', 'Hello World');34 await page.keyboard.press('Enter');35 await page.waitForNavigation();36 await page.waitForSelector('input[type="text"]');37 await page.type('input[type="text"]', 'Hello World');38 await page.keyboard.press('Enter');39 await page.waitForNavigation();40 await page.waitForSelector('input[type="text"]');41 await page.type('input[type="text"]', 'Hello World');42 await page.keyboard.press('Enter');43 await page.waitForNavigation();44 await page.waitForSelector('input[type="text"]');45 await page.type('input[type="text"]', 'Hello World');46 await page.keyboard.press('Enter');47 await page.waitForNavigation();48 await page.waitForSelector('input[type="text"]');49 await page.type('input[type="text"]',
Using AI Code Generation
1const puppeteer = require('puppeteer');2(async () => {3 const browser = await puppeteer.launch();4 const page = await browser.newPage();5 await page.setRequestInterception(true);6 await page.on('request', (request) => {7 if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))8 request.abort();9 request.continue();10 });11 await page.waitForSelector('img');12 await page.setRequestInterception(false);13 await page.waitFor(3000);14 await browser.close();15})();16const puppeteer = require('puppeteer');17(async () => {18 const browser = await puppeteer.launch();19 const page = await browser.newPage();20 await page.setRequestInterception(true);21 await page.on('request', (request) => {22 if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))23 request.abort();24 request.continue();25 });26 await page.waitForSelector('img');27 await page.setRequestInterception(false);28 await page.waitFor(3000);29 await browser.close();30})();31const puppeteer = require('puppeteer');32(async () => {33 const browser = await puppeteer.launch();34 const page = await browser.newPage();35 await page.setRequestInterception(true);36 await page.on('request', (request) => {37 if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))38 request.abort();39 request.continue();40 });41 await page.waitForSelector('img');42 await page.setRequestInterception(false);43 await page.waitFor(3000);44 await browser.close();45})();46const puppeteer = require('puppeteer');47(async () => {48 const browser = await puppeteer.launch();49 const page = await browser.newPage();50 await page.setRequestInterception(true);51 await page.on('request', (request) => {52 if (request.url().endsWith('.png') || request.url().endsWith('.jpg'))
Using AI Code Generation
1const puppeteer = require('puppeteer');2(async () => {3 const browser = await puppeteer.launch();4 const page = await browser.newPage();5 await page.setRequestInterception(true);6 await page.on('request', request => {7 console.log('Intercepted request: ' + request.url());8 request.continue();9 });10 await page.on('requestfinished', request => {11 console.log('Request finished: ' + request.url());12 });13 await page.on('requestfailed', request => {14 console.log('Request failed: ' + request.url());15 });16 await page.clearCustomQueryHandlers();17 await browser.close();18})();19const puppeteer = require('puppeteer');20(async () => {21 const browser = await puppeteer.launch();22 const page = await browser.newPage();23 await page.setRequestInterception(true);24 await page.on('request', request => {25 console.log('Intercepted request: ' + request.url());26 request.continue();27 });28 await page.on('requestfinished', request => {29 console.log('Request finished: ' + request.url());30 });31 await page.on('requestfailed', request => {32 console.log('Request failed: ' + request.url());33 });34 await page.setRequestInterception(false);35 await browser.close();36})();37const puppeteer = require('puppeteer');38(async () => {39 const browser = await puppeteer.launch();40 const page = await browser.newPage();41 await page.setRequestInterception(true);42 await page.on('request', request => {43 console.log('Intercepted request: ' + request.url());44 request.continue();45 });46 await page.on('request
Using AI Code Generation
1const puppeteer = require('puppeteer');2(async () => {3 const browser = await puppeteer.launch({ headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox'] });4 const page = await browser.newPage();5 await page.setRequestInterception(true);6 page.on('request', request => {7 if (request.resourceType() === 'image')8 request.abort();9 request.continue();10 });11 await page.click('input[name="q"]');12 await page.keyboard.type('Puppeteer');13 await page.keyboard.press('Enter');14 await page.waitForSelector('div[id="search"]');15 await page.setRequestInterception(false);16 await browser.close();17})();18const puppeteer = require('puppeteer');19(async () => {20 const browser = await puppeteer.launch({ headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox'] });21 const page = await browser.newPage();22 await page.setRequestInterception(true);23 page.on('request', request => {24 if (request.resourceType() === 'image')25 request.abort();26 request.continue();27 });28 await page.click('input[name="q"]');29 await page.keyboard.type('Puppeteer');30 await page.keyboard.press('Enter');31 await page.waitForSelector('div[id="search"]');32 await page.setRequestInterception(false);33 await browser.close();34})();35const puppeteer = require('puppeteer');36(async () => {37 const browser = await puppeteer.launch({ headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox'] });38 const page = await browser.newPage();39 await page.setRequestInterception(true);40 page.on('request', request => {41 if (request.resourceType() === 'image')42 request.abort();43 request.continue();44 });45 await page.click('input
Using AI Code Generation
1const puppeteer = require('puppeteer');2(async () => {3 const browser = await puppeteer.launch();4 const page = await browser.newPage();5 await page.evaluate(() => {6 window.customQueryHandlers = {7 custom: () => document.querySelector('div'),8 };9 });10 await page.evaluate(() => {11 window.customQueryHandlers = {};12 });13 await browser.close();14})();15const puppeteer = require('puppeteer');16(async () => {17 const browser = await puppeteer.launch();18 const page = await browser.newPage();19 await page.evaluate(() => {20 window.customQueryHandlers = {21 custom: () => document.querySelector('div'),22 };23 });24 await page.evaluate(() => {25 delete window.customQueryHandlers;26 });27 await browser.close();28})();29const puppeteer = require('puppeteer');30(async () => {31 const browser = await puppeteer.launch();32 const page = await browser.newPage();33 await page.evaluate(() => {34 window.customQueryHandlers = {35 custom: () => document.querySelector('div'),36 };37 });38 await page.evaluate(() => {39 window.customQueryHandlers = undefined;40 });41 await browser.close();42})();43const puppeteer = require('puppeteer');44(async () => {45 const browser = await puppeteer.launch();46 const page = await browser.newPage();47 await page.evaluate(() => {48 window.customQueryHandlers = {49 custom: () => document.querySelector('div'),50 };51 });52 await page.evaluate(() => {53 window.customQueryHandlers = null;54 });55 await browser.close();56})();57const puppeteer = require('puppeteer');58(async () => {59 const browser = await puppeteer.launch();60 const page = await browser.newPage();61 await page.evaluate(() =>
Using AI Code Generation
1const puppeteer = require('puppeteer');2(async () => {3 const browser = await puppeteer.launch();4 const page = await browser.newPage();5 await page.waitForSelector('input[title="Search"]');6 await page.type('input[title="Search"]', 'Puppeteer');7 await page.keyboard.press('Enter');8 await page.waitForNavigation();9 await page.waitForSelector('h3');10 await page.evaluate(() => {11 window.clearCustomQueryHandlers();12 });13 await page.waitForSelector('h3');14 await page.close();15 await browser.close();16})();
Using AI Code Generation
1const { PuppeteerCrawler } = require('apify');2const crawler = new PuppeteerCrawler({3 handlePageFunction: async () => {},4 handleFailedRequestFunction: async () => {},5 sessionPoolOptions: {6 sessionOptions: {7 },8 },9});10const { CheerioCrawler } = require('apify');11const crawler = new CheerioCrawler({12 handlePageFunction: async () => {},13 handleFailedRequestFunction: async () => {},14 sessionPoolOptions: {15 sessionOptions: {16 },17 },18});19const { PlaywrightCrawler } = require('apify');20const crawler = new PlaywrightCrawler({21 handlePageFunction: async () => {},22 handleFailedRequestFunction: async () => {},23 sessionPoolOptions: {24 sessionOptions: {25 },26 },27});28const { PuppeteerPool } = require('apify');29const pool = new PuppeteerPool({30 launchPuppeteerFunction: async () => {},31});
Using AI Code Generation
1const puppeteer = require('puppeteer');2const browser = await puppeteer.launch({headless:false});3const page = await browser.newPage();4await page.setRequestInterception(true);5await page.on('request', (req) => {6 if (req.resourceType() == 'image') {7 req.abort();8 }9 else {10 req.continue();11 }12});13await page.screenshot({path: 'google.png'});14await page.setRequestInterception(false);15await page.screenshot({path: 'google.png'});16await browser.close();
Using AI Code Generation
1page.setCustomQueryHandler(name, queryHandler)2await page.setCustomQueryHandler('newQueryHandler', (name, options) => {3});4page.customQueryHandler(name, options)5await page.customQueryHandler('newQueryHandler', {6});7page.clearCustomQueryHandlers(name)8await page.clearCustomQueryHandlers('newQueryHandler');
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run Puppeteer automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
