How to use setExtraHTTPHeaders method in Puppeteer

Best JavaScript code snippet using puppeteer

fnguide.js

Source:fnguide.js Github

copy

Full Screen

...91 }92 async massDateCrawl() {93 const page = this.page;94 // set headers to fool Fnguide95 await page.setExtraHTTPHeaders({96 Referer: 'http://www.fnguide.com/fgdd/StkIndmByTime',97 'X-Requested-With': 'XMLHttpRequest',98 });99 const dateURL = URL.API.date.format(this.todayDate);100 await page.goto(dateURL);101 const dateData = await page.evaluate(() => {102 const data = JSON.parse(document.querySelector('body').innerText);103 return data;104 });105 return dateData;106 }107 async getKospiTickers(date) {108 const page = this.page;109 // set headers to fool Fnguide110 await page.setExtraHTTPHeaders({111 Referer: 'http://www.fnguide.com/fgdd/StkIndByTime',112 'X-Requested-With': 'XMLHttpRequest',113 });114 const kospiTickersURL = URL.API.kospi_tickers.format(date);115 await page.goto(kospiTickersURL);116 const kospiTickersData = await page.evaluate(() => {117 const data = JSON.parse(document.querySelector('body').innerText);118 return data119 });120 return kospiTickersData;121 }122 async getKosdaqTickers(date) {123 const page = this.page;124 // set headers to fool Fnguide125 await page.setExtraHTTPHeaders({126 Referer: 'http://www.fnguide.com/fgdd/StkIndByTime',127 'X-Requested-With': 'XMLHttpRequest',128 });129 const kosdaqTickersURL = URL.API.kosdaq_tickers.format(date);130 await page.goto(kosdaqTickersURL);131 const kosdaqTickersData = await page.evaluate(() => {132 const data = JSON.parse(document.querySelector('body').innerText);133 return data134 });135 return kosdaqTickersData;136 }137 async getETFTickers(date) {138 const page = this.page;139 // set headers to fool Fnguide140 await page.setExtraHTTPHeaders({141 Referer: 'http://fnguide.com/fgdd/StkEtf',142 'X-Requested-With': 'XMLHttpRequest',143 });144 const ETFTickersURL = URL.API.etf.format(date);145 await page.goto(ETFTickersURL);146 const ETFTickersData = await page.evaluate(() => {147 const data = JSON.parse(document.querySelector('body').innerText);148 return data149 });150 return ETFTickersData;151 }152 async getStockInfo() {153 const page = this.page;154 // set headers to fool Fnguide155 await page.setExtraHTTPHeaders({156 Referer: 'http://fnguide.com/fgdd/StkAllItemInfo',157 'X-Requested-With': 'XMLHttpRequest',158 });159 const stockInfoURL = URL.API.stock_info.format();160 await page.goto(stockInfoURL);161 const stockInfoData = await page.evaluate(() => {162 const data = JSON.parse(document.querySelector('body').innerText);163 return data164 });165 return stockInfoData;166 }167 async massIndexCrawl(date) {168 const page = this.page;169 // set headers to fool Fnguide170 await page.setExtraHTTPHeaders({171 Referer: 'http://www.fnguide.com/fgdd/StkIndByTime',172 'X-Requested-With': 'XMLHttpRequest',173 });174 // let indexURL = URL.API.index.format(this.todayDate)175 const indexURL = URL.API.index.format(date);176 await page.goto(indexURL);177 const indexData = await page.evaluate(() => {178 const data = JSON.parse(document.querySelector('body').innerText);179 return data;180 });181 return indexData;182 }183 async massETFCrawl(date) {184 const page = this.page;185 // set headers to fool Fnguide186 await page.setExtraHTTPHeaders({187 Referer: 'http://fnguide.com/fgdd/StkEtf',188 'X-Requested-With': 'XMLHttpRequest',189 });190 // let indexURL = URL.API.index.format(this.todayDate)191 const ETFURL = URL.API.etf.format(date);192 await page.goto(ETFURL);193 const ETFData = await page.evaluate(() => {194 const data = JSON.parse(document.querySelector('body').innerText);195 return data;196 });197 return ETFData;198 }199 async massOHLCVCrawl(date) {200 const page = this.page;201 // set headers to fool Fnguide202 await page.setExtraHTTPHeaders({203 Referer: 'http://fnguide.com/fgdd/StkIndByTime',204 'X-Requested-With': 'XMLHttpRequest',205 });206 // let indexURL = URL.API.index.format(this.todayDate)207 const ohlcvURL = URL.API.ohlcv.format(date);208 await page.goto(ohlcvURL);209 const ohlcvData = await page.evaluate(() => {210 const data = JSON.parse(document.querySelector('body').innerText);211 return data;212 });213 return ohlcvData;214 }215 async massMktCapCrawl(date) {216 const page = this.page;217 // set headers to fool Fnguide218 await page.setExtraHTTPHeaders({219 Referer: 'http://fnguide.com/fgdd/StkItemDateCap',220 'X-Requested-With': 'XMLHttpRequest',221 });222 // let indexURL = URL.API.index.format(this.todayDate)223 const MktCapURL = URL.API.mkt_cap.format(date);224 await page.goto(MktCapURL);225 const mktCapData = await page.evaluate(() => {226 const data = JSON.parse(document.querySelector('body').innerText);227 return data;228 });229 return mktCapData;230 }231 async massBuysellCrawl(date) {232 const page = this.page;233 // set headers to fool Fnguide234 await page.setExtraHTTPHeaders({235 Referer: 'http://fnguide.com/fgdd/StkJInvTrdTrend',236 'X-Requested-With': 'XMLHttpRequest',237 });238 const buysellURL = URL.API.buysell.format(date);239 await page.goto(buysellURL);240 const buysellData = await page.evaluate(() => {241 const data = JSON.parse(document.querySelector('body').innerText);242 return data;243 });244 return buysellData;245 }246 async massFactorCrawl(date) {247 const page = this.page;248 // set headers to fool Fnguide249 await page.setExtraHTTPHeaders({250 Referer: 'http://www.fnguide.com/fgdd/StkDateShareIndx',251 'X-Requested-With': 'XMLHttpRequest',252 });253 const factorURL = URL.API.factor.format(date);254 await page.goto(factorURL);255 const factorData = await page.evaluate(() => {256 const data = JSON.parse(document.querySelector('body').innerText);257 return data;258 });259 return factorData;260 }261 async done() {262 await this.browser.close();263 }...

Full Screen

Full Screen

pdf.js

Source:pdf.js Github

copy

Full Screen

...74 if (authenticate) {75 await page.authenticate(authenticate);76 }77 if (setExtraHTTPHeaders) {78 await page.setExtraHTTPHeaders(setExtraHTTPHeaders);79 }80 if (setJavaScriptEnabled !== null) {81 await page.setJavaScriptEnabled(setJavaScriptEnabled);82 }83 if (rejectRequestPattern.length || requestInterceptors.length) {84 await page.setRequestInterception(true);85 page.on('request', (req) => {86 if (rejectRequestPattern.find((pattern) => req.url().match(pattern))) {87 return req.abort();88 }89 const interceptor = requestInterceptors90 .find(r => req.url().match(r.pattern));91 if (interceptor) {92 return req.respond(interceptor.response);...

Full Screen

Full Screen

scrape.js

Source:scrape.js Github

copy

Full Screen

...72 if (authenticate) {73 await page.authenticate(authenticate);74 }75 if (setExtraHTTPHeaders) {76 await page.setExtraHTTPHeaders(setExtraHTTPHeaders);77 }78 if (rejectRequestPattern.length || requestInterceptors.length) {79 await page.setRequestInterception(true);80 page.on('request', (req) => {81 if (rejectRequestPattern.find((pattern) => req.url().match(pattern))) {82 return req.abort();83 }84 const interceptor = requestInterceptors85 .find(r => req.url().match(r.pattern));86 if (interceptor) {87 return req.respond(interceptor.response);88 }89 return req.continue();90 });...

Full Screen

Full Screen

screenshot.js

Source:screenshot.js Github

copy

Full Screen

...40 if (authenticate) {41 await page.authenticate(authenticate);42 }43 if (setExtraHTTPHeaders) {44 await page.setExtraHTTPHeaders(setExtraHTTPHeaders);45 }46 if (setJavaScriptEnabled !== null) {47 await page.setJavaScriptEnabled(setJavaScriptEnabled);48 }49 if (rejectRequestPattern.length || requestInterceptors.length) {50 await page.setRequestInterception(true);51 page.on('request', (req) => {52 if (rejectRequestPattern.find((pattern) => req.url().match(pattern))) {53 return req.abort();54 }55 const interceptor = requestInterceptors56 .find(r => req.url().match(r.pattern));57 if (interceptor) {58 return req.respond(interceptor.response);...

Full Screen

Full Screen

content.js

Source:content.js Github

copy

Full Screen

...33 if (authenticate) {34 await page.authenticate(authenticate);35 }36 if (setExtraHTTPHeaders) {37 await page.setExtraHTTPHeaders(setExtraHTTPHeaders);38 }39 if (setJavaScriptEnabled !== null) {40 await page.setJavaScriptEnabled(setJavaScriptEnabled);41 }42 if (rejectRequestPattern.length || requestInterceptors.length) {43 await page.setRequestInterception(true);44 page.on('request', (req) => {45 if (rejectRequestPattern.find((pattern) => req.url().match(pattern))) {46 return req.abort();47 }48 const interceptor = requestInterceptors49 .find(r => req.url().match(r.pattern));50 if (interceptor) {51 return req.respond(interceptor.response);...

Full Screen

Full Screen

test-puppet.js

Source:test-puppet.js Github

copy

Full Screen

1const puppeteer = require('puppeteer');2(async () => {3 const browser = await puppeteer.launch();4 const page = await browser.newPage();5 await page.setExtraHTTPHeaders({ Referer: 'https://sparktoro.com/' })6 await page.goto('https://support.servicenow.com/kb?id=kb_article_view&sysparm_article=KB0597477');7 await page.waitForSelector('.kb-permalink');8 const availablePatchLinks = await page.evaluate(() => {9 const links = Array.from(document.querySelectorAll('li > a'))10 console.log(links);11 var patchLinks = links.filter(link => {12 if (link.title.indexOf('Patch') >= 0) {13 return true;14 }15 })16 return patchLinks.map(link => {17 return {18 "href": link.href,19 "title": link.title20 }21 })22 })23 console.log(availablePatchLinks);24 /*let patches = {}25 availablePatchLinks.forEach(async (patch) => {26 patches[patch.title] = [];27 28 const browser2 = await puppeteer.launch();29 const page2 = await browser2.newPage();30 await page2.setExtraHTTPHeaders({ Referer: 'https://sparktoro.com/' })31 await page2.goto(patch.href);32 await page2.waitForSelector('.kb-permalink');33 await page2.evaluate(() => {34 const links = Array.from(document.querySelectorAll('li > a'))35 console.log(links);36 patches[patch.title] = links.map(link => {37 return {38 "href": link.href,39 "title": link.title40 }41 })42 })43 console.log(availablePatchLinks);44 })...

Full Screen

Full Screen

https_basic_auth.js

Source:https_basic_auth.js Github

copy

Full Screen

...4puppeteer.launch({headless: false}).then(async browser => {5 const page = await browser.newPage();6 //const auth = new Buffer(`${userName}:${password}`).toString('base64');7 // console.log(auth)8 // await page.setExtraHTTPHeaders({9 // 'Authorization': `Basic ${auth}`10 // });11 //12 // await page.setExtraHTTPHeaders(headers);13 await page.authenticate({username:userName, password:password});14 await page.goto('http://the-internet.herokuapp.com/basic_auth');15 const text = await page.evaluate(() => document.querySelector('.example').textContent);16 console.log(text)17 await browser.close()...

Full Screen

Full Screen

_tests.js

Source:_tests.js Github

copy

Full Screen

1import * as assert from 'uvu/assert';2/** @type {import('test').TestMaker} */3export default function (test) {4 test('can access host through page store', null, async ({ base, page }) => {5 page.setExtraHTTPHeaders({6 'x-forwarded-host': 'forwarded.com'7 });8 await page.goto(`${base}/host`);9 assert.equal(await page.textContent('h1'), 'forwarded.com');10 // reset11 page.setExtraHTTPHeaders({});12 });...

Full Screen

Full Screen

Using AI Code Generation

copy

Full Screen

1const puppeteer = require('puppeteer');2(async () => {3 const browser = await puppeteer.launch();4 const page = await browser.newPage();5 await page.setExtraHTTPHeaders({6 });7 await page.screenshot({ path: 'google.png' });8 await browser.close();9})();10const puppeteer = require('puppeteer');11(async () => {12 const browser = await puppeteer.launch();13 const page = await browser.newPage();14 await page.setExtraHTTPHeaders({15 });16 await page.screenshot({ path: 'google.png' });17 await browser.close();18})();19const puppeteer = require('puppeteer');20(async () => {21 const browser = await puppeteer.launch();22 const page = await browser.newPage();23 await page.setExtraHTTPHeaders({24 });25 await page.screenshot({ path: 'google.png' });26 await browser.close();27})();

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Puppeteer automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful