Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use fsExtra.pathExists method in Cypress

Best JavaScript code snippet using cypress

pantiler.js

Source:pantiler.js

1import Util from 'util'2import FSExtra from 'fs-extra'3import * as Zod from 'zod'4import Scramjet from 'scramjet'5import Axios from 'axios'6import Unzipper from 'unzipper'7import Fontnik from 'fontnik'8import Gdal from 'gdal-next'9import Spritezero from '@mapbox/spritezero'10import Tippecanoe from './tippecanoe.js'11function setup(directory, cache = '.pantiler-cache', clearCache = false, bounds = null, alert = () => {}) {12    function validate(tiledata) {13        const inputSchema = Zod.object({14            name: Zod.string(),15            url: Zod.string().url().optional(), // otherwise use path16            path: Zod.string().optional(), // otherwise use url17            format: Zod.string().optional(),18            matching: Zod.string().optional()19        })20        const schema = Zod.object({21            host: Zod.string().url(),22            zoomFrom: Zod.number().positive(),23            zoomTo: Zod.number().positive(),24            fonts: Zod.array(inputSchema).optional(),25            sprites: Zod.array(inputSchema).optional(),26            sources: Zod.array(Zod.object({27                name: Zod.string(),28                system: Zod.string(),29                fieldLongitude: Zod.string().optional(),30                fieldLatitude: Zod.string().optional(),31                inputs: Zod.array(inputSchema),32                outputs: Zod.array(Zod.object({33                    name: Zod.string(),34                    layer: Zod.string().optional(),35                    filter: Zod.string().optional(),36                    fields: Zod.object({}).optional(),37                    zoomMin: Zod.number().optional(),38                    zoomMax: Zod.number().optional()39                }))40            })),41            styling: Zod.object({}) // not attempting to validate this42        })43        schema.parse(tiledata)44    }45    async function englyph(fonts) {46        await fonts.reduce(async (a, font) => {47            await a48            alert({49                process: 'Englyphing',50                input: font.name,51                message: 'in progress...'52            })53            const location = `${directory}/glyphs/${font.name}`54            await FSExtra.ensureDir(location)55            const data = font.path56                ? await FSExtra.readFile(font.path)57                : await (await Axios({ url: font.url, responseType: 'arraybuffer' })).data58            let ranges = []59            for (let i = 0; i < 65536; (i = i + 256)) {60                ranges.push({ start: i, end: Math.min(i + 255, 65535) })61            }62            const conversions = ranges.map(async range => {63                const result = await Util.promisify(Fontnik.range)({64                    font: data,65                    start: range.start,66                    end: range.end67                })68                return FSExtra.writeFile(`${location}/${range.start}-${range.end}.pbf`, result)69            })70            await Promise.all(conversions)71            alert({72                process: 'Englyphing',73                input: font.name,74                message: 'done'75            })76        }, Promise.resolve())77    }78    async function ensprite(sprites) {79        const ratios = [1, 2]80        await ratios.reduce(async (a, ratio) => {81            await a82            alert({83                process: 'Enspriting',84                input: `@${ratio}x`,85                message: 'in progress...'86            })87            const ratioAt =  ratio > 1 ? `@${ratio}x` : ''88            const images = sprites.map(async sprite => {89                const data = sprite.path90                    ? await FSExtra.readFile(sprite.path)91                    : await (await Axios({ url: sprite.url, responseType: 'arraybuffer' })).data92                return {93                    id: sprite.name,94                    svg: data95                }96            })97            const config = {98                imgs: await Promise.all(images),99                pixelRatio: ratio100            }101            const manifest = await Util.promisify(Spritezero.generateLayout)({ ...config, format: true })102            await FSExtra.writeJson(`${directory}/sprites${ratioAt}.json`, manifest)103            const layout = await Util.promisify(Spritezero.generateLayout)({ ...config, format: false })104            const image = await Util.promisify(Spritezero.generateImage)(layout)105            await FSExtra.writeFile(`${directory}/sprites${ratioAt}.png`, image)106            alert({107                process: 'Enspriting',108                input: `@${ratio}x`,109                message: 'done'110            })111        }, Promise.resolve())112    }113    async function fetch(name, inputs) {114        const downloads = inputs.map(async input => {115            const inputSpecifier = inputs.length > 1 ? `-${input.name}` : ''116            if (!input.url && !input.path) throw new Error(`${name}${inputSpecifier}: need to specify either url or path`)117            const extension = input.format || (input.path || input.url).split('.').pop()118            const file = `${cache}/${name}${inputSpecifier}.${extension}`119            const fileExists = await FSExtra.pathExists(file)120            if (fileExists) {121                alert({122                    process: 'Fetching',123                    input: name + inputSpecifier,124                    message: 'using cache'125                })126                return {127                    name: input.name,128                    path: file,129                    ...(input.matching ? { matching: input.matching } : {})130                }131            }132            if (input.path) {133                await FSExtra.ensureSymlink(input.path, file)134                alert({135                    process: 'Fetching',136                    input: name + inputSpecifier,137                    message: 'linked'138                })139                return {140                    name: input.name,141                    path: file,142                    ...(input.matching ? { matching: input.matching } : {})143                }144            }145            alert({146                process: 'Fetching',147                input: name + inputSpecifier,148                message: 'in progress...'149            })150            const response = await Axios({151                url: input.url,152                responseType: 'stream'153            })154            const writer = response.data.pipe(FSExtra.createWriteStream(file))155            await new Promise((resolve, reject) => {156                writer.on('error', reject)157                writer.on('close', resolve)158            })159            alert({160                process: 'Fetching',161                input: name + inputSpecifier,162                message: 'done'163            })164            return {165                name: input.name,166                path: file,167                ...(input.matching ? { matching: input.matching } : {})168            }169        })170        return Promise.all(downloads)171    }172    async function extract(name, archives) {173        const extractions = archives.map(async archive => {174            if (!archive.path.endsWith('zip')) {175                return archive // no extraction needed176            }177            const archiveSpecifier = archives.length > 1 ? `-${archive.name}` : ''178            const zip = await Unzipper.Open.file(archive.path)179            const entries = zip.files.filter(entry => {180                if (entry.type !== 'File') return false181                if (entry.path.match(/\.(pdf|txt)$/)) return false182                if (archive.matching && !entry.path.match(new RegExp(archive.matching))) return false183                return true184            })185            const extractions = entries.map(async entry => {186                const extension = entry.path.split('.').pop()187                const file = `${cache}/${name}${archiveSpecifier}.${extension}`188                const fileExists = await FSExtra.pathExists(file)189                if (fileExists) {190                    alert({191                        process: 'Extracting',192                        input: name + archiveSpecifier,193                        ...(entries.length > 1 ? { output: name + (entries.length > 1 ? `/${extension}` : '') } : {}),194                        message: 'using cache'195                    })196                    return { name: archive.name, path: file }197                }198                alert({199                    process: 'Extracting',200                    input: name + archiveSpecifier,201                    ...(entries.length > 1 ? { output: name + (entries.length > 1 ? `/${extension}` : '') } : {}),202                    message: 'in progress...'203                })204                const writer = entry.stream().pipe(FSExtra.createWriteStream(file))205                await new Promise((resolve, reject) => {206                    writer.on('error', reject)207                    writer.on('finish', resolve)208                })209                alert({210                    process: 'Extracting',211                    input: name + archiveSpecifier,212                    ...(entries.length > 1 ? { output: name + (entries.length > 1 ? `/${extension}` : '') } : {}),213                    message: 'done'214                })215                return { name: archive.name, path: file }216            })217            const extracted = await Promise.all(extractions)218            if (extracted.length === 1) return extracted[0]219            else if (extracted.find(file => file.path.endsWith('shp'))) return extracted.find(file => file.path.endsWith('shp'))220            else throw new Error(`${name}${archiveSpecifier}: archive has multiple files, you need to specify a input matching`)221        })222        return Promise.all(extractions)223    }224    async function convert(name, system, fieldLongitude, fieldLatitude, inputs, outputs) {225        const reprojection = new Gdal.CoordinateTransformation(Gdal.SpatialReference.fromProj4(system), Gdal.SpatialReference.fromProj4('+init=epsg:4326'))226        return outputs.reduce(async (previousOutput, output) => {227            await previousOutput228            const outputSpecifier = outputs.length > 1 ? `-${output.name}` : ''229            const file = `${cache}/${name}${outputSpecifier}.geo.json`230            const fileExists = await FSExtra.pathExists(file)231            if (fileExists) {232                alert({233                    process: 'Converting',234                    input: name,235                    ...(outputs.length > 1 || inputs.length > 1 ? { output: name + outputSpecifier } : {}),236                    message: 'using cache'237                })238                return239            }240            const outputData = Gdal.open(file, 'w', 'GeoJSON')241            const outputLayer = outputData.layers.create(`${name}${outputSpecifier}`, null, Gdal.wkbUnknown)242            const outputFieldDefinitions = Object.keys(output.fields || {}).map(key => {243                return new Gdal.FieldDefn(key, Gdal.OFTString)244            })245            outputLayer.fields.add(outputFieldDefinitions)246            inputs.forEach(input => {247                alert({248                    process: 'Converting',249                    input: name + (inputs.length > 1 ? `-${input.name}` : ''),250                    ...(outputs.length > 1 || inputs.length > 1 ? { output: name + outputSpecifier } : {}),251                    message: 'in progress...'252                })253                const inputData = Gdal.open(input.path)254                const inputLayer = output.filter255                    ? inputData.executeSQL(`select * from "${output.layer || inputData.layers.get(0).name}" where ${output.filter}`) // produces correct results, unlike using setAttributeFilter()256                    : inputData.layers.get(output.layer || 0)257                if (bounds) {258                    const reprojectionReverse = new Gdal.CoordinateTransformation(Gdal.SpatialReference.fromProj4('+init=epsg:4326'), Gdal.SpatialReference.fromProj4(system))259                    const min = reprojectionReverse.transformPoint(bounds[0], bounds[1])260                    const max = reprojectionReverse.transformPoint(bounds[2], bounds[3])261                    inputLayer.setSpatialFilter(min.x, min.y, max.x, max.y)262                }263                inputLayer.features.forEach(feature => {264                    const outputFeature = new Gdal.Feature(outputLayer)265                    const outputFields = Object.entries(output.fields || {}).map(([key, value]) => {266                        try {267                            return [key, feature.fields.get(value)]268                        }269                        catch (e) {270                            return [key, null]271                        }272                    })273                    outputFeature.fields.set(Object.fromEntries(outputFields))274                    const outputGeometry = (fieldLongitude && fieldLatitude)275                        ? Gdal.Geometry.fromGeoJson({ type: 'Point', coordinates: [Number(feature.fields.get(fieldLongitude)), Number(feature.fields.get(fieldLatitude))] })276                        : feature.getGeometry()277                    outputGeometry.transform(reprojection) // mutates in-place278                    outputFeature.setGeometry(outputGeometry)279                    outputLayer.features.add(outputFeature)280                })281                alert({282                    process: 'Converting',283                    input: name + (inputs.length > 1 ? `-${input.name}` : ''),284                    ...(outputs.length > 1 || inputs.length > 1 ? { output: name + outputSpecifier } : {}),285                    message: 'done'286                })287            })288            outputData.close()289            if (output.zoomMin || output.zoomMax) {290                const collection = await FSExtra.readJson(file)291                const tippecanoe = {292                    ...(output.zoomMin ? { minzoom: output.zoomMin } : {}),293                    ...(output.zoomMax ? { maxzoom: output.zoomMax } : {})294                }295                const features = collection.features.map(feature => ({ ...feature, tippecanoe }))296                await FSExtra.writeJson(file, { ...collection, features })297            }298        }, Promise.resolve())299    }300    async function tile(sources, zoomFrom, zoomTo) {301        alert({ process: 'Tiling', message: 'in progress...' })302        const sourcelist = sources.flatMap(source => {303            return source.outputs.map(output => {304                const outputSpecifier = source.outputs.length > 1 ? `-${output.name}` : ''305                return `named-layer=${output.name}:${cache}/${source.name}${outputSpecifier}.geo.json`306            })307        })308        const options = [309            `minimum-zoom=${zoomFrom}`,310            `maximum-zoom=${zoomTo}`,311            `output-to-directory=${directory}`,312            ...(bounds ? [`clip-bounding-box=${bounds.join(',')}`] : []),313            'generate-ids',314            'no-tile-compression',315            ...sourcelist316        ]317        await Tippecanoe(options, status => {318            alert({ process: 'Tiling', message: `in progress... ${status}` })319        })320        const metadata = await FSExtra.readJson(`${directory}/metadata.json`)321        await FSExtra.remove(`${directory}/metadata.json`)322        alert({ process: 'Tiling', message: 'done' })323        return metadata324    }325    async function style(metadata, styling, host, hasFonts, hasSprites) {326        const styles = {327            version: 8,328            metadata: {329                date: new Date().toISOString()330            },331            ...(hasFonts ? { glyphs: `${host}/glyphs/{fontstack}/{range}.pbf` } : {}),332            ...(hasSprites ? { sprite: `${host}/sprites` } : {}),333            sources: {334                primary: {335                    type: 'vector',336                    tiles: [`${host}/{z}/{x}/{y}.pbf`],337                    bounds: metadata.bounds.split(',').map(Number),338                    minzoom: Number(metadata.minzoom),339                    maxzoom: Number(metadata.maxzoom)340                }341            },342            ...styling343        }344        alert({ process: 'Styling', message: 'done' })345        return FSExtra.writeJson(`${directory}/style.json`, styles)346    }347    async function cleanup(clearCache) {348        if (!clearCache) return349        await FSExtra.remove(cache)350        alert({ process: 'Cleaning up', message: 'done' })351    }352    async function run(tiledata) {353        const directoryExists = await FSExtra.exists(directory)354        if (directoryExists) throw new Error('directory already exists')355        validate(tiledata)356        await FSExtra.ensureDir(directory)357        await FSExtra.ensureDir(cache)358        if (tiledata.fonts) await englyph(tiledata.fonts)359        if (tiledata.sprites) await ensprite(tiledata.sprites)360        await Scramjet.DataStream.from(tiledata.sources).each(async source => {361            const archives = await fetch(source.name, source.inputs)362            const inputs = await extract(source.name, archives)363            return convert(source.name, source.system, source.fieldLongitude, source.fieldLatitude, inputs, source.outputs)364        }).whenEnd()365        const metadata = await tile(tiledata.sources, tiledata.zoomFrom, tiledata.zoomTo)366        await style(metadata, tiledata.styling, tiledata.host, tiledata.fonts?.length > 0, tiledata.sprites?.length > 0)367        await cleanup(clearCache)368        alert({ message: 'Done!' })369    }370    return run371}...

index.js

Source:index.js

1#!/usr/bin/env node2const updateChk = require('../lib/update')3const downloadTpl = require('../lib/download')4const compileStr = require('../lib/compile')5const program = require('commander')6const inquirer = require('inquirer')7const fsextra = require('fs-extra')8const path = require('path')9const ora = require('ora')10const chalk = require('chalk')11const symbols = require('log-symbols')12const shell = require('child_process').exec13// version14program15  .version(require('../package.json').version, '-v, --version')16// upgrade17program18	.command('upgrade')19	.description("Check the VuePress-Creator's version.")20	.option('-t, --templete', 'Upgrade the VuePress templete version.')21	.action(() => {22    let _length = process.argv.slice(2).length23		if ( _length > 2) {24			program.outputHelp()25		} else {26      switch (_length){27        case 1:28          if (process.argv[2] == 'upgrade') {29            updateChk()30          }31          break32        case 2:33          if (process.argv[3] == '-t') {34            const templatePath = path.resolve(__dirname, '../template')35            fsextra.emptyDir(templatePath).then(() => {36              downloadTpl(templatePath)37            }).catch(err => {38              console.error(err)39            })40          }41          break42        default:43          program.outputHelp()44      }45		}46	})47// default templete48program49	.command('def <project>')50	.description('Initialize the default template for VuePress.')51	.action(project => {52		fsextra53			.pathExists(project)54			.then(exists => {55				if (!exists) {56					const templatePath = path.resolve(__dirname, '../template')57          const processPath = process.cwd()58          const targetPath = `${processPath}/${project}`59					fsextra.pathExists(templatePath).then(exists => {60						if (exists) {61							compileAction(project,templatePath,targetPath,{theme:null}).then(() => {62                showInfoAction(project)63              })64						} else {65							fsextra.emptyDir(templatePath).then(() => {66                downloadTpl(templatePath).then(() => {67                  compileAction(project,templatePath,targetPath,{theme:null}).then(() => {68                    showInfoAction(project)69                  })70                })71              }).catch(err => {72                console.error(err)73              })74						}75					})76				} else {77					console.log(symbols.error, chalk.red('The project already exists.'))78				}79			}).catch(err => {80				console.error(err)81			})82	})83// antdocs theme84program85	.command('ads <project>')86	.description('Initialize the template with AntDocs theme for VuePress.')87	.action(project => {88    fsextra89    .pathExists(project)90    .then(exists => {91      if (!exists) {92        const templatePath = path.resolve(__dirname, '../template')93        const processPath = process.cwd()94        const targetPath = `${processPath}/${project}`95        const stylePath = `${processPath}/${project}/docs/.vuepress/styles`96        fsextra.pathExists(templatePath).then(exists => {97          if (exists) {98            compileAction(project,templatePath,targetPath,{theme:'antdocs'}).then(() => {99              installThemeAction(project).then(()=>{100                fsextra.emptyDir(stylePath)101                showInfoAction(project)102              })103            })104          } else {105            fsextra.emptyDir(templatePath).then(() => {106              downloadTpl(templatePath).then(() => {107                compileAction(project,templatePath,targetPath,{theme:'antdocs'}).then(() => {108                  installThemeAction(project).then(()=>{109                    fsextra.emptyDir(stylePath)110                    showInfoAction(project)111                  })112                })113              })114            }).catch(err => {115              console.error(err)116            })117          }118        })119      } else {120        console.log(symbols.error, chalk.red('The project already exists.'))121      }122    }).catch(err => {123      console.error(err)124    })125  })126// init127program128	.name('vuepress-creator')129	.usage('<commands> [options]')130	.command('init <project>')131	.description('Create a VuePress project.')132	.action(project => {133    fsextra.pathExists(project).then(exists => {134      if (!exists) {135        inquirer.prompt([{136          type:'list',137          name:'theme',138          message:'Pick a theme for VuePress?',139          choices: [140            'Default',141            'AntDocs (Ant Design style)'142          ],143          default: 'Default'144        }]).then(answers => {145          let _theme = answers.theme == 'Default' ? null:'antdocs'146          const templatePath = path.resolve(__dirname, '../template')147          const processPath = process.cwd()148          const targetPath = `${processPath}/${project}`149          const stylePath = `${processPath}/${project}/docs/.vuepress/styles`150          switch (_theme){151            case null:152              fsextra.pathExists(templatePath).then(exists => {153                if (exists) {154                  compileAction(project,templatePath,targetPath,{theme:null}).then(() => {155                    showInfoAction(project)156                  })157                } else {158                  fsextra.emptyDir(templatePath).then(() => {159                    downloadTpl(templatePath).then(() => {160                      compileAction(project,templatePath,targetPath,{theme:null}).then(() => {161                        showInfoAction(project)162                      })163                    })164                  }).catch(err => {165                    console.error(err)166                  })167                }168              })169              break170            case 'antdocs':171              fsextra.pathExists(templatePath).then(exists => {172                if (exists) {173                  compileAction(project,templatePath,targetPath,{theme:'antdocs'}).then(() => {174                    installThemeAction(project).then(()=>{175                      fsextra.emptyDir(stylePath)176                      showInfoAction(project)177                    })178                  })179                } else {180                  fsextra.emptyDir(templatePath).then(() => {181                    downloadTpl(templatePath).then(() => {182                      compileAction(project,templatePath,targetPath,{theme:'antdocs'}).then(() => {183                        installThemeAction(project).then(()=>{184                          fsextra.emptyDir(stylePath)185                          showInfoAction(project)186                        })187                      })188                    })189                  }).catch(err => {190                    console.error(err)191                  })192                }193              })194              break195          }196        })        197      } else {198        console.log(symbols.error, chalk.red('The project already exists.'))199      }200    })201  })202program.on('--help', function() {203	console.log('')204	console.log('Examples:')205	console.log('  $ vuepress-creator init project')206	console.log('  $ vuepress-creator upgrade -t')207	console.log('')208})209program.parse(process.argv)210function compileAction(prjName,tplPath,tgrPath,cfgGather){211  return new Promise(function(resolve, reject) {212    fsextra.copy(tplPath, tgrPath).then(() => {213      const configPath = `${prjName}/docs/.vuepress/config.js`214      const configGather = cfgGather215      compileStr(configPath, configGather).then(() => {216        resolve()217      }).catch(err => {218        return err219      })220    }).catch(err => {221      return reject(err)222    })223  })224}225function installThemeAction(prjName){226  return new Promise(function(resolve, reject) {227    const installSpinner = ora(chalk.cyan('Install AntDocs theme...'))228    installSpinner.start()229    shell(`cd ${prjName} && npm i vuepress-theme-antdocs`, (err, stdout, stderr) => {230      if (err) {231        installSpinner.text = 'Install AntDocs theme failed.'232        installSpinner.fail()233        return reject(err)234      }235      installSpinner.text = 'Install AntDocs theme successful.'236      installSpinner.succeed()237      resolve()238    })239  })240}241function showInfoAction(prjName){242  console.log('')243  console.log('    To get started:')244  console.log('')245  console.log(chalk.yellow(`      cd ${prjName}`))246  console.log(`      ${chalk.yellow('npm install')} or ${chalk.yellow('yarn install')}`)247  console.log(`      ${chalk.yellow('npm run dev')} or ${chalk.yellow('yarn run dev')}`)248  console.log('')...

reconcile.js

Source:reconcile.js

1import Crypto from 'crypto'2import FSExtra from 'fs-extra'3import Scramjet from 'scramjet'4import Axios from 'axios'5import AxiosRetry from 'axios-retry'6import AxiosRateLimit from 'axios-rate-limit'7import FormData from 'form-data'8import Querystring from 'querystring'9function request(retries, cache, verbose, alert, limit, messages) {10    const cacheDirectory = '.reconcile-cache'11    const timeout = 45 * 100012    const toUrl = location => typeof location === 'string' ? location : location.url13    const toLocationName = location => {14        if (!location) throw new Error('Request location is blank')15        const method = location.method ? location.method.toUpperCase() : 'GET'16        const stringifyObject = object => Object.entries(object).map(([key, value]) => `${key}=${JSON.stringify(value)}`).join(' ')17        return `${method} ${toUrl(location)}`18            + (location.params ? '?' + Querystring.stringify(location.params) : '')19            + (location.dataQuery ? ' ' + Querystring.stringify(location.dataQuery) : '')20            + (location.dataForm ? ' <' + stringifyObject(location.dataForm) + '>' : '')21            + (location.data && !location.dataQuery && !location.dataForm ? ' [' + stringifyObject(location.data) + ']' : '')22    }23    const toErrorMessage = e => {24        const reconcilerError = e.response && messages(e)25        if (reconcilerError) return reconcilerError // look for reconciler-specific errors first26        const locationName = toLocationName(e.config)27        if (e.response) return `Received code ${e.response.status}: ${locationName}` // response recieved, but non-2xx28        if (e.code === 'ECONNABORTED') return `Timed out after ${timeout / 1000}ms: ${locationName}` // request timed out29        if (e.code) return `Error ${e.code}: ${locationName}` // request failed, with error code30        return e.message // request not made31    }32    const instance = Axios.create({ timeout })33    AxiosRetry(instance, {34        retries,35        shouldResetTimeout: true,36        retryCondition: e => {37            return !e.response || e.response.status >= 500 || e.response.status === 429 // no response, server error, or hit rate limit38        },39        retryDelay: (number, e) => {40            const message = toErrorMessage(e)41            const attempt = number > 0 && number <= retries && retries > 1 ? ' (retrying' + (number > 1 ? `, attempt ${number}` : '') + '...)' : ''42            if (number === 1) alert(`${message}${attempt}`)43            else alert(`  â ${message}${attempt}`)44            return 5 * 100045        }46    })47    AxiosRateLimit(instance, {48        maxRequests: limit, // so limit is number of requests per second49        perMilliseconds: 1 * 100050    })51    let cacheChecked = false52    return async location => {53        const hash = Crypto.createHash('sha1').update(JSON.stringify(typeof location === 'string' ? location : { ...location, auth: null })).digest('hex')54        if (location.dataQuery) {55            location.data = Querystring.stringify(location.dataQuery)56        }57        if (location.dataForm) {58            const form = new FormData()59            location.headers = form.getHeaders()60            Object.entries(location.dataForm).forEach(([key, value]) => form.append(key, JSON.stringify(value)))61            location.data = form62        }63        const locationName = toLocationName(location)64        if (cache) {65            if (!cacheChecked) {66                const cacheExists = await FSExtra.pathExists(cacheDirectory)67                if (cacheExists) alert('Cached data found!')68                else alert('No existing cached data found')69                cacheChecked = true70            }71            const isCached = await FSExtra.pathExists(`${cacheDirectory}/${hash}`)72            if (isCached) {73                if (verbose) alert(`Cached [${hash}]: ${locationName}`)74                const cacheData = await FSExtra.readJson(`${cacheDirectory}/${hash}`)75                return {76                    url: toUrl(location),77                    data: cacheData,78                    passthrough: location.passthrough79                }80            }81        }82        try {83            if (verbose) alert(`Requesting: ${locationName}`)84            const response = await instance(location)85            if (cache) {86                await FSExtra.ensureDir(cacheDirectory)87                await FSExtra.writeJson(`${cacheDirectory}/${hash}`, response.data)88            }89            return {90                url: toUrl(location),91                data: response.data,92                passthrough: location.passthrough93            }94        }95        catch (e) {96            alert(toErrorMessage(e))97        }98    }99}100async function load(command, filename, parameters = {}, retries = 5, cache = false, join = 'inner', verbose = false, alert = () => {}) {101    const die = message => {102        throw new Error(message)103    }104    const requestor = request.bind(null, retries, cache, verbose, alert)105    const { default: reconciler } = await import(`./reconcilers/${command}.js`)106    Object.keys(parameters).forEach(parameter => {107        if (!reconciler.details.parameters.find(p => p.name === parameter)) alert(`Ignoring unexpected parameter '${parameter}'`)108    })109    const batch = reconciler.details.batch || 1110    const execute = reconciler.initialise(parameters, requestor, die)111    const source = () => Scramjet.StringStream.from(FSExtra.createReadStream(filename)).CSVParse({ header: true })112    const columnsReconciler = reconciler.details.columns.map(column => column.name)113    const columnsSource = Object.keys((await source().slice(0, 1).toArray())[0])114    const columnMapEntries = columnsReconciler.map(column => {115        const columnUnique = (i = '') => {116            const attempt = `${column}${i}`117            if (columnsSource.find(name => name === attempt)) return columnUnique(Number(i) + 1)118            if (i) alert(`Column '${column}' from the reconciler has been renamed '${attempt}' so it does not overwrite the source`)119            return attempt120        }121        return [column, columnUnique()]122    })123    const columnMap = Object.fromEntries(columnMapEntries)124    const blank = Object.fromEntries(Object.values(columnMap).map(key => [key]))125    const length = async () => {126        const entries = await source().reduce(a => a + 1, 0)127        return Math.ceil(entries / batch)128    }129    const run = async () => source().batch(batch).setOptions({ maxParallel: 1 }).map(async items => {130        try {131            const executed = await execute(batch === 1 ? items[0] : items)132            return items.flatMap((item, i) => {133                const results = batch > 1 && Array.isArray(executed[i]) ? executed[i] // batch mode, reconciler is one-to-many134                    : batch > 1 ? [executed[i]] // batch mode, reconciler is one-to-one135                    : Array.isArray(executed) ? executed // reconciler is one-to-many136                    : [executed] // reconciler is one-to-one137                if (join === 'outer' && results.length === 0) return [{ ...item, ...blank }]138                return results.map(result => {139                    const resultRemapped = result140                        ? Object.fromEntries(Object.entries(result).map(([column, value]) => [columnMap[column], value]))141                        : Object.fromEntries(Object.entries(columnMap).map(([column]) => [column, ''])) // if there is no result (eg. not found)142                    return { ...item, ...resultRemapped }143                })144            })145        }146        catch (e) {147            alert(verbose ? e.stack : e.message)148            if (join === 'outer') {149                return items.map(item => ({ ...item, ...blank }))150            }151            else return []152        }153    })154    return { run, length }155}...

executor.js

Source:executor.js

1import Process from 'process'2import WorkerThreads from 'worker_threads'3import ObjectHash from 'object-hash'4import FSExtra from 'fs-extra'5async function sourcing() {6    const { method, ...settings } = watch.source // omit method name7    const { default: f } = await import(`./../methods/source-${watch.source.method}.js`)8    const items = await f(settings)9    return items.map(content => {10        const id = ObjectHash(content)11        return { id, content }12    })13}14async function diffing(items) {15    const key = ObjectHash(watch.source)16    const path = `.newsagent-cache/${key}`17    const hash = ObjectHash(items)18    const hashset = items.map(item => item.id)19    const cachefileExists = await FSExtra.pathExists(path)20    if (!cachefileExists) {21        await FSExtra.writeJson(path, { hash, hashset, amalgam: items })22        return []23    }24    const cachefile = await FSExtra.readJson(path)25    if (cachefile.hash === hash) return [] // whole-data hash is the same, so we can avoid looking at individual records26    const additions = () => {27        return items.filter(item => !cachefile.hashset.includes(item.id)).map(item => {28            return { ...item, difference: 'addition' }29        })30    }31    const removals = () => {32        return cachefile.hashset.filter(hash => !hashset.includes(hash)).map(hash => {33            const item = cachefile.amalgam.find(item => item.id === hash)34            return { ...item, difference: 'removal' }35        })36    }37    const changesAdditions = watch.monitor === 'additions-and-removals' || watch.monitor === 'additions-only' ? additions() : []38    const changesRemovals = watch.monitor === 'additions-and-removals' || watch.monitor === 'removals-only' ? removals() : []39    const amalgam = cachefile.amalgam40        .filter(item => !changesRemovals.find(change => change.id === item.id))41        .concat(changesAdditions.map(change => ({ id: change.id, content: change.content }))) // so omit difference field42    await FSExtra.writeJson(path, { hash, hashset, amalgam })43    return [...changesAdditions, ...changesRemovals]44}45async function processing(changes) {46    if (!watch.processes) return changes47    return watch.processes.reduce(async (a, process) => {48        const { method, ...settings } = process // omit method name49        const { default: f } = await import(`./../methods/process-${process.method}.js`)50        const all = await Promise.all((await a).map(async change => {51            const contentNew = await f(change.content, change.difference, settings)52            return { ...change, content: contentNew }53        }))54        return all.filter(x => x.content)55    }, changes)56}57async function alerting(results) {58    const firings = watch.alerts.map(async alert => {59        const { method, ...settings } = alert // omit method name60        const { default: f } = await import(`./../methods/alert-${alert.method}.js`)61        results.forEach(result => f(watch.name, result.content, settings))62    })63    await Promise.all(firings)64}65async function execute() {66    try {67        const items = await sourcing()68        const changes = await diffing(items)69        const processed = await processing(changes)70        await alerting(processed)71    }72    catch (e) {73        if (e.constructor.name === 'ZodError') {74            const error = e.errors[0]75            WorkerThreads.parentPort.postMessage({76                event: 'execution-invalid',77                data: { error: `${error.message}: ${error.path.join('.')} wanted ${error.expected} but got ${error.received}` }78            })79        }80        else {81            WorkerThreads.parentPort.postMessage({82                event: 'execution-failure',83                data: { error: e.message }84            })85        }86        Process.exit(1)87    }88}...

csv-fetch.js

Source:csv-fetch.js

1import FSExtra from 'fs-extra'2import Scramjet from 'scramjet'3import Axios from 'axios'4import AxiosRetry from 'axios-retry'5import AxiosRateLimit from 'axios-rate-limit'6function fetcher(urlColumn, nameColumn, depository, suffix, headers, limit, retries, check, verbose, alert) {7    const timeout = 45 * 10008    const toErrorMessage = e => {9        const locationName = e.config.url10        if (e.response) return `Received code ${e.response.status}: ${locationName}` // response recieved, but non-2xx11        if (e.code === 'ECONNABORTED') return `Timed out after ${timeout / 1000}ms: ${locationName}` // request timed out12        if (e.code) return `Error ${e.code}: ${locationName}` // request failed, with error code13        return e.message // request not made14    }15    const instance = Axios.create({ timeout })16    AxiosRetry(instance, {17        retries,18        shouldResetTimeout: true,19        retryCondition: e => {20            return !e.response || e.response.status >= 500 || e.response.status === 429 // no response, server error, or hit rate limit21        },22        retryDelay: (number, e) => {23            const message = toErrorMessage(e)24            const attempt = number > 0 && number <= retries && retries > 1 ? ' (retrying' + (number > 1 ? `, attempt ${number}` : '') + '...)' : ''25            if (number === 1) alert(`${message}${attempt}`)26            else alert(`  â ${message}${attempt}`)27            return 5 * 100028        }29    })30    AxiosRateLimit(instance, {31        maxRequests: limit, // so limit is number of requests per second32        perMilliseconds: 1 * 100033    })34    return async row => {35        const key = row[nameColumn]36        if (!key) {37            alert('Key column is empty!')38            return39        }40        const url = row[urlColumn]41        if (!url) {42            alert('URL column is empty!')43            return44        }45        const filename = key + (suffix || '')46        if (check) {47            const exists = await FSExtra.pathExists(`${depository}/${filename}`)48            if (exists && verbose) {49                alert(`Exists [${filename}]: ${url}`)50                return51            }52        }53        try {54            const headersValues = headers ? Object.fromEntries(headers.map(header => header.split(/: ?/))) : {}55            if (verbose) alert(`Requesting: ${url}` + (headersValues ? ' ' + JSON.stringify(headersValues, null, 2) : ''))56            const response = await instance({57                url,58                headers: headersValues,59                responseType: 'arraybuffer'60            })61            await FSExtra.writeFile(`${depository}/${filename}`, response.data)62        }63        catch (e) {64            alert(toErrorMessage(e))65        }66    }67}68function source(filename) {69    return Scramjet.StringStream.from(FSExtra.createReadStream(filename)).CSVParse({ header: true })70}71function length(filename) {72    return source(filename).reduce(a => a + 1, 0)73}74async function run(filename, urlColumn, nameColumn, depository, suffix, headers, limit, retries, check, verbose, alert) {75    await FSExtra.ensureDir(depository)76    const fetch = fetcher(urlColumn, nameColumn, depository, suffix, headers, limit, retries, check, verbose, alert)77    return source(filename).each(fetch)78}...

cli.js

Source:cli.js

1import Readline from 'readline'2import FSExtra from 'fs-extra'3import Process from 'process'4import Yargs from 'yargs'5import Progress from 'progress'6import csvFetch from './csv-fetch.js'7function alert(message) {8    Readline.clearLine(process.stderr)9    Readline.cursorTo(process.stderr, 0)10    console.error(message)11}12function ticker(text, total) {13    const progress = new Progress(text + ' |:bar| :percent / :etas left', {14        total,15        width: Infinity,16        complete: 'â',17        incomplete: ' '18    })19    return () => progress.tick()20}21async function setup() {22    const instructions = Yargs(Process.argv.slice(2))23        .usage('Usage: csv-fetch <url-column> <name-column> <depository> <filename>')24        .wrap(null)25        .option('s', { alias: 'suffix', type: 'string', describe: 'A suffix to add to the name of each file, such as an extension' })26        .option('h', { alias: 'header', type: 'string', array: true, nargs: 1, describe: 'Set a header to be sent with the request' })27        .option('l', { alias: 'limit', type: 'number', nargs: 1, describe: 'Limit the number of requests made per second' })28        .option('r', { alias: 'retries', type: 'number', nargs: 1, describe: 'Number of times a request should be retried', default: 5 })29        .option('c', { alias: 'check', type: 'boolean', describe: 'Check whether file has already been downloaded, and skip if so', default: false })30        .option('V', { alias: 'verbose', type: 'boolean', describe: 'Print every request made', default: false })31        .help('?').alias('?', 'help')32        .version().alias('v', 'version')33    if (instructions.argv._.length === 0) instructions.showHelp().exit(0)34    try {35        const {36            _: [urlColumn, nameColumn, depository, filename],37            suffix,38            header: headers,39            limit,40            retries,41            check,42            verbose43        } = instructions.argv44        if (filename === '-') throw new Error('reading from standard input not supported')45        const exists = await FSExtra.pathExists(filename)46        if (!exists) throw new Error(`${filename}: could not find file`)47        if (headers) headers.forEach(header => {48            if (!header.includes(':')) throw new Error(`"${header}" header is not valid`)49        })50        const total = await csvFetch.length(filename)51        console.error('Starting up...')52        const process = await csvFetch.run(filename, urlColumn, nameColumn, depository, suffix, headers, limit, retries, check, verbose, alert)53        await process54            .each(ticker('Working...', total))55            .whenEnd()56        console.error('Done!')57    }58    catch (e) {59        console.error(e.message)60        Process.exit(1)61    }62}...

compile.js

Source:compile.js

1const chalk = require('chalk')2const fsextra = require('fs-extra')3const handlebars = require('handlebars')4const symbols = require('log-symbols')5function compileStr(filePath, answersContent) {6  return new Promise(function(resolve, reject) {7		fsextra.pathExists(filePath).then(exists => {8      if (exists) {9        const _filePath = filePath10        const _Content = fsextra.readFileSync(_filePath).toString()11        const _Result = handlebars.compile(_Content)(answersContent)12        fsextra.writeFileSync(_filePath, _Result)13        console.log(symbols.success, `${chalk.cyan(filePath)} configured successful.`)14        resolve()15      } else {16        console.log(symbols.error, `${chalk.red(filePath)} not found.`)17        return reject()18      }19    }).catch(err => {20      console.log(symbols.error, chalk.red(err))21    })22  })23}...

Using AI Code Generation

1Cypress.Commands.add('pathExists', (path) => {2  return cy.task('pathExists', path)3})4module.exports = (on, config) => {5  on('task', {6    pathExists(path) {7      return fsExtra.pathExists(path)8    }9  })10}11Cypress.Commands.add('pathExists', (path) => {12  return cy.task('pathExists', path)13})14require('./commands')15describe('test', () => {16  it('test', () => {17    cy.pathExists('test')18  })19})20{21}

Using AI Code Generation

1const fsExtra = require('fs-extra');2describe('fsExtra.pathExists', () => {3  it('should return true for existing file', () => {4    fsExtra.pathExists('cypress.json').should('be.true');5  });6  it('should return false for non-existing file', () => {7    fsExtra.pathExists('non-existing-file.json').should('be.false');8  });9});

Using AI Code Generation

1const fsExtra = require('fs-extra');2const path = require('path');3const pathToTest = path.resolve(__dirname, './cypress/integration/MyTest.spec.js');4fsExtra.pathExists(pathToTest).then((exists) => {5  console.log(exists ? 'Path exists' : 'Path does not exist');6});7cy.get('#identifierId').type('

Using AI Code Generation

1describe('Cypress fsExtra', function() {2  it('fsExtra.pathExists', function() {3    cy.task('pathExists', 'path/to/file')4      .then((exists) => {5      })6  })7})8const fsExtra = require('fs-extra')9module.exports = (on) => {10  on('task', {11    pathExists(path) {12      return fsExtra.pathExists(path)13    },14  })15}16Cypress.Commands.add('pathExists', function(path) {17  return cy.task('pathExists', path)18})19describe('Cypress fsExtra', function() {20  it('fsExtra.pathExists', function() {21    cy.pathExists('path/to/file')22      .then((exists) => {23      })24  })25})

Using AI Code Generation

1const fsExtra = require('fs-extra');2const path = require('path');3const filePath = path.resolve('cypress', 'fixtures', 'data.json');4fsExtra.pathExists(filePath).then((exists) => {5  if (exists) {6    console.log('File exists');7  } else {8    console.log('File does not exist');9  }10});11describe('My First Test', () => {12  it('Does not do much!', () => {13    cy.contains('type').click();14    cy.url().should('include', '/commands/actions');15    cy.get('.action-email').type('

Cypress Tutorial

Cypress is a renowned Javascript-based open-source, easy-to-use end-to-end testing framework primarily used for testing web applications. Cypress is a relatively new player in the automation testing space and has been gaining much traction lately, as evidenced by the number of Forks (2.7K) and Stars (42.1K) for the project. LambdaTest’s Cypress Tutorial covers step-by-step guides that will help you learn from the basics till you run automation tests on LambdaTest.

Chapters:

What is Cypress? -
Why Cypress? - Learn why Cypress might be a good choice for testing your web applications.
Features of Cypress Testing - Learn about features that make Cypress a powerful and flexible tool for testing web applications.
Cypress Drawbacks - Although Cypress has many strengths, it has a few limitations that you should be aware of.
Cypress Architecture - Learn more about Cypress architecture and how it is designed to be run directly in the browser, i.e., it does not have any additional servers.
Browsers Supported by Cypress - Cypress is built on top of the Electron browser, supporting all modern web browsers. Learn browsers that support Cypress.
Selenium vs Cypress: A Detailed Comparison - Compare and explore some key differences in terms of their design and features.
Cypress Learning: Best Practices - Take a deep dive into some of the best practices you should use to avoid anti-patterns in your automation tests.
How To Run Cypress Tests on LambdaTest? - Set up a LambdaTest account, and now you are all set to learn how to run Cypress tests.

Certification

You can elevate your expertise with end-to-end testing using the Cypress automation framework and stay one step ahead in your career by earning a Cypress certification. Check out our Cypress 101 Certification.