How to use fsExtra.pathExists method in Cypress

1import Util from 'util'2import FSExtra from 'fs-extra'3import * as Zod from 'zod'4import Scramjet from 'scramjet'5import Axios from 'axios'6import Unzipper from 'unzipper'7import Fontnik from 'fontnik'8import Gdal from 'gdal-next'9import Spritezero from '@mapbox/spritezero'10import Tippecanoe from './tippecanoe.js'11function setup(directory, cache = '.pantiler-cache', clearCache = false, bounds = null, alert = () => {}) {12    function validate(tiledata) {13        const inputSchema = Zod.object({14            name: Zod.string(),15            url: Zod.string().url().optional(), // otherwise use path16            path: Zod.string().optional(), // otherwise use url17            format: Zod.string().optional(),18            matching: Zod.string().optional()19        })20        const schema = Zod.object({21            host: Zod.string().url(),22            zoomFrom: Zod.number().positive(),23            zoomTo: Zod.number().positive(),24            fonts: Zod.array(inputSchema).optional(),25            sprites: Zod.array(inputSchema).optional(),26            sources: Zod.array(Zod.object({27                name: Zod.string(),28                system: Zod.string(),29                fieldLongitude: Zod.string().optional(),30                fieldLatitude: Zod.string().optional(),31                inputs: Zod.array(inputSchema),32                outputs: Zod.array(Zod.object({33                    name: Zod.string(),34                    layer: Zod.string().optional(),35                    filter: Zod.string().optional(),36                    fields: Zod.object({}).optional(),37                    zoomMin: Zod.number().optional(),38                    zoomMax: Zod.number().optional()39                }))40            })),41            styling: Zod.object({}) // not attempting to validate this42        })43        schema.parse(tiledata)44    }45    async function englyph(fonts) {46        await fonts.reduce(async (a, font) => {47            await a48            alert({49                process: 'Englyphing',50                input: font.name,51                message: 'in progress...'52            })53            const location = `${directory}/glyphs/${font.name}`54            await FSExtra.ensureDir(location)55            const data = font.path56                ? await FSExtra.readFile(font.path)57                : await (await Axios({ url: font.url, responseType: 'arraybuffer' })).data58            let ranges = []59            for (let i = 0; i < 65536; (i = i + 256)) {60                ranges.push({ start: i, end: Math.min(i + 255, 65535) })61            }62            const conversions = ranges.map(async range => {63                const result = await Util.promisify(Fontnik.range)({64                    font: data,65                    start: range.start,66                    end: range.end67                })68                return FSExtra.writeFile(`${location}/${range.start}-${range.end}.pbf`, result)69            })70            await Promise.all(conversions)71            alert({72                process: 'Englyphing',73                input: font.name,74                message: 'done'75            })76        }, Promise.resolve())77    }78    async function ensprite(sprites) {79        const ratios = [1, 2]80        await ratios.reduce(async (a, ratio) => {81            await a82            alert({83                process: 'Enspriting',84                input: `@${ratio}x`,85                message: 'in progress...'86            })87            const ratioAt =  ratio > 1 ? `@${ratio}x` : ''88            const images = sprites.map(async sprite => {89                const data = sprite.path90                    ? await FSExtra.readFile(sprite.path)91                    : await (await Axios({ url: sprite.url, responseType: 'arraybuffer' })).data92                return {93                    id: sprite.name,94                    svg: data95                }96            })97            const config = {98                imgs: await Promise.all(images),99                pixelRatio: ratio100            }101            const manifest = await Util.promisify(Spritezero.generateLayout)({ ...config, format: true })102            await FSExtra.writeJson(`${directory}/sprites${ratioAt}.json`, manifest)103            const layout = await Util.promisify(Spritezero.generateLayout)({ ...config, format: false })104            const image = await Util.promisify(Spritezero.generateImage)(layout)105            await FSExtra.writeFile(`${directory}/sprites${ratioAt}.png`, image)106            alert({107                process: 'Enspriting',108                input: `@${ratio}x`,109                message: 'done'110            })111        }, Promise.resolve())112    }113    async function fetch(name, inputs) {114        const downloads = inputs.map(async input => {115            const inputSpecifier = inputs.length > 1 ? `-${input.name}` : ''116            if (!input.url && !input.path) throw new Error(`${name}${inputSpecifier}: need to specify either url or path`)117            const extension = input.format || (input.path || input.url).split('.').pop()118            const file = `${cache}/${name}${inputSpecifier}.${extension}`119            const fileExists = await FSExtra.pathExists(file)120            if (fileExists) {121                alert({122                    process: 'Fetching',123                    input: name + inputSpecifier,124                    message: 'using cache'125                })126                return {127                    name: input.name,128                    path: file,129                    ...(input.matching ? { matching: input.matching } : {})130                }131            }132            if (input.path) {133                await FSExtra.ensureSymlink(input.path, file)134                alert({135                    process: 'Fetching',136                    input: name + inputSpecifier,137                    message: 'linked'138                })139                return {140                    name: input.name,141                    path: file,142                    ...(input.matching ? { matching: input.matching } : {})143                }144            }145            alert({146                process: 'Fetching',147                input: name + inputSpecifier,148                message: 'in progress...'149            })150            const response = await Axios({151                url: input.url,152                responseType: 'stream'153            })154            const writer = response.data.pipe(FSExtra.createWriteStream(file))155            await new Promise((resolve, reject) => {156                writer.on('error', reject)157                writer.on('close', resolve)158            })159            alert({160                process: 'Fetching',161                input: name + inputSpecifier,162                message: 'done'163            })164            return {165                name: input.name,166                path: file,167                ...(input.matching ? { matching: input.matching } : {})168            }169        })170        return Promise.all(downloads)171    }172    async function extract(name, archives) {173        const extractions = archives.map(async archive => {174            if (!archive.path.endsWith('zip')) {175                return archive // no extraction needed176            }177            const archiveSpecifier = archives.length > 1 ? `-${archive.name}` : ''178            const zip = await Unzipper.Open.file(archive.path)179            const entries = zip.files.filter(entry => {180                if (entry.type !== 'File') return false181                if (entry.path.match(/\.(pdf|txt)$/)) return false182                if (archive.matching && !entry.path.match(new RegExp(archive.matching))) return false183                return true184            })185            const extractions = entries.map(async entry => {186                const extension = entry.path.split('.').pop()187                const file = `${cache}/${name}${archiveSpecifier}.${extension}`188                const fileExists = await FSExtra.pathExists(file)189                if (fileExists) {190                    alert({191                        process: 'Extracting',192                        input: name + archiveSpecifier,193                        ...(entries.length > 1 ? { output: name + (entries.length > 1 ? `/${extension}` : '') } : {}),194                        message: 'using cache'195                    })196                    return { name: archive.name, path: file }197                }198                alert({199                    process: 'Extracting',200                    input: name + archiveSpecifier,201                    ...(entries.length > 1 ? { output: name + (entries.length > 1 ? `/${extension}` : '') } : {}),202                    message: 'in progress...'203                })204                const writer = entry.stream().pipe(FSExtra.createWriteStream(file))205                await new Promise((resolve, reject) => {206                    writer.on('error', reject)207                    writer.on('finish', resolve)208                })209                alert({210                    process: 'Extracting',211                    input: name + archiveSpecifier,212                    ...(entries.length > 1 ? { output: name + (entries.length > 1 ? `/${extension}` : '') } : {}),213                    message: 'done'214                })215                return { name: archive.name, path: file }216            })217            const extracted = await Promise.all(extractions)218            if (extracted.length === 1) return extracted[0]219            else if (extracted.find(file => file.path.endsWith('shp'))) return extracted.find(file => file.path.endsWith('shp'))220            else throw new Error(`${name}${archiveSpecifier}: archive has multiple files, you need to specify a input matching`)221        })222        return Promise.all(extractions)223    }224    async function convert(name, system, fieldLongitude, fieldLatitude, inputs, outputs) {225        const reprojection = new Gdal.CoordinateTransformation(Gdal.SpatialReference.fromProj4(system), Gdal.SpatialReference.fromProj4('+init=epsg:4326'))226        return outputs.reduce(async (previousOutput, output) => {227            await previousOutput228            const outputSpecifier = outputs.length > 1 ? `-${output.name}` : ''229            const file = `${cache}/${name}${outputSpecifier}.geo.json`230            const fileExists = await FSExtra.pathExists(file)231            if (fileExists) {232                alert({233                    process: 'Converting',234                    input: name,235                    ...(outputs.length > 1 || inputs.length > 1 ? { output: name + outputSpecifier } : {}),236                    message: 'using cache'237                })238                return239            }240            const outputData = Gdal.open(file, 'w', 'GeoJSON')241            const outputLayer = outputData.layers.create(`${name}${outputSpecifier}`, null, Gdal.wkbUnknown)242            const outputFieldDefinitions = Object.keys(output.fields || {}).map(key => {243                return new Gdal.FieldDefn(key, Gdal.OFTString)244            })245            outputLayer.fields.add(outputFieldDefinitions)246            inputs.forEach(input => {247                alert({248                    process: 'Converting',249                    input: name + (inputs.length > 1 ? `-${input.name}` : ''),250                    ...(outputs.length > 1 || inputs.length > 1 ? { output: name + outputSpecifier } : {}),251                    message: 'in progress...'252                })253                const inputData = Gdal.open(input.path)254                const inputLayer = output.filter255                    ? inputData.executeSQL(`select * from "${output.layer || inputData.layers.get(0).name}" where ${output.filter}`) // produces correct results, unlike using setAttributeFilter()256                    : inputData.layers.get(output.layer || 0)257                if (bounds) {258                    const reprojectionReverse = new Gdal.CoordinateTransformation(Gdal.SpatialReference.fromProj4('+init=epsg:4326'), Gdal.SpatialReference.fromProj4(system))259                    const min = reprojectionReverse.transformPoint(bounds[0], bounds[1])260                    const max = reprojectionReverse.transformPoint(bounds[2], bounds[3])261                    inputLayer.setSpatialFilter(min.x, min.y, max.x, max.y)262                }263                inputLayer.features.forEach(feature => {264                    const outputFeature = new Gdal.Feature(outputLayer)265                    const outputFields = Object.entries(output.fields || {}).map(([key, value]) => {266                        try {267                            return [key, feature.fields.get(value)]268                        }269                        catch (e) {270                            return [key, null]271                        }272                    })273                    outputFeature.fields.set(Object.fromEntries(outputFields))274                    const outputGeometry = (fieldLongitude && fieldLatitude)275                        ? Gdal.Geometry.fromGeoJson({ type: 'Point', coordinates: [Number(feature.fields.get(fieldLongitude)), Number(feature.fields.get(fieldLatitude))] })276                        : feature.getGeometry()277                    outputGeometry.transform(reprojection) // mutates in-place278                    outputFeature.setGeometry(outputGeometry)279                    outputLayer.features.add(outputFeature)280                })281                alert({282                    process: 'Converting',283                    input: name + (inputs.length > 1 ? `-${input.name}` : ''),284                    ...(outputs.length > 1 || inputs.length > 1 ? { output: name + outputSpecifier } : {}),285                    message: 'done'286                })287            })288            outputData.close()289            if (output.zoomMin || output.zoomMax) {290                const collection = await FSExtra.readJson(file)291                const tippecanoe = {292                    ...(output.zoomMin ? { minzoom: output.zoomMin } : {}),293                    ...(output.zoomMax ? { maxzoom: output.zoomMax } : {})294                }295                const features = collection.features.map(feature => ({ ...feature, tippecanoe }))296                await FSExtra.writeJson(file, { ...collection, features })297            }298        }, Promise.resolve())299    }300    async function tile(sources, zoomFrom, zoomTo) {301        alert({ process: 'Tiling', message: 'in progress...' })302        const sourcelist = sources.flatMap(source => {303            return source.outputs.map(output => {304                const outputSpecifier = source.outputs.length > 1 ? `-${output.name}` : ''305                return `named-layer=${output.name}:${cache}/${source.name}${outputSpecifier}.geo.json`306            })307        })308        const options = [309            `minimum-zoom=${zoomFrom}`,310            `maximum-zoom=${zoomTo}`,311            `output-to-directory=${directory}`,312            ...(bounds ? [`clip-bounding-box=${bounds.join(',')}`] : []),313            'generate-ids',314            'no-tile-compression',315            ...sourcelist316        ]317        await Tippecanoe(options, status => {318            alert({ process: 'Tiling', message: `in progress... ${status}` })319        })320        const metadata = await FSExtra.readJson(`${directory}/metadata.json`)321        await FSExtra.remove(`${directory}/metadata.json`)322        alert({ process: 'Tiling', message: 'done' })323        return metadata324    }325    async function style(metadata, styling, host, hasFonts, hasSprites) {326        const styles = {327            version: 8,328            metadata: {329                date: new Date().toISOString()330            },331            ...(hasFonts ? { glyphs: `${host}/glyphs/{fontstack}/{range}.pbf` } : {}),332            ...(hasSprites ? { sprite: `${host}/sprites` } : {}),333            sources: {334                primary: {335                    type: 'vector',336                    tiles: [`${host}/{z}/{x}/{y}.pbf`],337                    bounds: metadata.bounds.split(',').map(Number),338                    minzoom: Number(metadata.minzoom),339                    maxzoom: Number(metadata.maxzoom)340                }341            },342            ...styling343        }344        alert({ process: 'Styling', message: 'done' })345        return FSExtra.writeJson(`${directory}/style.json`, styles)346    }347    async function cleanup(clearCache) {348        if (!clearCache) return349        await FSExtra.remove(cache)350        alert({ process: 'Cleaning up', message: 'done' })351    }352    async function run(tiledata) {353        const directoryExists = await FSExtra.exists(directory)354        if (directoryExists) throw new Error('directory already exists')355        validate(tiledata)356        await FSExtra.ensureDir(directory)357        await FSExtra.ensureDir(cache)358        if (tiledata.fonts) await englyph(tiledata.fonts)359        if (tiledata.sprites) await ensprite(tiledata.sprites)360        await Scramjet.DataStream.from(tiledata.sources).each(async source => {361            const archives = await fetch(source.name, source.inputs)362            const inputs = await extract(source.name, archives)363            return convert(source.name, source.system, source.fieldLongitude, source.fieldLatitude, inputs, source.outputs)364        }).whenEnd()365        const metadata = await tile(tiledata.sources, tiledata.zoomFrom, tiledata.zoomTo)366        await style(metadata, tiledata.styling, tiledata.host, tiledata.fonts?.length > 0, tiledata.sprites?.length > 0)367        await cleanup(clearCache)368        alert({ message: 'Done!' })369    }370    return run371}...

1#!/usr/bin/env node2const updateChk = require('../lib/update')3const downloadTpl = require('../lib/download')4const compileStr = require('../lib/compile')5const program = require('commander')6const inquirer = require('inquirer')7const fsextra = require('fs-extra')8const path = require('path')9const ora = require('ora')10const chalk = require('chalk')11const symbols = require('log-symbols')12const shell = require('child_process').exec13// version14program15  .version(require('../package.json').version, '-v, --version')16// upgrade17program18	.command('upgrade')19	.description("Check the VuePress-Creator's version.")20	.option('-t, --templete', 'Upgrade the VuePress templete version.')21	.action(() => {22    let _length = process.argv.slice(2).length23		if ( _length > 2) {24			program.outputHelp()25		} else {26      switch (_length){27        case 1:28          if (process.argv[2] == 'upgrade') {29            updateChk()30          }31          break32        case 2:33          if (process.argv[3] == '-t') {34            const templatePath = path.resolve(__dirname, '../template')35            fsextra.emptyDir(templatePath).then(() => {36              downloadTpl(templatePath)37            }).catch(err => {38              console.error(err)39            })40          }41          break42        default:43          program.outputHelp()44      }45		}46	})47// default templete48program49	.command('def <project>')50	.description('Initialize the default template for VuePress.')51	.action(project => {52		fsextra53			.pathExists(project)54			.then(exists => {55				if (!exists) {56					const templatePath = path.resolve(__dirname, '../template')57          const processPath = process.cwd()58          const targetPath = `${processPath}/${project}`59					fsextra.pathExists(templatePath).then(exists => {60						if (exists) {61							compileAction(project,templatePath,targetPath,{theme:null}).then(() => {62                showInfoAction(project)63              })64						} else {65							fsextra.emptyDir(templatePath).then(() => {66                downloadTpl(templatePath).then(() => {67                  compileAction(project,templatePath,targetPath,{theme:null}).then(() => {68                    showInfoAction(project)69                  })70                })71              }).catch(err => {72                console.error(err)73              })74						}75					})76				} else {77					console.log(symbols.error, chalk.red('The project already exists.'))78				}79			}).catch(err => {80				console.error(err)81			})82	})83// antdocs theme84program85	.command('ads <project>')86	.description('Initialize the template with AntDocs theme for VuePress.')87	.action(project => {88    fsextra89    .pathExists(project)90    .then(exists => {91      if (!exists) {92        const templatePath = path.resolve(__dirname, '../template')93        const processPath = process.cwd()94        const targetPath = `${processPath}/${project}`95        const stylePath = `${processPath}/${project}/docs/.vuepress/styles`96        fsextra.pathExists(templatePath).then(exists => {97          if (exists) {98            compileAction(project,templatePath,targetPath,{theme:'antdocs'}).then(() => {99              installThemeAction(project).then(()=>{100                fsextra.emptyDir(stylePath)101                showInfoAction(project)102              })103            })104          } else {105            fsextra.emptyDir(templatePath).then(() => {106              downloadTpl(templatePath).then(() => {107                compileAction(project,templatePath,targetPath,{theme:'antdocs'}).then(() => {108                  installThemeAction(project).then(()=>{109                    fsextra.emptyDir(stylePath)110                    showInfoAction(project)111                  })112                })113              })114            }).catch(err => {115              console.error(err)116            })117          }118        })119      } else {120        console.log(symbols.error, chalk.red('The project already exists.'))121      }122    }).catch(err => {123      console.error(err)124    })125  })126// init127program128	.name('vuepress-creator')129	.usage('<commands> [options]')130	.command('init <project>')131	.description('Create a VuePress project.')132	.action(project => {133    fsextra.pathExists(project).then(exists => {134      if (!exists) {135        inquirer.prompt([{136          type:'list',137          name:'theme',138          message:'Pick a theme for VuePress?',139          choices: [140            'Default',141            'AntDocs (Ant Design style)'142          ],143          default: 'Default'144        }]).then(answers => {145          let _theme = answers.theme == 'Default' ? null:'antdocs'146          const templatePath = path.resolve(__dirname, '../template')147          const processPath = process.cwd()148          const targetPath = `${processPath}/${project}`149          const stylePath = `${processPath}/${project}/docs/.vuepress/styles`150          switch (_theme){151            case null:152              fsextra.pathExists(templatePath).then(exists => {153                if (exists) {154                  compileAction(project,templatePath,targetPath,{theme:null}).then(() => {155                    showInfoAction(project)156                  })157                } else {158                  fsextra.emptyDir(templatePath).then(() => {159                    downloadTpl(templatePath).then(() => {160                      compileAction(project,templatePath,targetPath,{theme:null}).then(() => {161                        showInfoAction(project)162                      })163                    })164                  }).catch(err => {165                    console.error(err)166                  })167                }168              })169              break170            case 'antdocs':171              fsextra.pathExists(templatePath).then(exists => {172                if (exists) {173                  compileAction(project,templatePath,targetPath,{theme:'antdocs'}).then(() => {174                    installThemeAction(project).then(()=>{175                      fsextra.emptyDir(stylePath)176                      showInfoAction(project)177                    })178                  })179                } else {180                  fsextra.emptyDir(templatePath).then(() => {181                    downloadTpl(templatePath).then(() => {182                      compileAction(project,templatePath,targetPath,{theme:'antdocs'}).then(() => {183                        installThemeAction(project).then(()=>{184                          fsextra.emptyDir(stylePath)185                          showInfoAction(project)186                        })187                      })188                    })189                  }).catch(err => {190                    console.error(err)191                  })192                }193              })194              break195          }196        })        197      } else {198        console.log(symbols.error, chalk.red('The project already exists.'))199      }200    })201  })202program.on('--help', function() {203	console.log('')204	console.log('Examples:')205	console.log('  $ vuepress-creator init project')206	console.log('  $ vuepress-creator upgrade -t')207	console.log('')208})209program.parse(process.argv)210function compileAction(prjName,tplPath,tgrPath,cfgGather){211  return new Promise(function(resolve, reject) {212    fsextra.copy(tplPath, tgrPath).then(() => {213      const configPath = `${prjName}/docs/.vuepress/config.js`214      const configGather = cfgGather215      compileStr(configPath, configGather).then(() => {216        resolve()217      }).catch(err => {218        return err219      })220    }).catch(err => {221      return reject(err)222    })223  })224}225function installThemeAction(prjName){226  return new Promise(function(resolve, reject) {227    const installSpinner = ora(chalk.cyan('Install AntDocs theme...'))228    installSpinner.start()229    shell(`cd ${prjName} && npm i vuepress-theme-antdocs`, (err, stdout, stderr) => {230      if (err) {231        installSpinner.text = 'Install AntDocs theme failed.'232        installSpinner.fail()233        return reject(err)234      }235      installSpinner.text = 'Install AntDocs theme successful.'236      installSpinner.succeed()237      resolve()238    })239  })240}241function showInfoAction(prjName){242  console.log('')243  console.log('    To get started:')244  console.log('')245  console.log(chalk.yellow(`      cd ${prjName}`))246  console.log(`      ${chalk.yellow('npm install')} or ${chalk.yellow('yarn install')}`)247  console.log(`      ${chalk.yellow('npm run dev')} or ${chalk.yellow('yarn run dev')}`)248  console.log('')...

1import Crypto from 'crypto'2import FSExtra from 'fs-extra'3import Scramjet from 'scramjet'4import Axios from 'axios'5import AxiosRetry from 'axios-retry'6import AxiosRateLimit from 'axios-rate-limit'7import FormData from 'form-data'8import Querystring from 'querystring'9function request(retries, cache, verbose, alert, limit, messages) {10    const cacheDirectory = '.reconcile-cache'11    const timeout = 45 * 100012    const toUrl = location => typeof location === 'string' ? location : location.url13    const toLocationName = location => {14        if (!location) throw new Error('Request location is blank')15        const method = location.method ? location.method.toUpperCase() : 'GET'16        const stringifyObject = object => Object.entries(object).map(([key, value]) => `${key}=${JSON.stringify(value)}`).join(' ')17        return `${method} ${toUrl(location)}`18            + (location.params ? '?' + Querystring.stringify(location.params) : '')19            + (location.dataQuery ? ' ' + Querystring.stringify(location.dataQuery) : '')20            + (location.dataForm ? ' <' + stringifyObject(location.dataForm) + '>' : '')21            + (location.data && !location.dataQuery && !location.dataForm ? ' [' + stringifyObject(location.data) + ']' : '')22    }23    const toErrorMessage = e => {24        const reconcilerError = e.response && messages(e)25        if (reconcilerError) return reconcilerError // look for reconciler-specific errors first26        const locationName = toLocationName(e.config)27        if (e.response) return `Received code ${e.response.status}: ${locationName}` // response recieved, but non-2xx28        if (e.code === 'ECONNABORTED') return `Timed out after ${timeout / 1000}ms: ${locationName}` // request timed out29        if (e.code) return `Error ${e.code}: ${locationName}` // request failed, with error code30        return e.message // request not made31    }32    const instance = Axios.create({ timeout })33    AxiosRetry(instance, {34        retries,35        shouldResetTimeout: true,36        retryCondition: e => {37            return !e.response || e.response.status >= 500 || e.response.status === 429 // no response, server error, or hit rate limit38        },39        retryDelay: (number, e) => {40            const message = toErrorMessage(e)41            const attempt = number > 0 && number <= retries && retries > 1 ? ' (retrying' + (number > 1 ? `, attempt ${number}` : '') + '...)' : ''42            if (number === 1) alert(`${message}${attempt}`)43            else alert(`  → ${message}${attempt}`)44            return 5 * 100045        }46    })47    AxiosRateLimit(instance, {48        maxRequests: limit, // so limit is number of requests per second49        perMilliseconds: 1 * 100050    })51    let cacheChecked = false52    return async location => {53        const hash = Crypto.createHash('sha1').update(JSON.stringify(typeof location === 'string' ? location : { ...location, auth: null })).digest('hex')54        if (location.dataQuery) {55            location.data = Querystring.stringify(location.dataQuery)56        }57        if (location.dataForm) {58            const form = new FormData()59            location.headers = form.getHeaders()60            Object.entries(location.dataForm).forEach(([key, value]) => form.append(key, JSON.stringify(value)))61            location.data = form62        }63        const locationName = toLocationName(location)64        if (cache) {65            if (!cacheChecked) {66                const cacheExists = await FSExtra.pathExists(cacheDirectory)67                if (cacheExists) alert('Cached data found!')68                else alert('No existing cached data found')69                cacheChecked = true70            }71            const isCached = await FSExtra.pathExists(`${cacheDirectory}/${hash}`)72            if (isCached) {73                if (verbose) alert(`Cached [${hash}]: ${locationName}`)74                const cacheData = await FSExtra.readJson(`${cacheDirectory}/${hash}`)75                return {76                    url: toUrl(location),77                    data: cacheData,78                    passthrough: location.passthrough79                }80            }81        }82        try {83            if (verbose) alert(`Requesting: ${locationName}`)84            const response = await instance(location)85            if (cache) {86                await FSExtra.ensureDir(cacheDirectory)87                await FSExtra.writeJson(`${cacheDirectory}/${hash}`, response.data)88            }89            return {90                url: toUrl(location),91                data: response.data,92                passthrough: location.passthrough93            }94        }95        catch (e) {96            alert(toErrorMessage(e))97        }98    }99}100async function load(command, filename, parameters = {}, retries = 5, cache = false, join = 'inner', verbose = false, alert = () => {}) {101    const die = message => {102        throw new Error(message)103    }104    const requestor = request.bind(null, retries, cache, verbose, alert)105    const { default: reconciler } = await import(`./reconcilers/${command}.js`)106    Object.keys(parameters).forEach(parameter => {107        if (!reconciler.details.parameters.find(p => p.name === parameter)) alert(`Ignoring unexpected parameter '${parameter}'`)108    })109    const batch = reconciler.details.batch || 1110    const execute = reconciler.initialise(parameters, requestor, die)111    const source = () => Scramjet.StringStream.from(FSExtra.createReadStream(filename)).CSVParse({ header: true })112    const columnsReconciler = reconciler.details.columns.map(column => column.name)113    const columnsSource = Object.keys((await source().slice(0, 1).toArray())[0])114    const columnMapEntries = columnsReconciler.map(column => {115        const columnUnique = (i = '') => {116            const attempt = `${column}${i}`117            if (columnsSource.find(name => name === attempt)) return columnUnique(Number(i) + 1)118            if (i) alert(`Column '${column}' from the reconciler has been renamed '${attempt}' so it does not overwrite the source`)119            return attempt120        }121        return [column, columnUnique()]122    })123    const columnMap = Object.fromEntries(columnMapEntries)124    const blank = Object.fromEntries(Object.values(columnMap).map(key => [key]))125    const length = async () => {126        const entries = await source().reduce(a => a + 1, 0)127        return Math.ceil(entries / batch)128    }129    const run = async () => source().batch(batch).setOptions({ maxParallel: 1 }).map(async items => {130        try {131            const executed = await execute(batch === 1 ? items[0] : items)132            return items.flatMap((item, i) => {133                const results = batch > 1 && Array.isArray(executed[i]) ? executed[i] // batch mode, reconciler is one-to-many134                    : batch > 1 ? [executed[i]] // batch mode, reconciler is one-to-one135                    : Array.isArray(executed) ? executed // reconciler is one-to-many136                    : [executed] // reconciler is one-to-one137                if (join === 'outer' && results.length === 0) return [{ ...item, ...blank }]138                return results.map(result => {139                    const resultRemapped = result140                        ? Object.fromEntries(Object.entries(result).map(([column, value]) => [columnMap[column], value]))141                        : Object.fromEntries(Object.entries(columnMap).map(([column]) => [column, ''])) // if there is no result (eg. not found)142                    return { ...item, ...resultRemapped }143                })144            })145        }146        catch (e) {147            alert(verbose ? e.stack : e.message)148            if (join === 'outer') {149                return items.map(item => ({ ...item, ...blank }))150            }151            else return []152        }153    })154    return { run, length }155}...

1import Process from 'process'2import WorkerThreads from 'worker_threads'3import ObjectHash from 'object-hash'4import FSExtra from 'fs-extra'5async function sourcing() {6    const { method, ...settings } = watch.source // omit method name7    const { default: f } = await import(`./../methods/source-${watch.source.method}.js`)8    const items = await f(settings)9    return items.map(content => {10        const id = ObjectHash(content)11        return { id, content }12    })13}14async function diffing(items) {15    const key = ObjectHash(watch.source)16    const path = `.newsagent-cache/${key}`17    const hash = ObjectHash(items)18    const hashset = items.map(item => item.id)19    const cachefileExists = await FSExtra.pathExists(path)20    if (!cachefileExists) {21        await FSExtra.writeJson(path, { hash, hashset, amalgam: items })22        return []23    }24    const cachefile = await FSExtra.readJson(path)25    if (cachefile.hash === hash) return [] // whole-data hash is the same, so we can avoid looking at individual records26    const additions = () => {27        return items.filter(item => !cachefile.hashset.includes(item.id)).map(item => {28            return { ...item, difference: 'addition' }29        })30    }31    const removals = () => {32        return cachefile.hashset.filter(hash => !hashset.includes(hash)).map(hash => {33            const item = cachefile.amalgam.find(item => item.id === hash)34            return { ...item, difference: 'removal' }35        })36    }37    const changesAdditions = watch.monitor === 'additions-and-removals' || watch.monitor === 'additions-only' ? additions() : []38    const changesRemovals = watch.monitor === 'additions-and-removals' || watch.monitor === 'removals-only' ? removals() : []39    const amalgam = cachefile.amalgam40        .filter(item => !changesRemovals.find(change => change.id === item.id))41        .concat(changesAdditions.map(change => ({ id: change.id, content: change.content }))) // so omit difference field42    await FSExtra.writeJson(path, { hash, hashset, amalgam })43    return [...changesAdditions, ...changesRemovals]44}45async function processing(changes) {46    if (!watch.processes) return changes47    return watch.processes.reduce(async (a, process) => {48        const { method, ...settings } = process // omit method name49        const { default: f } = await import(`./../methods/process-${process.method}.js`)50        const all = await Promise.all((await a).map(async change => {51            const contentNew = await f(change.content, change.difference, settings)52            return { ...change, content: contentNew }53        }))54        return all.filter(x => x.content)55    }, changes)56}57async function alerting(results) {58    const firings = watch.alerts.map(async alert => {59        const { method, ...settings } = alert // omit method name60        const { default: f } = await import(`./../methods/alert-${alert.method}.js`)61        results.forEach(result => f(watch.name, result.content, settings))62    })63    await Promise.all(firings)64}65async function execute() {66    try {67        const items = await sourcing()68        const changes = await diffing(items)69        const processed = await processing(changes)70        await alerting(processed)71    }72    catch (e) {73        if (e.constructor.name === 'ZodError') {74            const error = e.errors[0]75            WorkerThreads.parentPort.postMessage({76                event: 'execution-invalid',77                data: { error: `${error.message}: ${error.path.join('.')} wanted ${error.expected} but got ${error.received}` }78            })79        }80        else {81            WorkerThreads.parentPort.postMessage({82                event: 'execution-failure',83                data: { error: e.message }84            })85        }86        Process.exit(1)87    }88}...

1import FSExtra from 'fs-extra'2import Scramjet from 'scramjet'3import Axios from 'axios'4import AxiosRetry from 'axios-retry'5import AxiosRateLimit from 'axios-rate-limit'6function fetcher(urlColumn, nameColumn, depository, suffix, headers, limit, retries, check, verbose, alert) {7    const timeout = 45 * 10008    const toErrorMessage = e => {9        const locationName = e.config.url10        if (e.response) return `Received code ${e.response.status}: ${locationName}` // response recieved, but non-2xx11        if (e.code === 'ECONNABORTED') return `Timed out after ${timeout / 1000}ms: ${locationName}` // request timed out12        if (e.code) return `Error ${e.code}: ${locationName}` // request failed, with error code13        return e.message // request not made14    }15    const instance = Axios.create({ timeout })16    AxiosRetry(instance, {17        retries,18        shouldResetTimeout: true,19        retryCondition: e => {20            return !e.response || e.response.status >= 500 || e.response.status === 429 // no response, server error, or hit rate limit21        },22        retryDelay: (number, e) => {23            const message = toErrorMessage(e)24            const attempt = number > 0 && number <= retries && retries > 1 ? ' (retrying' + (number > 1 ? `, attempt ${number}` : '') + '...)' : ''25            if (number === 1) alert(`${message}${attempt}`)26            else alert(`  → ${message}${attempt}`)27            return 5 * 100028        }29    })30    AxiosRateLimit(instance, {31        maxRequests: limit, // so limit is number of requests per second32        perMilliseconds: 1 * 100033    })34    return async row => {35        const key = row[nameColumn]36        if (!key) {37            alert('Key column is empty!')38            return39        }40        const url = row[urlColumn]41        if (!url) {42            alert('URL column is empty!')43            return44        }45        const filename = key + (suffix || '')46        if (check) {47            const exists = await FSExtra.pathExists(`${depository}/${filename}`)48            if (exists && verbose) {49                alert(`Exists [${filename}]: ${url}`)50                return51            }52        }53        try {54            const headersValues = headers ? Object.fromEntries(headers.map(header => header.split(/: ?/))) : {}55            if (verbose) alert(`Requesting: ${url}` + (headersValues ? ' ' + JSON.stringify(headersValues, null, 2) : ''))56            const response = await instance({57                url,58                headers: headersValues,59                responseType: 'arraybuffer'60            })61            await FSExtra.writeFile(`${depository}/${filename}`, response.data)62        }63        catch (e) {64            alert(toErrorMessage(e))65        }66    }67}68function source(filename) {69    return Scramjet.StringStream.from(FSExtra.createReadStream(filename)).CSVParse({ header: true })70}71function length(filename) {72    return source(filename).reduce(a => a + 1, 0)73}74async function run(filename, urlColumn, nameColumn, depository, suffix, headers, limit, retries, check, verbose, alert) {75    await FSExtra.ensureDir(depository)76    const fetch = fetcher(urlColumn, nameColumn, depository, suffix, headers, limit, retries, check, verbose, alert)77    return source(filename).each(fetch)78}...

1import Readline from 'readline'2import FSExtra from 'fs-extra'3import Process from 'process'4import Yargs from 'yargs'5import Progress from 'progress'6import csvFetch from './csv-fetch.js'7function alert(message) {8    Readline.clearLine(process.stderr)9    Readline.cursorTo(process.stderr, 0)10    console.error(message)11}12function ticker(text, total) {13    const progress = new Progress(text + ' |:bar| :percent / :etas left', {14        total,15        width: Infinity,16        complete: '█',17        incomplete: ' '18    })19    return () => progress.tick()20}21async function setup() {22    const instructions = Yargs(Process.argv.slice(2))23        .usage('Usage: csv-fetch <url-column> <name-column> <depository> <filename>')24        .wrap(null)25        .option('s', { alias: 'suffix', type: 'string', describe: 'A suffix to add to the name of each file, such as an extension' })26        .option('h', { alias: 'header', type: 'string', array: true, nargs: 1, describe: 'Set a header to be sent with the request' })27        .option('l', { alias: 'limit', type: 'number', nargs: 1, describe: 'Limit the number of requests made per second' })28        .option('r', { alias: 'retries', type: 'number', nargs: 1, describe: 'Number of times a request should be retried', default: 5 })29        .option('c', { alias: 'check', type: 'boolean', describe: 'Check whether file has already been downloaded, and skip if so', default: false })30        .option('V', { alias: 'verbose', type: 'boolean', describe: 'Print every request made', default: false })31        .help('?').alias('?', 'help')32        .version().alias('v', 'version')33    if (instructions.argv._.length === 0) instructions.showHelp().exit(0)34    try {35        const {36            _: [urlColumn, nameColumn, depository, filename],37            suffix,38            header: headers,39            limit,40            retries,41            check,42            verbose43        } = instructions.argv44        if (filename === '-') throw new Error('reading from standard input not supported')45        const exists = await FSExtra.pathExists(filename)46        if (!exists) throw new Error(`${filename}: could not find file`)47        if (headers) headers.forEach(header => {48            if (!header.includes(':')) throw new Error(`"${header}" header is not valid`)49        })50        const total = await csvFetch.length(filename)51        console.error('Starting up...')52        const process = await csvFetch.run(filename, urlColumn, nameColumn, depository, suffix, headers, limit, retries, check, verbose, alert)53        await process54            .each(ticker('Working...', total))55            .whenEnd()56        console.error('Done!')57    }58    catch (e) {59        console.error(e.message)60        Process.exit(1)61    }62}...

1const chalk = require('chalk')2const fsextra = require('fs-extra')3const handlebars = require('handlebars')4const symbols = require('log-symbols')5function compileStr(filePath, answersContent) {6  return new Promise(function(resolve, reject) {7		fsextra.pathExists(filePath).then(exists => {8      if (exists) {9        const _filePath = filePath10        const _Content = fsextra.readFileSync(_filePath).toString()11        const _Result = handlebars.compile(_Content)(answersContent)12        fsextra.writeFileSync(_filePath, _Result)13        console.log(symbols.success, `${chalk.cyan(filePath)} configured successful.`)14        resolve()15      } else {16        console.log(symbols.error, `${chalk.red(filePath)} not found.`)17        return reject()18      }19    }).catch(err => {20      console.log(symbols.error, chalk.red(err))21    })22  })23}...