How to use tempDir.path method in Appium Xcuitest Driver

1exports.cmd_run_process = cmd_run_process;2exports.cleanup = cleanup; // in case process terminated prematurely3const KBClient = require('kbclient').v1;4const LariClient = require('lariclient').v1;5const async = require('async');6let DEBUG = true;7let tempdir_for_cleanup = '';8let files_to_cleanup = [];9let keep_tempdir = false;10let processor_job_id_for_cleanup = '';11let common = require(__dirname + '/common.js');12let prv_utils = require(__dirname + '/prv_utils.js');13let db_utils = require(__dirname + '/db_utils.js');14let SystemProcess = new require(__dirname + '/systemprocess.js').SystemProcess;15let sha1 = require('node-sha1');16let max_num_simultaneous_processor_jobs = 2;17let canonical_stringify = require('canonical-json');18function cmd_run_process(processor_name, opts, callback) {19  if (opts.verbose == 'minimal' || opts.verbose == 'jupyter') {20    console.info = function() {};21    console.log = function() {};22  }23  if (opts.verbose == 'none') {24    console.info = function() {};25    console.log = function() {};26    console.warn = function() {};27    console.error = function() {};28  }29  opts.lari_id = opts.lari_id || process.env.LARI_ID;30  opts.lari_passcode = opts.lari_passcode || process.env.LARI_PASSCODE;31  console.info('[ Getting processor spec... ]');32  let spec_opts = {33    lari_id: opts.lari_id,34    lari_passcode: opts.lari_passcode,35    mp_file: opts.mp_file||undefined,36    mp_file_args: opts.mp_file_args||undefined,37    container: opts.container||undefined38  };39  common.get_processor_spec(processor_name, spec_opts, function(err, spec0) {40    if (err) {41      callback(err);42      return;43    }44    if (!spec0) {45      callback(`Processor not found: ${processor_name}`);46      return;47    }48    if ((opts.lari_id)&&(!opts.mp_file)) {49      cmd_run_process_lari(processor_name, spec0, opts, callback);50      return;51    }52    spec0.outputs = spec0.outputs || [];53    spec0.outputs.push({54      name: 'console_out',55      optional: true,56    });57    run_process_2(processor_name, opts, spec0, callback);58  });59}60function LariJob() {61  this.setLariId = function(id, passcode) {62    m_lari_id = id;63    m_lari_passcode = passcode || '';64  };65  this.setLariOutFile = function(outfile) {66    m_lari_out_file = outfile;67  };68  this.runProcess = function(69    processor_name,70    inputs,71    outputs,72    parameters,73    opts74  ) {75    m_processor_name = processor_name;76    m_outputs = JSON.parse(JSON.stringify(outputs));77    let outputs2 = {};78    for (let okey in m_outputs) {79      // let fname = m_outputs[okey];80      outputs2[okey] = true;81    }82    get_prv_objects_for_inputs(inputs, function(err, inputs2) {83      if (err) {84        console.error(err);85        process.exit(-1);86      }87      opts.lari_passcode = m_lari_passcode;88      m_client89        .runProcess(90          m_lari_id,91          processor_name,92          inputs2,93          outputs2,94          parameters,95          opts96        )97        .then(function(resp) {98          if (!resp.success) {99            console.error('Error running process: ' + resp.error);100            process.exit(-1);101          }102          m_job_id = resp.job_id;103          write_lari_out_file();104          setTimeout(function() {105            probe_process();106          }, 500);107        })108        .catch(function(err) {109          console.error(err);110          process.exit(-1);111        });112    });113  };114  let m_lari_id = '';115  let m_lari_passcode = '';116  let m_lari_out_file = '';117  let m_job_id = '';118  let m_client = new LariClient();119  let m_processor_name = '';120  let m_outputs = {};121  // terminal color codes122  /* let ccc = {123    Reset: '\x1b[0m',124    Bright: '\x1b[1m',125    Dim: '\x1b[2m',126    Underscore: '\x1b[4m',127    Blink: '\x1b[5m',128    Reverse: '\x1b[7m',129    Hidden: '\x1b[8m',130    FgBlack: '\x1b[30m',131    FgRed: '\x1b[31m',132    FgGreen: '\x1b[32m',133    FgYellow: '\x1b[33m',134    FgBlue: '\x1b[34m',135    FgMagenta: '\x1b[35m',136    FgCyan: '\x1b[36m',137    FgWhite: '\x1b[37m',138    BgBlack: '\x1b[40m',139    BgRed: '\x1b[41m',140    BgGreen: '\x1b[42m',141    BgYellow: '\x1b[43m',142    BgBlue: '\x1b[44m',143    BgMagenta: '\x1b[45m',144    BgCyan: '\x1b[46m',145    BgWhite: '\x1b[47m',146  };*/147  function write_lari_out_file() {148    if (!m_lari_out_file) return;149    let obj = {150      lari_id: m_lari_id,151      lari_job_id: m_job_id,152    };153    if (!common.write_json_file(m_lari_out_file, obj)) {154      console.error('Unable to write lari out file: ' + m_lari_out_file + '. Aborting.');155      process.exit(-1);156    }157  }158  function probe_process() {159    m_client160      .probeProcess(m_lari_id, m_job_id, {161        lari_passcode: m_lari_passcode,162      })163      .then(function(resp) {164        let msec = 3000;165        if (!('stdout' in resp)) {166          // old system167          if (resp.console_output) {168            let lines = resp.console_output.split('\n');169            for (let i in lines) {170              console.info(lines[i]);171              // console.info(ccc.BgBlack, ccc.FgCyan, lines[i], ccc.Reset);172            }173            msec = 1000;174          }175        } else {176          if (resp.stdout) {177            let lines = resp.stdout.split('\n');178            for (let i in lines) {179              console.info(lines[i]);180              // console.info(ccc.BgBlack, ccc.FgCyan, lines[i], ccc.Reset);181            }182            msec = 1000;183          }184          if (resp.stderr) {185            let lines = resp.stderr.split('\n');186            for (let i in lines) {187              console.info('STDERR: '+lines[i]);188              // console.info(ccc.BgRed, ccc.FgCyan, lines[i], ccc.Reset);189            }190            msec = 1000;191          }192        }193        if (resp.is_complete) {194          let result = resp.result || {};195          if (!result.success) {196            console.error(`${m_processor_name} completed with error: ${result.error}`);197            /*198             console.info(199             ccc.BgBlack,200             ccc.FgRed,201             `${m_processor_name} completed with error: ${result.error}`,202             ccc.Reset203             );204             */205            process.exit(-1);206          }207          let output_keys = Object.keys(m_outputs);208          async.eachSeries(209            output_keys,210            function(okey, cb) {211              let output0 = result.outputs[okey] || null;212              if (!output0) {213                console.error(`Unexpected missing output for ${okey}.`);214                process.exit(-1);215              }216              if (!output0.original_checksum) {217                console.error(`Unexpected format for output ${okey}.`);218                process.exit(-1);219              }220              let fname = m_outputs[okey];221              if (!common.ends_with(fname, '.prv')) {222                if (output0.original_size > 1024 * 1024) {223                  if (require('fs').existsSync(fname)) {224require('fs').unlinkSync(fname);225} // there can be trouble if we don't delete fname226                  fname += '.prv';227                  console.warn(228                    `Output ${okey} is too large to automatically download. Saving .prv file instead: ${fname}`229                  );230                }231              }232              if (common.ends_with(fname, '.prv')) {233                console.info(`Writing output ${okey} to file: ${fname}`);234                common.write_json_file(fname, output0);235                cb();236              } else {237                let KBC = new KBClient();238                KBC.downloadFile(239                  'sha1://' + output0.original_checksum,240                  fname, {}241                )242                  .then(function() {243                    cb();244                  })245                  .catch(function(err) {246                    console.error(err);247                    console.error(248                      `Error downloading output ${okey}: ${err.message}`249                    );250                    process.exit(-1);251                  });252              }253            },254            function() {255              console.info(`${m_processor_name} completed successfully.`);256              /*257                 console.info(258                 ccc.BgBlack,259                 ccc.FgGreen,260                 `${m_processor_name} completed successfully.`,261                 ccc.Reset262                 );263                 */264              process.exit(0);265            }266          );267          return;268        }269        setTimeout(function() {270          probe_process();271        }, msec);272      })273      .catch(function(err) {274        console.error(err);275        process.exit(-1);276      });277  }278  function get_prv_objects_for_inputs(inputs, callback) {279    let ret = JSON.parse(JSON.stringify(inputs));280    let ikeys = Object.keys(ret);281    async.eachSeries(282      ikeys,283      function(ikey, cb) {284        let val = ret[ikey];285        if (val instanceof Array) {286          let indices = Object.keys(val);287          async.eachSeries(288            indices,289            function(ii, cb2) {290              get_prv_object_for_input(val[ii], function(err, obj) {291                if (err) {292                  callback(293                    `Problem getting prv object for input ${ikey}[${ii}]: ${err}`294                  );295                  return;296                }297                val[ii] = obj;298                cb2();299              });300            },301            cb302          );303        } else {304          get_prv_object_for_input(val, function(err, obj) {305            if (err) {306              callback(`Problem getting prv object for input ${ikey}: ${err}`);307              return;308            }309            ret[ikey] = obj;310            cb();311          });312        }313      },314      function() {315        callback(null, ret);316      }317    );318  }319  function get_prv_object_for_input(input, callback) {320    if (typeof input != 'string') {321      callback('Input is not a string.');322      return;323    }324    if (!common.ends_with(input, '.prv')) {325      if (!file_exists(input) && file_exists(input + '.prv')) input += '.prv';326    }327    if (common.ends_with(input, '.prv')) {328      let obj = common.read_json_file(input);329      if (!obj) {330        callback('Error parsing json in prv file.');331        return;332      }333      callback(null, obj);334    } else if (input.startsWith('kbucket://') || input.startsWith('sha1://')) {335      callback(null, input);336    } else {337      prv_utils.prv_create(input, function(err, obj) {338        callback(err, obj);339      });340    }341  }342}343function cmd_run_process_lari(processor_name, spec0, opts, callback) {344  // todo: this functionality is duplicated below, try to combine code345  let inputs; let outputs; let parameters;346  try {347    inputs = parse_iop(opts.inputs || '', 'input');348    outputs = parse_iop(opts.outputs || '', 'output');349    parameters = parse_iop(opts.parameters || '', 'parameter');350    let iops = parse_iop(opts.iops || '', 'iop');351    separate_iops(352      inputs,353      outputs,354      parameters,355      iops,356      spec0.inputs || [],357      spec0.outputs || [],358      spec0.parameters || []359    );360    check_iop(inputs, spec0.inputs || [], 'input');361    check_iop(outputs, spec0.outputs || [], 'output');362    check_iop(parameters, spec0.parameters || [], 'parameter');363  } catch (err) {364    console.error(err.stack);365    callback(err.message);366    return;367  }368  // //////////////////////////////////////////////////////////////369  // let LC =  // Not Used370  new LariClient();371  let p_opts = {};372  if ('force_run' in opts) p_opts.force_run = opts.force_run;373  if ('container' in opts) p_opts.container = opts.container;374  // important -- do not pass through the opts here, because there would be security concerns. Keep the interface minimal. For example, processor_command_prefix should be configured on the server side.375  let LJ = new LariJob();376  LJ.setLariId(opts.lari_id, opts.lari_passcode);377  if (opts.lari_out) LJ.setLariOutFile(opts.lari_out);378  LJ.runProcess(processor_name, inputs, outputs, parameters, p_opts);379}380function remove_processor_job_from_database(job_id, callback) {381  db_utils.removeDocuments(382    'processor_jobs', {383      _id: job_id,384    },385    function(err) {386      callback(err);387    }388  );389}390function run_process_2(processor_name, opts, spec0, callback) {391  let inputs; let outputs; let parameters;392  try {393    inputs = parse_iop(opts.inputs || '', 'input');394    outputs = parse_iop(opts.outputs || '', 'output');395    parameters = parse_iop(opts.parameters || '', 'parameter');396    let iops = parse_iop(opts.iops || '', 'iop');397    separate_iops(398      inputs,399      outputs,400      parameters,401      iops,402      spec0.inputs || [],403      spec0.outputs || [],404      spec0.parameters || []405    );406    check_iop(inputs, spec0.inputs || [], 'input');407    check_iop(outputs, spec0.outputs || [], 'output');408    check_iop(parameters, spec0.parameters || [], 'parameter');409  } catch (err) {410    console.error(err.stack);411    callback(err.message);412    return;413  }414  let process_signature = '';415  let pending_output_prvs = [];416  let mode = opts.mode || 'run';417  let already_completed = false;418  let tempdir_path = '';419  let queued_processor_job_id = '';420  let original_inputs = JSON.parse(JSON.stringify(inputs));421  let temporary_outputs = null;422  let temporary_inputs = null;423  let steps = [];424  // Check inputs, set default parameters and substitute prvs425  steps.push(function(cb) {426    console.info('[ Checking inputs and substituting prvs ... ]');427    check_inputs_and_substitute_prvs(inputs, '', opts, function(err) {428      if (err) {429        finalize(err);430        return;431      }432      cb();433    });434  });435  // Compute process signature436  steps.push(function(cb) {437    // if (mode=='exec') {438    //  cb();439    //  return;440    // }441    console.info('[ Computing process signature ... ]');442    compute_process_signature(spec0, inputs, parameters, function(err, sig) {443      if (err) {444        finalize(err);445        return;446      }447      process_signature = sig;448      console.info(`Process signature: ${process_signature}`);449      cb();450    });451  });452  // Check outputs453  steps.push(function(cb) {454    console.info('[ Checking outputs... ]');455    check_outputs_and_substitute_prvs(outputs, process_signature, function(456      err,457      tmp458    ) {459      if (err) {460        finalize(err);461        return;462      }463      pending_output_prvs = tmp.pending_output_prvs;464      cb();465    });466  });467  // Check process cache468  steps.push(function(cb) {469    if (mode == 'exec' || opts.force_run) {470      cb();471      return;472    }473    console.info('[ Checking process cache ... ]');474    find_in_process_cache(process_signature, outputs, function(err, doc0) {475      if (err) {476        finalize(err);477        return;478      }479      if (doc0) {480        console.info(`[ Process ${processor_name} already completed. ]`);481        already_completed = true;482      }483      cb();484    });485  });486  // Wait for ready run487  steps.push(function(cb) {488    if (already_completed) {489      cb();490      return;491    }492    if (mode == 'exec' || mode == 'run') {493      cb();494      return;495    }496    console.info('[ Waiting for ready to run ... ]');497    wait_for_ready_run(spec0, inputs, outputs, parameters, function(498      err,499      job_id500    ) {501      if (err) {502        finalize(err);503        return;504      }505      queued_processor_job_id = job_id;506      processor_job_id_for_cleanup = job_id;507      cb();508    });509  });510  // Create temporary directory511  steps.push(function(cb) {512    if (already_completed) {513      cb();514      return;515    }516    console.info('[ Creating temporary directory ... ]');517    let tmp_dir = common.temporary_directory();518    tempdir_path =519      tmp_dir +520      '/tempdir_' +521      process_signature.slice(0, 10) +522      '_' +523      common.make_random_id(6);524    tempdir_for_cleanup = tempdir_path; // in case process is terminated prematurely525    if (opts.keep_tempdir) keep_tempdir = true;526    common.mkdir_if_needed(tempdir_path);527    cb();528  });529  // Create links to input files530  steps.push(function(cb) {531    if (already_completed) {532      cb();533      return;534    }535    console.info('[ Creating links to input files... ]');536    link_inputs(537      inputs,538      original_inputs, {539        tempdir_path: tempdir_path,540      },541      function(err, tmp) {542        if (err) {543          finalize(err);544          return;545        }546        temporary_inputs = tmp.temporary_inputs;547        cb();548      }549    );550  });551  // Make temporary outputs552  steps.push(function(cb) {553    if (already_completed) {554      cb();555      return;556    }557    console.info('[ Preparing temporary outputs... ]');558    make_temporary_outputs(559      outputs,560      process_signature, {561        tempdir_path: tempdir_path,562      },563      function(err, tmp) {564        if (err) {565          finalize(err);566          return;567        }568        temporary_outputs = tmp.temporary_outputs;569        cb();570      }571    );572  });573  // Run the process574  steps.push(function(cb) {575    if (already_completed) {576      cb();577      return;578    }579    console.info('[ Initializing process ... ]');580    do_run_process(581      spec0,582      temporary_inputs,583      outputs,584      temporary_outputs,585      parameters, {586        tempdir_path: tempdir_path,587        queued_processor_job_id: queued_processor_job_id,588        processor_command_prefix: opts.processor_command_prefix || '',589      },590      function(err) {591        if (err) {592          finalize(err);593          return;594        }595        cb();596      }597    );598  });599  // Handle pending output prvs600  steps.push(function(cb) {601    common.foreach_async(602      pending_output_prvs,603      function(ii, X, cb2) {604        console.info(`[ Creating output prv for ${X.name} ... ]`);605        prv_utils.prv_create(X.output_fname, function(err, obj) {606          if (err) {607            finalize(err);608            return;609          }610          common.write_json_file(X.prv_fname, obj);611          cb2();612        });613      },614      function() {615        cb();616      }617    );618  });619  // Save to process cache620  steps.push(function(cb) {621    if (already_completed) {622      cb();623      return;624    }625    if (mode == 'exec') {626      cb();627      return;628    }629    console.info('[ Saving to process cache ... ]');630    save_to_process_cache(631      process_signature,632      spec0,633      inputs,634      outputs,635      parameters,636      function(err) {637        if (err) {638          finalize(err);639        }640        cb();641      }642    );643  });644  // Remove from database (if mode=queued)645  steps.push(function(cb) {646    if (!queued_processor_job_id) {647      cb();648      return;649    }650    console.info('[ Removing processor job from database ... ]');651    remove_processor_job_from_database(queued_processor_job_id, function(err) {652      if (err) {653        finalize(err);654      }655      cb();656    });657  });658  common.foreach_async(659    steps,660    function(ii, step, cb) {661      step(cb);662    },663    function() {664      finalize(null);665    }666  );667  function finalize(err00) {668    remove_temporary_files(files_to_cleanup, function(err) {669      if (err) {670        console.warn('Error removing temporary files: ' + err);671      }672      remove_temporary_directory(tempdir_path, function(err) {673        if (err) {674          console.warn(675            'Error removing temporary directory (' + tempdir_path + '): ' + err676          );677        }678        if (!err00) {679          console.info('[ Done. ]');680        }681        callback(err00);682      });683    });684  }685}686function move_file(srcpath, dstpath, callback) {687  require('fs').rename(srcpath, dstpath, function(err) {688    if (err) {689      console.warn(690        `This is only a warning: Unable to rename file ${srcpath} -> ${dstpath}. Perhaps temporary directory is not on the same device as the output file directory. Copying instead.`691      );692      require('fs').copyFile(srcpath, dstpath, function(err) {693        if (err) {694          callback(695            `Error renaming file ${srcpath} -> ${dstpath}: ${err.message}`696          );697          return;698        }699        require('fs').unlink(srcpath, function(err) {700          if (err) {701            callback(702              `Error removing file after copy... ${srcpath}: ${err.message}`703            );704            return;705          }706          callback(null);707        });708      });709      return;710    }711    callback(null);712  });713}714function move_file_or_files(srcpath, dstpath, callback) {715  if (srcpath instanceof Array) {716    srcpath.forEach(function(cv, i, arr) {717      move_file(cv, dstpath[i], function(err) {718        if (err) {719          callback(err);720        }721      });722    });723    callback(null);724  } else {725    move_file(srcpath, dstpath, callback);726  }727}728function move_outputs(src_outputs, dst_outputs, callback) {729  let output_keys = Object.keys(src_outputs);730  async.eachSeries(output_keys, function(key, cb) {731    console.info(`Finalizing output ${key}`);732    move_file_or_files(src_outputs[key], dst_outputs[key], function(err) {733      if (err) {734        callback(err);735        return;736      }737      cb();738    });739  }, function() {740    callback(null);741  });742}743function cleanup(callback) {744  // only called if process is terminated prematurely745  cleanup_temp_files(function() {746    cleanup_tempdir(function() {747      remove_from_database(function() {748        callback();749      });750    });751  });752  function cleanup_temp_files(cb) {753    remove_temporary_files(files_to_cleanup, function(err) {754      if (err) {755        console.warn('Error removing temporary files: ' + err);756      }757      cb();758    });759  }760  function cleanup_tempdir(cb) {761    if (tempdir_for_cleanup) {762      remove_temporary_directory(tempdir_for_cleanup, function() {763        cb();764      });765    } else {766      cb();767    }768  }769  function remove_from_database(cb) {770    if (processor_job_id_for_cleanup) {771      remove_processor_job_from_database(772        processor_job_id_for_cleanup,773        function() {774          cb();775        }776      );777    } else {778      cb();779    }780  }781}782function remove_temporary_files(tmp_files, callback) {783  async.eachSeries(tmp_files, function(fname, cb) {784    try {785      if (require('fs').existsSync(fname)) {786require('fs').unlinkSync(fname);787}788    } catch (err) {789      console.warn('Problem removing temporary file: ' + fname);790    }791    cb();792  }, function() {793    callback(null);794  });795}796function remove_temporary_directory(tempdir_path, callback) {797  if (keep_tempdir) {798    callback(null);799    return;800  }801  if (!tempdir_path) {802    callback(null);803    return;804  }805  console.info('[ Removing temporary directory ... ]');806  if (!common.starts_with(tempdir_path, common.temporary_directory() + '/')) {807    // for safety808    callback('Invalid (unsafe) path for temporary directory: ' + tempdir_path);809    return;810  }811  let files = common.read_dir_safe(tempdir_path);812  common.foreach_async(813    files,814    function(ii, file, cb) {815      let fname = tempdir_path + '/' + file;816      let stat0 = common.stat_file(fname);817      if (stat0) {818        if (stat0.isFile()) {819          try {820            require('fs').unlinkSync(fname);821          } catch (err) {822            callback(823              'Unable to remove file from temporary directory: ' + fname824            );825            return;826          }827          cb();828        } else if (stat0.isDirectory()) {829          remove_temporary_directory(fname, function(err0) {830            if (err0) {831              callback(err0);832              return;833            }834            cb();835          });836        } else {837          callback('File is not a file or directory: ' + fname);838          return;839        }840      }841    },842    function() {843      require('fs').rmdir(tempdir_path, function(err) {844        if (err) {845          callback('Unable to remove directory: ' + tempdir_path);846          return;847        }848        callback(null);849      });850    }851  );852}853function compute_input_file_stats(inputs, callback) {854  let ret = {};855  for (let key in inputs) {856    let val = inputs[key];857    if (val instanceof Array) {858      let list = [];859      for (let ii in val) {860        // var stat0=compute_input_file_stat(val[ii]);861        let stat0 = common.stat_file(val[ii]);862        if (!stat0) {863          callback(864            'Problem computing stat for input file: ' + key + '[' + ii + ']'865          );866          return;867        }868        list.push(stat0);869      }870      ret[key] = list;871    } else {872      // var stat0=compute_input_file_stat(val);873      let stat0 = common.stat_file(val);874      if (!stat0) {875        callback('Problem computing stat for input file: ' + key);876        return;877      }878      ret[key] = stat0;879    }880  }881  callback(null, ret);882}883function check_input_file_stats_are_consistent(884  inputs,885  input_file_stats,886  callback887) {888  compute_input_file_stats(inputs, function(err, stats2) {889    if (err) {890      callback(err);891      return;892    }893    let same =894      canonical_stringify(input_file_stats) == canonical_stringify(stats2);895    if (!same) {896      callback('Detected a change in input files.');897      return;898    }899    callback(null);900  });901}902function add_processor_job_to_queue(903  spec0,904  inputs,905  outputs,906  parameters,907  callback908) {909  let doc0 = {910    spec: spec0,911    inputs: inputs,912    outputs: outputs,913    parameters: parameters,914    status: 'queued',915    queued_timestamp: new Date() - 0,916    checked_timestamp: new Date() - 0,917  };918  let job_id = common.make_random_id(10);919  doc0._id = job_id;920  db_utils.saveDocument('processor_jobs', doc0, function(err) {921    if (err) {922      callback(err);923      return;924    }925    callback(null, job_id);926  });927}928function check_queued_job_ready_to_run(job_id, callback) {929  if (debugging) console.info('check_queued_job_ready_to_run');930  db_utils.findDocuments('processor_jobs', {}, function(err, docs) {931    if (err) {932      callback(err);933      return;934    }935    let earliest_queued_index = -1;936    let this_job_index = -1;937    let num_running = 0;938    for (let i = 0; i < docs.length; i++) {939      let doc0 = docs[i];940      if (doc0.status == 'queued') {941        if (942          earliest_queued_index < 0 ||943          is_earlier_than(doc0, docs[earliest_queued_index])944        ) {945          earliest_queued_index = i;946        }947        if (doc0._id == job_id) {948          this_job_index = i;949        }950      } else if (doc0.status == 'running') {951        num_running++;952      }953      if (doc0.status == 'queued') {954        let elapsed_since_last_checked =955          new Date() - Number(doc0.checked_timestamp);956        if (elapsed_since_last_checked > 12 * 1000) {957          console.warn(958            'Removing queued processor job that has not been checked for a while.'959          );960          db_utils.removeDocuments(961            'processor_jobs', {962              _id: doc0._id,963            },964            function(err0) {965              if (err0) {966                console.error(967                  'Problem removing queued processor job from database.'968                );969              }970            }971          );972        }973      }974    }975    if (this_job_index < 0) {976      callback('Unable to find queued job in database.');977      return;978    }979    if (debugging) {980console.info('earliest_queued_index=' + earliest_queued_index);981}982    if (debugging) console.info('this_job_index=' + this_job_index);983    if (debugging) console.info('num_running=' + num_running);984    if (debugging) {985console.info(986        'max_num_simultaneous_processor_jobs=' +987        max_num_simultaneous_processor_jobs988      );989}990    if (991      num_running < max_num_simultaneous_processor_jobs &&992      earliest_queued_index == this_job_index993    ) {994      // ready995      if (debugging) console.info('looks like we are ready');996      let doc0 = docs[this_job_index];997      doc0.status = 'running';998      db_utils.saveDocument('processor_jobs', doc0, function(err) {999        if (err) {1000          callback(err);1001          return;1002        }1003        callback(null, true);1004      });1005    } else {1006      // not ready1007      if (debugging) console.info('not ready yet');1008      let doc0 = docs[this_job_index];1009      doc0.checked_timestamp = new Date() - 0;1010      db_utils.saveDocument('processor_jobs', doc0, function(err) {1011        if (err) {1012          callback(err);1013          return;1014        }1015        callback(null, false);1016      });1017    }1018  });1019  function is_earlier_than(doc0, doc1) {1020    if (Number(doc0.queued_timestamp) < Number(doc1.queued_timestamp)) {1021return true;1022} else if (Number(doc0.queued_timestamp) == Number(doc1.queued_timestamp)) {1023if (doc0._id < doc1._id) return true;1024}1025    return false;1026  }1027}1028const debugging = false;1029function wait_for_ready_run(spec0, inputs, outputs, parameters, callback) {1030  // TODO: finish this1031  if (debugging) console.info('wait_for_ready_run');1032  compute_input_file_stats(inputs, function(err, input_file_stats) {1033    if (err) {1034      callback(err);1035      return;1036    }1037    add_processor_job_to_queue(spec0, inputs, outputs, parameters, function(1038      err,1039      job_id1040    ) {1041      if (err) {1042        callback(err);1043        return;1044      }1045      do_check();1046      function do_check() {1047        if (debugging) console.info('do_check');1048        check_input_file_stats_are_consistent(1049          inputs,1050          input_file_stats,1051          function(err) {1052            if (err) {1053              callback(err);1054              return;1055            }1056            if (debugging) console.info('input file stats are consistent');1057            check_queued_job_ready_to_run(job_id, function(err, ready) {1058              if (err) {1059                callback(err);1060                return;1061              }1062              if (ready) {1063                callback(null, job_id);1064              } else {1065                setTimeout(do_check, 1000);1066              }1067            });1068          }1069        );1070      }1071    });1072  });1073}1074function erase_output_files(outputs) {1075  for (let key in outputs) {1076    let fname = outputs[key];1077    if (require('fs').existsSync(fname)) {1078      require('fs').unlinkSync(fname);1079    }1080  }1081}1082function do_run_process(1083  spec0,1084  temporary_inputs,1085  outputs,1086  temporary_outputs,1087  parameters,1088  info,1089  callback1090) {1091  erase_output_files(outputs);1092  let singularity_bind_mode=(spec0.exe_command.indexOf('$(singularity_bind)')>=0);1093  let cmd = filter_exe_command(1094    spec0.exe_command,1095    spec0,1096    temporary_inputs,1097    temporary_outputs,1098    info,1099    parameters1100  );1101  if (info.processor_command_prefix) {1102    cmd = info.processor_command_prefix + ' ' + cmd;1103  }1104  console.info('[ Running ... ] ' + cmd);1105  let timer = new Date();1106  let P = new SystemProcess();1107  P.setCommand(cmd);1108  let tempdir_path=info.tempdir_path||'';1109  if (singularity_bind_mode) {1110    tempdir_path='/tmp';1111  }1112  P.setTempdirPath(tempdir_path);1113  if ('console_out' in outputs) {1114    P.setConsoleOutFile(outputs['console_out']);1115  }1116  P.onFinished(function() {1117    if (P.error()) {1118      callback(P.error());1119      return;1120    }1121    if (!P.error()) {1122      let elapsed = new Date() - timer;1123      console.info(1124        `Elapsed time for processor ${spec0.name}: ${elapsed / 1000} sec`1125      );1126    }1127    move_outputs(temporary_outputs, outputs, function(err) {1128      if (err) {1129        callback(err);1130        return;1131      }1132      callback(null);1133    });1134  });1135  P.start();1136}1137function filter_exe_command(1138  cmd,1139  spec,1140  inputs_in,1141  outputs_in,1142  info,1143  parameters1144) {1145  let inputs = JSON.parse(JSON.stringify(inputs_in));1146  let outputs = JSON.parse(JSON.stringify(outputs_in));1147  for (let i in spec.inputs || []) {1148    let ikey = spec.inputs[i].name;1149    if (!(ikey in inputs)) inputs[ikey] = '';1150  }1151  for (let i in spec.outputs || []) {1152    let okey = spec.outputs[i].name;1153    if (!(okey in outputs)) outputs[okey] = '';1154  }1155  let iop = {};1156  for (let key in inputs) iop[key] = inputs[key];1157  for (let key in outputs) iop[key] = outputs[key];1158  for (let key in parameters) iop[key] = parameters[key];1159  let singularity_bind_string='';1160  let singularity_bind_mode=(cmd.indexOf('$(singularity_bind)')>=0);1161  let singularity_bind_index=01162  function handle_singularity_bind(key,val,tempdir) {1163    if (!val) return val;1164    if (!singularity_bind_mode) {1165      return val;1166    }1167    if ((key in inputs)||(key in outputs)) {1168      if (!val.startsWith(tempdir+'/')) {1169        console.error(`Problem with ${key}: `+val);1170        console.error('When using singularity bind, all input and output files must be in the temporary directory for this job.');1171        process.exit(-1);1172      }1173      return '/tmp/'+val.slice((tempdir+'/').length);1174    }1175    else {1176      return val;1177    }1178    1179  }1180  let arguments = [];1181  {1182    let tempdir=info.tempdir_path;1183    if (singularity_bind_mode) {1184      singularity_bind_string+=`-B ${tempdir}:/tmp `;1185      tempdir='/tmp';1186    }1187    arguments.push(`--_tempdir=${tempdir}`);1188    cmd = cmd.split('$(tempdir)').join(tempdir);1189  }1190  let argfile_lines = [];1191  for (let key in iop) {1192    let val = iop[key];1193    if (val !== undefined) {1194      if (typeof val != 'object') {1195        let val2=handle_singularity_bind(key,val,info.tempdir_path);1196        if (key != 'console_out') {1197          arguments.push(`--${key}=${val2}`);1198          argfile_lines.push(`${key}=${val2}`);1199        }1200        cmd = cmd.split('$' + key + '$').join(val2);1201      } else {1202        for (let i in val) {1203          let val2=handle_singularity_bind(key,val[i],info.tempdir_path);1204          arguments.push(`--${key}=${val2}`);1205        }1206      }1207    } else {1208      cmd = cmd.split('$' + key + '$').join('');1209    }1210  } 1211  1212  cmd = cmd.split('$(arguments)').join(arguments.join(' '));1213  if (cmd.indexOf('$(argfile)') >= 0) {1214    let argfile_fname = info.tempdir_path + '/argfile.txt';1215    if (!common.write_text_file(argfile_fname, argfile_lines.join('\n'))) {1216      console.warn('Unable to write argfile: ' + argfile_fname); // but we don't have ability to return an error. :(1217    }1218    if (singularity_bind_mode) {1219      argfile_fname='/tmp/argfile.txt';1220    }1221    cmd = cmd.split('$(argfile)').join(argfile_fname);1222  }1223  if (singularity_bind_mode) {1224    cmd = cmd.split('$(singularity_bind)').join(singularity_bind_string);  1225  }1226  return cmd;1227}1228function check_inputs_and_substitute_prvs(inputs, prefix, opts, callback) {1229  let ikeys = Object.keys(inputs);1230  common.foreach_async(1231    ikeys,1232    function(ii, key, cb) {1233      if (typeof inputs[key] != 'object') {1234        let fname = inputs[key];1235        if (!fname.startsWith('kbucket://') &&1236          !fname.startsWith('sha1://') &&1237          !fname.startsWith('http://') &&1238          !fname.startsWith('https://')1239        ) {1240fname = require('path').resolve(process.cwd(), fname);1241}1242        let KBC = new KBClient();1243        let opts0 = {1244          download_if_needed: true,1245        };1246        if (!common.ends_with(fname, '.prv') &&1247          !file_exists(fname) &&1248          file_exists(fname + '.prv')1249        ) {1250          fname = fname + '.prv';1251        }1252        KBC.realizeFile(fname, opts0)1253          .then(function(path) {1254            inputs[key] = path;1255            cb();1256          })1257          .catch(function(err) {1258            callback(1259              `Error in input ${(prefix || '') + key} (${fname}): ${err}`1260            );1261          });1262      } else {1263        check_inputs_and_substitute_prvs(inputs[key], key + '/', opts, function(1264          err1265        ) {1266          if (err) {1267            callback(err);1268            return;1269          }1270          cb();1271        });1272      }1273    },1274    function() {1275      callback(null);1276    }1277  );1278}1279function get_file_extension_including_dot_ignoring_prv(prv_fname) {1280  let fname = require('path').basename(prv_fname);1281  if (common.ends_with(fname, '.prv')) {1282    fname = fname.slice(0, fname.length - 4);1283  }1284  let list = fname.split('.');1285  if (list.length == 0) return '.';1286  return '.' + (list[list.length - 1] || '');1287}1288/*1289   function get_file_extension_including_dot(fname) {1290   var list1 = fname.split('/');1291   var list2 = list1[list1.length - 1].split('.');1292   if (list2.length >= 2) {1293//important: must have length at least 2, otherwise extension is empty1294return '.' + list2[list2.length - 1];1295} else {1296return '';1297}1298}1299*/1300function check_outputs_and_substitute_prvs(1301  outputs,1302  process_signature,1303  callback1304) {1305  let pending_output_prvs = [];1306  if (DEBUG) {1307    console.log(JSON.stringify(outputs));1308  }1309  let okeys = Object.keys(outputs);1310  let tmp_dir = common.temporary_directory();1311  common.foreach_async(okeys, function(ii, key, cb) {1312    let fname = outputs[key];1313    if (DEBUG) {1314      console.log('Processing ouput - '+fname);1315      console.log(fname instanceof Array);1316    }1317    if (fname instanceof Array) {1318      console.log('Trying Recursive Resolve');1319      outputs[key] = fname.map(function c(fnm, i) {1320        let tmp = resolve_file(fnm, pending_output_prvs, tmp_dir, key, process_signature);1321        console.log(tmp);1322        return tmp;1323      });1324    } else { // Process File1325      outputs[key] = resolve_file(fname, pending_output_prvs, tmp_dir, key, process_signature);1326    }1327    console.log(JSON.stringify(outputs));1328    cb();1329  }, function() {1330    callback(null, {1331      pending_output_prvs: pending_output_prvs,1332    });1333  });1334}1335function resolve_file(fname, pending_output_prvs, tmp_dir, key, process_signature) {1336  fname = require('path').resolve(process.cwd(), fname);1337  if (common.ends_with(fname, '.prv')) {1338    let file_extension_including_dot = get_file_extension_including_dot_ignoring_prv(fname);1339    let fname2 = tmp_dir + `/output_${process_signature}_${key}${file_extension_including_dot}`;1340    pending_output_prvs.push({1341      name: key,1342      prv_fname: fname,1343      output_fname: fname2,1344    });1345    return fname2;1346  }1347  return fname;1348}1349function make_temporary_outputs(outputs, process_signature, info, callback) {1350  let temporary_outputs = {};1351  let okeys = Object.keys(outputs);1352  common.foreach_async(okeys, function(ii, key, cb) {1353    let fname = outputs[key];1354    if (DEBUG) {1355      console.log('Processing ouput - '+fname);1356      console.log(fname instanceof Array);1357    }1358    if (fname instanceof Array) {1359      temporary_outputs[key] = fname.map(function(fnm, i, arr) {1360        fnm = require('path').resolve(process.cwd(), fnm);1361        let file_extension_including_dot = get_file_extension_including_dot_ignoring_prv(fnm);1362        return info.tempdir_path + `/output_${key}_${i}${file_extension_including_dot}`;1363      });1364    } else {1365      fname = require('path').resolve(process.cwd(), fname);1366      let file_extension_including_dot = get_file_extension_including_dot_ignoring_prv(fname);1367      temporary_outputs[key] = info.tempdir_path + `/output_${key}${file_extension_including_dot}`;1368    }1369    cb();1370  }, function() {1371    callback(null, {1372      temporary_outputs: temporary_outputs,1373    });1374  }1375  );1376}1377function link_inputs(inputs, original_inputs, info, callback) {1378  let ret = {1379    temporary_inputs: JSON.parse(JSON.stringify(inputs)),1380  };1381  info.key_prefix = info.key_prefix || '';1382  let ikeys = Object.keys(inputs);1383  common.foreach_async(1384    ikeys,1385    function(ii, key, cb) {1386      let fname = inputs[key];1387      if (typeof fname != 'object') {1388        let original_fname = original_inputs[key];1389        fname = require('path').resolve(process.cwd(), fname);1390        let file_extension_including_dot = get_file_extension_including_dot_ignoring_prv(fname);1391        let desired_file_extension_including_dot = get_file_extension_including_dot_ignoring_prv(original_fname);1392        //if (file_extension_including_dot == desired_file_extension_including_dot) {1393        //  cb();1394        //  return;1395        //}1396        /* get_file_extension_including_dot(1397            original_fname1398            );*/1399        let new_fname = info.tempdir_path+`/input_${info.key_prefix}${key}_${common.make_random_id(8)}${desired_file_extension_including_dot}`;1400        //let new_fname = `${fname}.tmplink.${common.make_random_id(8)}${desired_file_extension_including_dot}`;1401        make_hard_link_or_copy(fname, new_fname, function(err) {1402          if (err) {1403            callback(`Error creating hard link for input: ${fname} -> ${new_fname}: ` + err);1404            return;1405          }1406          ret.temporary_inputs[key] = new_fname;1407          //files_to_cleanup.push(new_fname);1408          cb();1409        });1410      } else {1411        let info2 = JSON.parse(JSON.stringify(info));1412        info2.key_prefix = key + '-';1413        link_inputs(inputs[key], original_inputs[key], info2, function(err, tmp) {1414          if (err) {1415            callback(err);1416            return;1417          }1418          ret.temporary_inputs[key] = tmp.temporary_inputs;1419          cb();1420        });1421      }1422    },1423    function() {1424      callback(null, ret);1425    }1426  );1427}1428function make_hard_link_or_copy(src_fname, dst_fname, callback) {1429  require('fs').link(src_fname, dst_fname, function(err) {1430    if (err) {1431      console.warn(1432        `This is only a warning: Unable to hard link file ${src_fname} -> ${dst_fname}. Perhaps temporary directory is not on the same device as the output file directory. Copying instead.`1433      );1434      require('fs').copyFile(src_fname, dst_fname, function(err) {1435        if (err) {1436          callback(1437            `Error hard linking or copying file ${src_fname} -> ${dst_fname}: ${err.message}`1438          );1439          return;1440        }1441        callback(null);1442      });1443      return;1444    }1445    callback(null);1446  });1447}1448function compute_process_signature(spec0, inputs, parameters, callback) {1449  compute_process_signature_object(spec0, inputs, parameters, function(1450    err,1451    obj1452  ) {1453    if (err) {1454      callback(err);1455      return;1456    }1457    callback(null, sha1(canonical_stringify(obj)));1458  });1459}1460function compute_process_signature_object(spec0, inputs, parameters, callback) {1461  let signature_object = {};1462  signature_object.version = '0.1';1463  signature_object.processor_name = spec0.name;1464  signature_object.processor_version = spec0.version;1465  signature_object.parameters = parameters;1466  compute_process_signature_object_inputs(inputs, function(err, inputs0) {1467    if (err) {1468      callback(err);1469      return;1470    }1471    signature_object.inputs = inputs0;1472    callback('', signature_object);1473  });1474}1475function compute_process_signature_object_inputs(inputs, callback) {1476  // See the warning for the outputs and file extensions elsewhere in this file1477  get_checksums_for_files(1478    inputs, {1479      mode: 'process_signature',1480    },1481    callback1482  );1483}1484function find_in_process_cache(process_signature, outputs, callback) {1485  db_utils.findDocuments(1486    'process_cache', {1487      process_signature: process_signature,1488    },1489    function(err, docs) {1490      if (err) {1491        callback(err);1492        return;1493      }1494      if (docs.length == 0) {1495        callback(null, null);1496        return;1497      }1498      async.eachSeries(docs, function(doc, cb) {1499        check_outputs_consistent_with_process_cache(outputs, doc, function(1500          err,1501          consistent,1502          msg1503        ) {1504          if (consistent) {1505            callback(null, doc);1506          } else {1507            cb();1508          }1509        });1510      }, function() {1511        callback(null, null);1512      });1513    }1514  );1515}1516function check_outputs_consistent_with_process_cache(outputs, doc0, callback) {1517  /*1518     Warning: if we ever decide to allow copying of outputs with different1519     file paths, we need to make sure we respect the file extension,1520     because processor behavior may be different depending on the1521     output file extension.1522     Note that we don't have to worry about such things for input files,1523     because the sha-1 hash is computed for those. Persumably if the1524     extension is different, and it matters for functionality, then1525     the sha-1 will be different as well. (I suppose there could be1526     an exception to this, but I'd be surprised)1527     */1528  for (let key in outputs) {1529    let fname = outputs[key];1530    let stat0 = common.stat_file(fname);1531    if (!stat0) {1532      callback(`Unable to stat output file (${key}): ${fname}`);1533      return;1534    }1535    let output0 = doc0.outputs[key] || {};1536    if (output0.path != fname) {1537      callback(null, false, `${output0.path} <> ${fname}`);1538      return;1539    }1540    if (1541      output0.size != stat0.size ||1542      output0.mtime != stat0.mtime.toISOString() ||1543      // output0.ctime != stat0.ctime.toISOString() || //not sure why the ctime is having trouble1544      output0.ino != stat0.ino1545    ) {1546      callback(null, false, `Stats do not match: ${output0.size} ${stat0.size} ${output0.mtime} ${stat0.mtime.toISOString()} ${output0.ctime} ${stat0.ctime.toISOString()} ${output0.ino} ${stat0.ino} ${fname}`);1547      return;1548    }1549  }1550  callback(null, true, '');1551}1552function save_to_process_cache(1553  process_signature,1554  spec0,1555  inputs,1556  outputs,1557  parameters,1558  callback1559) {1560  db_utils.removeDocuments(1561    'process_cache', {1562      process_signature: process_signature,1563    },1564    function(err) {1565      if (err) {1566        callback(err);1567        return;1568      }1569      get_checksums_for_files(1570        inputs, {1571          mode: 'process_cache',1572        },1573        function(err, inputs_with_checksums) {1574          if (err) {1575            callback(err);1576            return;1577          }1578          get_checksums_for_files(1579            outputs, {1580              mode: 'process_cache',1581            },1582            function(err, outputs_with_checksums) {1583              if (err) {1584                callback(err);1585                return;1586              }1587              let doc0 = {1588                process_signature: process_signature,1589                spec: spec0,1590                inputs: inputs_with_checksums,1591                outputs: outputs_with_checksums,1592                parameters: parameters,1593              };1594              db_utils.saveDocument('process_cache', doc0, function(err) {1595                callback(err);1596              });1597            }1598          );1599        }1600      );1601    }1602  );1603}1604function get_checksums_for_files(inputs, opts, callback) {1605  let ret = {};1606  let keys = Object.keys(inputs);1607  common.foreach_async(1608    keys,1609    function(ii, key, cb) {1610      let val = inputs[key];1611      if (typeof val != 'object') {1612        let stat0 = common.stat_file(val);1613        if (!stat0) {1614          callback(`Unable to stat file (${key}): ${val}`);1615          return;1616        }1617        prv_utils.compute_file_sha1(val, function(err, sha1) {1618          if (err) {1619            callback(err);1620            return;1621          }1622          if (opts.mode == 'process_cache') {1623            ret[key] = {1624              path: val,1625              sha1: sha1,1626              mtime: stat0.mtime.toISOString(),1627              // ctime: stat0.ctime.toISOString(), //not sure why the ctime is having trouble1628              size: stat0.size,1629              ino: stat0.ino,1630            };1631          } else if (opts.mode == 'process_signature') {1632            ret[key] = sha1;1633          } else {1634            callback(1635              'Unexpected mode in get_checksums_for_files: ' + opts.mode1636            );1637            return;1638          }1639          cb();1640        });1641      } else {1642        get_checksums_for_files(inputs[key], opts, function(err, tmp) {1643          if (err) {1644            callback(err);1645            return;1646          }1647          ret[key] = tmp;1648          cb();1649        });1650      }1651    },1652    function() {1653      callback(null, ret);1654    }1655  );1656}1657function separate_iops(1658  inputs,1659  outputs,1660  parameters,1661  iops,1662  spec_inputs,1663  spec_outputs,1664  spec_parameters1665) {1666  let B_inputs = {};1667  for (let i in spec_inputs) {1668    let key0 = spec_inputs[i].name;1669    if (key0) {1670      B_inputs[key0] = spec_inputs[i];1671    }1672  }1673  let B_outputs = {};1674  for (let i in spec_outputs) {1675    let key0 = spec_outputs[i].name;1676    if (key0) {1677      B_outputs[key0] = spec_outputs[i];1678    }1679  }1680  let B_parameters = {};1681  for (let i in spec_parameters) {1682    let key0 = spec_parameters[i].name;1683    if (key0) {1684      B_parameters[key0] = spec_parameters[i];1685    }1686  }1687  for (let key in iops) {1688    if (key in B_inputs) {1689      inputs[key] = iops[key];1690    } else if (key in B_outputs) {1691      outputs[key] = iops[key];1692    } else if (key in B_parameters) {1693      parameters[key] = iops[key];1694    } else {1695      throw new Error(`Unexpected argument: ${key}`);1696    }1697  }1698}1699function check_iop(A, Bspec, iop_name, opts) {1700  if (!opts) opts = {};1701  let B = {};1702  for (let i in Bspec) {1703    let key0 = Bspec[i].name;1704    if (key0) {1705      B[key0] = Bspec[i];1706    }1707  }1708  for (let key in A) {1709    if (!(key in B)) {1710      throw new Error(`Unexpected ${iop_name}: ${key}`);1711    }1712  }1713  for (let key in B) {1714    if (!(key in A)) {1715      if (B[key].optional) {1716        if (opts.substitute_defaults) {1717          A[key] = B[key].default_value;1718        }1719      } else {1720        throw new Error(`Missing required ${iop_name}: ${key}`);1721      }1722    }1723  }1724  return true;1725}1726function parse_iop(str, iop_name) {1727  let ret = {};1728  let list = str.split('[---]');1729  for (let i in list) {1730    let str0 = list[i];1731    if (str0) {1732      let ind0 = str0.indexOf(':');1733      if (ind0 < 0) {1734        throw new Error(`Error in ${iop_name}: ${str0}`);1735      }1736      let key0 = str0.slice(0, ind0);1737      let val0 = str0.slice(ind0 + 1);1738      if (!(key0 in ret)) {1739        ret[key0] = val0;1740      } else {1741        if (typeof ret[key0] != 'object') {1742          ret[key0] = [ret[key0], val0];1743        } else {1744          ret[key0].push(val0);1745        }1746      }1747    }1748  }1749  return ret;1750}1751function file_exists(fname) {1752  return require('fs').existsSync(fname);...

1#! /usr/bin/env node2var port = process.env.PORT || 8000;3var root_path = (function _init_root_path () {4  var fs = require('fs');5  var path = require('path');6  return path.dirname(fs.realpathSync(__filename));7})();8// TODO this has to be "per user / account"9// TODO this has to be user modifiable10var path = require('path');11// all keys are `echo -n '{name}' | sha1sum | awk '{print$1}'`12// TODO spec.json, desc13var primitive_program_table = {14  "/eefffcb0e1ac7d638f1e5fa98d62b7ef79e2652d": {15    "name": "unpack archive",16    "command": path.join(root_path, 'components', 'unpack-archive', 'index')17  },18  "/7f3be10a123829d7cdd1b7a8e76b0dcf20a7ea45": {19    "name": "espresso build",20    "command": path.join(root_path, 'components', 'espresso-build', 'index')21  },22  "/9fa4a451844a412af7dc7655863c3a245744aad3": {23    "name": "package PhoneGap/Android",24    "command": path.join(root_path, 'components', 'package-apk', 'index')25  },26  "/2af293fe70e5be472f10b4165467ba05e5831380": {27    "name": "sendsrc",28    "command": path.join(root_path, 'components', 'sendsrc', 'index')29  },30  "/40f01312839eff6335c03b2b2b56e3c2bbeb9d15": {31    "name": "wget -m",32    "command": path.join(root_path, 'components', 'wget_-m')33  },34  "/527365ccf5cd192d7b05eb02294d8b6edc23065b": {35    "name": "generate manifest",36    "command": path.join(root_path, 'components', 'generate_manifest')37  },38};39var program_table = {40  "/dec438b22939a4c1d74bf070aa93e9745b913d82": {41    "name": "espresso build Android package",42    "program": [43      // TODO? "upload"44      "/eefffcb0e1ac7d638f1e5fa98d62b7ef79e2652d", // unpack archive45      "/7f3be10a123829d7cdd1b7a8e76b0dcf20a7ea45", // espresso build46      "/9fa4a451844a412af7dc7655863c3a245744aad3", // package PhoneGap/Android47      "/2af293fe70e5be472f10b4165467ba05e5831380", // sendsrc48      // TODO? "download"49    ]50  },51  "/82f794873c12912aac592f82f866f237cd195e40": {52    "name": "generate manifest from URL",53    "program": [54      "/40f01312839eff6335c03b2b2b56e3c2bbeb9d15", // wget -m55      "/527365ccf5cd192d7b05eb02294d8b6edc23065b", // generate manifest56      "/2af293fe70e5be472f10b4165467ba05e5831380", // sendsrc57    ]58  },59};60// TODO61//  prologue (create temporary directory, upload files, setup environment,62//            chroot, etc.)63//  agenda64//  epilogue (send response, cleanup; inverse of prologue)65var handle_externally = (function _init_handle_externally () {66  var spawn = require('child_process').spawn;67  var path = require('path');68  return function _impl_handle_externally (req, res, options, callback) {69    var log = options.log;70    if (req.method === 'POST') {71      var env = JSON.parse(JSON.stringify(options.env)); // copy72      // the external handler may run a long time... don't time out.73      res.socket.setTimeout(0 * 60 * 1000);74      var agenda = [ req.url ];75      (function _execute_loop () {76        log.format('agenda', agenda)77        if (agenda.length > 0) {78          // pop head79          var id = agenda[0];80          agenda = agenda.slice(1);81          if (id in primitive_program_table) {82            var program = primitive_program_table[id];83            log.format('execute primitive program', program.name);84            var child = (function _init_child () {85              var command = program.command;86              // TODO merge some optionally predefined program.envp87              var argv = program.argv || [];88              var spawn_options = {89                cwd: options.cwd || root_path,90                env: env91              };92              log.format('spawn', { command: command, options: spawn_options });93              return spawn(command, argv, spawn_options);94            })();95            // TODO minimal buffering96            var data = [];97            child.stdout.on('data', function _cb_buffer_res_chunk (chunk) {98              data.push(chunk);99            });100            101            child.stderr.on('data', function _cb_write_child_stderr (chunk) {102              log.raw(chunk.toString(), '');103            });104            105            child.on('exit', function _cb_child_exited (code) {106              log.format('child', 'exit', code);107              if (code === 0) {108                if (agenda.length === 0) {109                  // TODO do we have a use case to not generate the whole HTTP110                  // response?111                  data.forEach(function _cb_write_res_chunks (chunk) {112                    res.socket.write(chunk);113                  });114                  res.socket.end();115                  callback(); // TODO ok, no error116                } else {117                  // TODO write logfiles?118                  data.forEach(function _cb_write_intermediate_chunks (chunk) {119                    log.raw(chunk.toString(), '');120                  });121                  // continue122                  process.nextTick(_execute_loop);123                };124              } else {125                res.writeHead(500, {126                  'Content-Length': 0127                });128                res.end();129                callback(); // TODO error130              };131            });132          } else if (id in program_table) {133            var sub = program_table[id];134            log.format('schedule program', sub.name,135                '=\n' + require('util').inspect(sub.program));136            // reschedule137            agenda = sub.program.concat(agenda);138            process.nextTick(_execute_loop);139          } else {140            // TODO 404 when it's the first?141            log.raw('program not found: ' + JSON.stringify(id), '');142            res.writeHead(500, {143              'Content-Length': 0144            });145            res.end();146            callback();147          };148        } else {149          log.raw('agenda is empty', '');150          res.writeHead(500, {151            'Content-Length': 0152          });153          res.end();154          callback(); // TODO error155        };156      })();157    } else {158      log.raw('Method not allowed', '');159      res.writeHead(405, {160        'Content-Length': 0161      });162      res.end();163      callback(); // TODO error164    };165  };166})();167var mktempdir = (function _init_mktempdir () {168  var uuid = require('node-uuid');169  var fs = require('fs');170  var path = require('path');171  return function _mktempdir (callback) {172    // TODO portable path173    var dirname = path.join('/tmp', uuid());174    var mode = '0700'175    fs.mkdir(dirname, mode, function (err) {176      // TODO cycle177      callback(err, dirname);178    });179  };180})();181var rmfR = (function _init_rmfR () {182  var fs = require('fs');183  var path = require('path');184  return function _rmfR (name, callback) {185    fs.stat(name, function(err, stat) {186      if (err) {187        console.error('stat Error', err);188      };189      if (stat.isFile()) {190        fs.unlink(name, function (err) {191          console.log('unlink', name, err);192          callback(null);193        });194      } else {195        fs.readdir(name, function (err, files) {196          if (err) {197            console.error('readdir Error', err);198            callback(null);199          } else {200            (function _rec () {201              if (files.length > 0) {202                _rmfR(path.join(name, files.pop()), _rec);203              } else {204                fs.rmdir(name, function (err) {205                  console.log('rmdir', name, err);206                  callback(null);207                });208              };209            })();210          };211        });212      };213    });214  };215})();216var listener = (function _init_listener () {217  218  var IncomingForm = require('formidable').IncomingForm;219  var Logger = require('./log');220  var url = require('url');221  return function _impl_listener (req, res) {222    var log = Logger.create();223    log.raw([req.method, req.url, 'HTTP/' + req.httpVersion].join(' '));224    log.format(req.headers);225    var re = new RegExp('^multipart/form-data(?:;|$)');226    if (re.test(req.headers['content-type'])) {227      var form = new IncomingForm();228      form.parse(req, function(err, fields, files) {229        (function _scope_pretty_print_form () {230          var pretty = {};231          Object.keys(fields).forEach(function _cb_cpy_fields (key) {232            pretty[key] = fields[key];233          });234          Object.keys(files).forEach(function _cb_cpy_partial_files (key) {235            pretty[key] = {};236            [ 'name', 'size', 'type', 'path'237            ].forEach(function _cb_cpy_some_file_props (property) {238              pretty[key][property] = files[key][property];239            });240          });241          log.format(pretty);242        })();243        // TODO whitelist fields and files?244        var env = {};245        Object.keys(fields).forEach(function _cb_export_fields (key) {246          env['TEXT_' + key] = fields[key];247        });248        Object.keys(files).forEach(function _cb_export_files (key) {249          env['FILE_' + key] = files[key].path;250          env['FILE_' + key + '_SIZE'] = files[key].size;251          env['FILE_' + key + '_TYPE'] = files[key].type;252        });253        // TODO? load environment from query string254        mktempdir(function (err, tempdir_path) {255          if (err) {256            log.raw('mktempdir err:' + err, '');257            res.writeHead(500, {258              'Content-Length': 0259            });260            res.end();261          } else {262            log.format('tempdir_path', tempdir_path);263            var options = {264              log: log,265              env: env,266              cwd: tempdir_path267            };268            handle_externally(req, res, options, function _cb_form_cleanup () {269              var fs = require('fs');270              Object.keys(files).forEach(function _cb_unlink_files (key) {271                var file = files[key];272                fs.unlink(file.path, function _cb_file_unlinked (exn) {273                  log.format('unlink', file.path, exn);274                });275              });276              rmfR(tempdir_path, function (err) {277                log.format('rm -fR', tempdir_path, err);278              });279            });280          };281        });282      });283    } else {284      var env = {};285      mktempdir(function (err, tempdir_path) {286        if (err) {287          log.raw('mktempdir err:' + err, '');288          res.writeHead(500, {289            'Content-Length': 0290          });291          res.end();292        } else {293          log.format('tempdir_path', tempdir_path);294          // load environment from query string295          req.url = url.parse(req.url, true);296          Object.keys(req.url.query).forEach(function (key) {297            env['TEXT_' + key] = req.url.query[key];298          });299          req.url = req.url.pathname;300          var options = {301            log: log,302            env: env,303            cwd: tempdir_path304          };305          handle_externally(req, res, options, function _cb_noformclean () {306            rmfR(tempdir_path, function (err) {307              log.format('rm -fR', tempdir_path, err);308            });309          });310        };311      });312    };313  };314})();315316(function _scope_setup_https_server () {317  var https = require('https');318  var fs = require('fs');319  var path = require('path');320  var keys_path = path.join(root_path, 'test', 'fixtures', 'keys');321  var key_path = path.join(keys_path, 'agent2-key.pem');322  var cert_path = path.join(keys_path, 'agent2-cert.pem');323  console.log('key:', key_path);324  console.log('cert:', cert_path);325  var options = {326    key: fs.readFileSync(key_path),327    cert: fs.readFileSync(cert_path)328  };329  https.createServer(options, listener).listen(port, function _cb_motd () {330    console.log('Serving HTTPS on 0.0.0.0 port', port);331  });332})();333334(function _scope_setup_suicide_facility () {335  process.on('SIGTERM', function _cb_sigterm () {336    console.log('Got SIGTERM ==> shutting down...');337    process.exit(0);338  });339  var fs = require('fs');340  var Inotify = require('inotify').Inotify;341  var inotify = new Inotify();342  inotify.addWatch({343    path: fs.realpathSync(__filename), // TODO all the files344    watch_for: Inotify.IN_ATTRIB345             | Inotify.IN_DELETE_SELF346             | Inotify.IN_MODIFY347             | Inotify.IN_MOVE_SELF348             ,349    callback: function _cb_hara_kiri (event) {350      process.kill(process.pid, 'SIGTERM');351    }352  });...