How to use markSystemFilesForCleanup method in Appium Xcuitest Driver

...381        await adjustWDAAttachmentsPermissions(this.wda, this.opts.preventWDAAttachments ? '555' : '755');382        this.logEvent('wdaPermsAdjusted');383      }384      if (this.opts.clearSystemFiles) {385        await markSystemFilesForCleanup(this.wda);386      }387      // we expect certain socket errors until this point, but now388      // mark things as fully working389      this.wda.fullyStarted = true;390      this.logEvent('wdaStarted');391    });392  }393  // create an alias so we can actually unit test createSession by stubbing394  // this395  async extractBundleId (app) {396    return await appUtils.extractBundleId(app);397  }398  async runReset (opts = null) {399    this.logEvent('resetStarted');...

1import B from 'bluebird';2import { fs, util } from 'appium-support';3import path from 'path';4import { utils as iosUtils } from 'appium-ios-driver';5import { SubProcess, exec } from 'teen_process';6import xcode from 'appium-xcode';7import _ from 'lodash';8import log from './logger';9const DEFAULT_TIMEOUT_KEY = 'default';10async function detectUdid () {11  log.debug('Auto-detecting real device udid...');12  let  cmd, args = [];13  try {14    cmd = await fs.which('idevice_id');15    args.push('-l');16    log.debug('Using idevice_id');17  } catch (err) {18    log.debug('Using udidetect');19    cmd = require.resolve('udidetect');20  }21  let udid;22  try {23    let {stdout} = await exec(cmd, args, {timeout: 3000});24    let udids = _.filter(stdout.split('\n'), Boolean);25    udid = _.last(udids);26    if (udids.length > 1) {27      log.warn(`Multiple devices found: ${udids.join(', ')}`);28      log.warn(`Choosing '${udid}'. If this is wrong, manually set with 'udid' desired capability`);29    }30  } catch (err) {31    log.errorAndThrow(`Error detecting udid: ${err.message}`);32  }33  if (!udid || udid.length <= 2) {34    throw new Error('Could not detect udid.');35  }36  log.debug(`Detected real device udid: '${udid}'`);37  return udid;38}39async function getAndCheckXcodeVersion () {40  let version;41  try {42    version = await xcode.getVersion(true);43  } catch (err) {44    log.debug(err);45    log.errorAndThrow(`Could not determine Xcode version: ${err.message}`);46  }47  if (!version.toolsVersion) {48    try {49      version.toolsVersion = await xcode.getCommandLineToolsVersion();50    } catch (ign) {}51  }52  // we do not support Xcodes < 7.3,53  if (version.versionFloat < 7.3) {54    log.errorAndThrow(`Xcode version '${version.versionString}'. Support for ` +55                      `Xcode ${version.versionString} is not supported. ` +56                      `Please upgrade to version 7.3 or higher`);57  }58  return version;59}60async function getAndCheckIosSdkVersion () {61  let versionNumber;62  try {63    versionNumber = await xcode.getMaxIOSSDK();64  } catch (err) {65    log.errorAndThrow(`Could not determine iOS SDK version: ${err.message}`);66  }67  return versionNumber;68}69async function translateDeviceName (xcodeVersion, platformVersion, devName = '') {70  let deviceName = devName;71  switch (devName.toLowerCase().trim()) {72    case 'iphone simulator':73      deviceName = 'iPhone 6';74      break;75    case 'ipad simulator':76      // no need to worry about floating point comparison because of the77      //   nature of the numbers being compared78      // iPad Retina is no longer available for ios 10.379      //   so we pick another iPad to use as default80      deviceName = (parseFloat(platformVersion) < 10.3) ? 'iPad Retina' : 'iPad Air';81      break;82    case 'iphone 8':83    case 'iphone 8 plus':84    case 'iphone x':85      // Xcode 9.0(.0) mis-named the new devices86      if (xcodeVersion.major === 9 &&87          xcodeVersion.minor === 0 &&88          (!util.hasValue(xcodeVersion.patch) || xcodeVersion.patch === 0)) {89        const namesMapping = {90          'iphone 8': 'iPhone2017-A',91          'iphone 8 plus': 'iPhone2017-B',92          'iphone x': 'iPhone2017-C'93        };94        deviceName = namesMapping[devName.toLowerCase().trim()];95      }96      break;97  }98  if (deviceName !== devName) {99    log.debug(`Changing deviceName from '${devName}' to '${deviceName}'`);100  }101  return deviceName;102}103// This map contains derived data attachment folders as keys104// and values are stacks of permssion masks105// It is used to synchronize permissions change106// on shared folders107const derivedDataPermissionsStacks = new Map();108async function adjustWDAAttachmentsPermissions (wda, perms) {109  if (!wda || !await wda.retrieveDerivedDataPath()) {110    log.warn('No WebDriverAgent derived data available, so unable to set permissions on WDA attachments folder');111    return;112  }113  const attachmentsFolder = path.join(await wda.retrieveDerivedDataPath(), 'Logs/Test/Attachments');114  const permsStack = derivedDataPermissionsStacks.get(attachmentsFolder) || [];115  if (permsStack.length) {116    if (_.last(permsStack) === perms) {117      permsStack.push(perms);118      log.info(`Not changing permissions of '${attachmentsFolder}' to '${perms}', because they were already set by the other session`);119      return;120    }121    if (permsStack.length > 1) {122      permsStack.pop();123      log.info(`Not changing permissions of '${attachmentsFolder}' to '${perms}', because the other session does not expect them to be changed`);124      return;125    }126  }127  derivedDataPermissionsStacks.set(attachmentsFolder, [perms]);128  if (await fs.exists(attachmentsFolder)) {129    log.info(`Setting '${perms}' permissions to '${attachmentsFolder}' folder`);130    await fs.chmod(attachmentsFolder, perms);131    return;132  }133  log.info(`There is no ${attachmentsFolder} folder, so not changing permissions`);134}135// This map contains derived data logs folders as keys136// and values are the count of times the particular137// folder has been scheduled for removal138const derivedDataCleanupMarkers = new Map();139async function markSystemFilesForCleanup (wda) {140  if (!wda || !await wda.retrieveDerivedDataPath()) {141    log.warn('No WebDriverAgent derived data available, so unable to mark system files for cleanup');142    return;143  }144  const logsRoot = path.resolve(await wda.retrieveDerivedDataPath(), 'Logs');145  let markersCount = 0;146  if (derivedDataCleanupMarkers.has(logsRoot)) {147    markersCount = derivedDataCleanupMarkers.get(logsRoot);148  }149  derivedDataCleanupMarkers.set(logsRoot, ++markersCount);150}151async function clearSystemFiles (wda) {152  // only want to clear the system files for the particular WDA xcode run153  if (!wda || !await wda.retrieveDerivedDataPath()) {154    log.warn('No WebDriverAgent derived data available, so unable to clear system files');155    return;156  }157  const logsRoot = path.resolve(await wda.retrieveDerivedDataPath(), 'Logs');158  if (derivedDataCleanupMarkers.has(logsRoot)) {159    let markersCount = derivedDataCleanupMarkers.get(logsRoot);160    derivedDataCleanupMarkers.set(logsRoot, --markersCount);161    if (markersCount > 0) {162      log.info(`Not cleaning '${logsRoot}' folder, because the other session does not expect it to be cleaned`);163      return;164    }165  }166  derivedDataCleanupMarkers.set(logsRoot, 0);167  // Cleaning up big temporary files created by XCTest: https://github.com/appium/appium/issues/9410168  const cleanupCmd = `find -E /private/var/folders ` +169    `-regex '.*/Session-WebDriverAgentRunner.*\\.log$|.*/StandardOutputAndStandardError\\.txt$' ` +170    `-type f -exec sh -c 'echo "" > "{}"' \\;`;171  const cleanupTask = new SubProcess('bash', ['-c', cleanupCmd], {172    detached: true,173    stdio: ['ignore', 'pipe', 'pipe'],174  });175  await cleanupTask.start(0);176  // Do not wait for the task to be completed, since it might take a lot of time177  // We keep it running after Appium process is killed178  cleanupTask.proc.unref();179  log.debug(`Started background XCTest logs cleanup: ${cleanupCmd}`);180  if (await fs.exists(logsRoot)) {181    log.info(`Cleaning test logs in '${logsRoot}' folder`);182    await iosUtils.clearLogs([logsRoot]);183    return;184  }185  log.info(`There is no ${logsRoot} folder, so not cleaning files`);186}187async function checkAppPresent (app) {188  log.debug(`Checking whether app '${app}' is actually present on file system`);189  if (!(await fs.exists(app))) {190    log.errorAndThrow(`Could not find app at '${app}'`);191  }192  log.debug('App is present');193}194async function getDriverInfo () {195  let stat = await fs.stat(path.resolve(__dirname, '..'));196  let built = stat.mtime.getTime();197  // get the package.json and the version from it198  let pkg = require(__filename.indexOf('build/lib/utils') !== -1 ? '../../package.json' : '../package.json');199  let version = pkg.version;200  let info = {201    built,202    version,203  };204  return info;205}206function normalizeCommandTimeouts (value) {207  // The value is normalized already208  if (typeof value !== 'string') {209    return value;210  }211  let result = {};212  // Use as default timeout for all commands if a single integer value is provided213  if (!isNaN(value)) {214    result[DEFAULT_TIMEOUT_KEY] = _.toInteger(value);215    return result;216  }217  // JSON object has been provided. Let's parse it218  try {219    result = JSON.parse(value);220    if (!_.isPlainObject(result)) {221      throw new Error();222    }223  } catch (err) {224    log.errorAndThrow(`"commandTimeouts" capability should be a valid JSON object. "${value}" was given instead`);225  }226  for (let [cmd, timeout] of _.toPairs(result)) {227    if (!_.isInteger(timeout) || timeout <= 0) {228      log.errorAndThrow(`The timeout for "${cmd}" should be a valid natural number of milliseconds. "${timeout}" was given instead`);229    }230  }231  return result;232}233/**234 * Get the process id of the most recent running application235 * having the particular command line pattern.236 *237 * @param {string} pgrepPattern - pgrep-compatible search pattern.238 * @return {string} Either a process id or null if no matches were found.239 */240async function getPidUsingPattern (pgrepPattern) {241  const args = ['-nif', pgrepPattern];242  try {243    const {stdout} = await exec('pgrep', args);244    const pid = parseInt(stdout, 10);245    if (isNaN(pid)) {246      log.debug(`Cannot parse process id from 'pgrep ${args.join(' ')}' output: ${stdout}`);247      return null;248    }249    return `${pid}`;250  } catch (err) {251    log.debug(`'pgrep ${args.join(' ')}' didn't detect any matching processes. Return code: ${err.code}`);252    return null;253  }254}255/**256 * Kill a process having the particular command line pattern.257 * This method tries to send SIGINT, SIGTERM and SIGKILL to the258 * matched processes in this order if the process is still running.259 *260 * @param {string} pgrepPattern - pgrep-compatible search pattern.261 */262async function killAppUsingPattern (pgrepPattern) {263  for (const signal of [2, 15, 9]) {264    if (!await getPidUsingPattern(pgrepPattern)) {265      return;266    }267    const args = [`-${signal}`, '-if', pgrepPattern];268    try {269      await exec('pkill', args);270    } catch (err) {271      log.debug(`pkill ${args.join(' ')} -> ${err.message}`);272    }273    await B.delay(100);274  }275}276/**277 * Kills running XCTest processes for the particular device.278 *279 * @param {string} udid - The device UDID.280 * @param {boolean} isSimulator - Equals to true if the current device is a Simulator281 * @param {object} opts - Additional options mapping. Possible keys are:282 *   - {string|number} wdaLocalPort: The number of local port WDA is listening on.283 */284async function resetXCTestProcesses (udid, isSimulator, opts = {}) {285  const processPatterns = [`xcodebuild.*${udid}`];286  if (opts.wdaLocalPort) {287    processPatterns.push(`iproxy ${opts.wdaLocalPort}`);288  } else if (!isSimulator) {289    processPatterns.push(`iproxy.*${udid}`);290  }291  if (isSimulator) {292    processPatterns.push(`${udid}.*XCTRunner`);293  }294  log.debug(`Killing running processes '${processPatterns.join(', ')}' for the device ${udid}...`);295  for (const pgrepPattern of processPatterns) {296    await killAppUsingPattern(pgrepPattern);297  }298}299async function printUser () {300  try {301    let {stdout} = await exec('whoami');302    log.debug(`Current user: '${stdout.trim()}'`);303  } catch (err) {304    log.debug(`Unable to get username running server: ${err.message}`);305  }306}307async function printLibimobiledeviceInfo () {308  try {309    let {stdout} = await exec('brew', ['info', 'libimobiledevice']);310    let match = /libimobiledevice:(.+)/.exec(stdout);311    if (match && match[1]) {312      log.debug(`Current version of libimobiledevice: ${match[1].trim()}`);313    }314  } catch (err) {315    log.debug(`Unable to get version of libimobiledevice: ${err.message}`);316  }317}318/**319 * Get the IDs of processes listening on the particular system port.320 * It is also possible to apply additional filtering based on the321 * process command line.322 *323 * @param {string|number} port - The port number.324 * @param {?Function} filteringFunc - Optional lambda function, which325 *                                    receives command line string of the particular process326 *                                    listening on given port, and is expected to return327 *                                    either true or false to include/exclude the corresponding PID328 *                                    from the resulting array.329 * @returns {Array<string>} - the list of matched process ids.330 */331async function getPIDsListeningOnPort (port, filteringFunc = null) {332  const result = [];333  try {334    // This only works since Mac OS X El Capitan335    const {stdout} = await exec('lsof', ['-ti', `tcp:${port}`]);336    result.push(...(stdout.trim().split(/\n+/)));337  } catch (e) {338    return result;339  }340  if (!_.isFunction(filteringFunc)) {341    return result;342  }343  return await B.filter(result, async (x) => {344    const {stdout} = await exec('ps', ['-p', x, '-o', 'command']);345    return await filteringFunc(stdout);346  });347}348export { detectUdid, getAndCheckXcodeVersion, getAndCheckIosSdkVersion,349         adjustWDAAttachmentsPermissions, checkAppPresent, getDriverInfo,350         clearSystemFiles, translateDeviceName, normalizeCommandTimeouts,351         DEFAULT_TIMEOUT_KEY, resetXCTestProcesses, getPidUsingPattern,352         markSystemFilesForCleanup, printUser, printLibimobiledeviceInfo,...

...42      mocks.iosUtils.expects('clearLogs')43        .once()44        .withExactArgs([`${DERIVED_DATA_ROOT}/Logs`])45        .returns();46      await markSystemFilesForCleanup(wda);47      await markSystemFilesForCleanup(wda);48      await clearSystemFiles(wda);49      await clearSystemFiles(wda);50    });51    it('should do nothing if no derived data path is found', async function () {52      let wda = {53        retrieveDerivedDataPath () {54          return null;55        }56      };57      mocks.iosUtils.expects('clearLogs')58        .never();59      await clearSystemFiles(wda);60    });61  }));...