Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use ping method in lisa

Best Python code snippet using lisa_python

TelemetryStorage.jsm

Source:TelemetryStorage.jsm

1/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */2/* This Source Code Form is subject to the terms of the Mozilla Public3 * License, v. 2.0. If a copy of the MPL was not distributed with this4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */5"use strict";6this.EXPORTED_SYMBOLS = ["TelemetryStorage"];7const Cc = Components.classes;8const Ci = Components.interfaces;9const Cr = Components.results;10const Cu = Components.utils;11Cu.import("resource://gre/modules/AppConstants.jsm", this);12Cu.import("resource://gre/modules/Log.jsm");13Cu.import("resource://gre/modules/Services.jsm", this);14Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);15Cu.import("resource://gre/modules/osfile.jsm", this);16Cu.import("resource://gre/modules/Task.jsm", this);17Cu.import("resource://gre/modules/TelemetryUtils.jsm", this);18Cu.import("resource://gre/modules/Promise.jsm", this);19Cu.import("resource://gre/modules/Preferences.jsm", this);20const LOGGER_NAME = "Toolkit.Telemetry";21const LOGGER_PREFIX = "TelemetryStorage::";22const Telemetry = Services.telemetry;23const Utils = TelemetryUtils;24// Compute the path of the pings archive on the first use.25const DATAREPORTING_DIR = "datareporting";26const PINGS_ARCHIVE_DIR = "archived";27const ABORTED_SESSION_FILE_NAME = "aborted-session-ping";28const DELETION_PING_FILE_NAME = "pending-deletion-ping";29const SESSION_STATE_FILE_NAME = "session-state.json";30XPCOMUtils.defineLazyGetter(this, "gDataReportingDir", function() {31  return OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR);32});33XPCOMUtils.defineLazyGetter(this, "gPingsArchivePath", function() {34  return OS.Path.join(gDataReportingDir, PINGS_ARCHIVE_DIR);35});36XPCOMUtils.defineLazyGetter(this, "gAbortedSessionFilePath", function() {37  return OS.Path.join(gDataReportingDir, ABORTED_SESSION_FILE_NAME);38});39XPCOMUtils.defineLazyGetter(this, "gDeletionPingFilePath", function() {40  return OS.Path.join(gDataReportingDir, DELETION_PING_FILE_NAME);41});42XPCOMUtils.defineLazyModuleGetter(this, "CommonUtils",43                                  "resource://services-common/utils.js");44// Maxmimum time, in milliseconds, archive pings should be retained.45const MAX_ARCHIVED_PINGS_RETENTION_MS = 60 * 24 * 60 * 60 * 1000;  // 60 days46// Maximum space the archive can take on disk (in Bytes).47const ARCHIVE_QUOTA_BYTES = 120 * 1024 * 1024; // 120 MB48// Maximum space the outgoing pings can take on disk, for Desktop (in Bytes).49const PENDING_PINGS_QUOTA_BYTES_DESKTOP = 15 * 1024 * 1024; // 15 MB50// Maximum space the outgoing pings can take on disk, for Mobile (in Bytes).51const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB52// The maximum size a pending/archived ping can take on disk.53const PING_FILE_MAXIMUM_SIZE_BYTES = 1024 * 1024; // 1 MB54// This special value is submitted when the archive is outside of the quota.55const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300;56// This special value is submitted when the pending pings is outside of the quota, as57// we don't know the size of the pings above the quota.58const PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE = 17;59const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;60/**61 * This is thrown by |TelemetryStorage.loadPingFile| when reading the ping62 * from the disk fails.63 */64function PingReadError(message="Error reading the ping file", becauseNoSuchFile = false) {65  Error.call(this, message);66  let error = new Error();67  this.name = "PingReadError";68  this.message = message;69  this.stack = error.stack;70  this.becauseNoSuchFile = becauseNoSuchFile;71}72PingReadError.prototype = Object.create(Error.prototype);73PingReadError.prototype.constructor = PingReadError;74/**75 * This is thrown by |TelemetryStorage.loadPingFile| when parsing the ping JSON76 * content fails.77 */78function PingParseError(message="Error parsing ping content") {79  Error.call(this, message);80  let error = new Error();81  this.name = "PingParseError";82  this.message = message;83  this.stack = error.stack;84}85PingParseError.prototype = Object.create(Error.prototype);86PingParseError.prototype.constructor = PingParseError;87/**88 * This is a policy object used to override behavior for testing.89 */90var Policy = {91  now: () => new Date(),92  getArchiveQuota: () => ARCHIVE_QUOTA_BYTES,93  getPendingPingsQuota: () => (AppConstants.platform in ["android", "gonk"])94                                ? PENDING_PINGS_QUOTA_BYTES_MOBILE95                                : PENDING_PINGS_QUOTA_BYTES_DESKTOP,96};97/**98 * Wait for all promises in iterable to resolve or reject. This function99 * always resolves its promise with undefined, and never rejects.100 */101function waitForAll(it) {102  let dummy = () => {};103  let promises = Array.from(it, p => p.catch(dummy));104  return Promise.all(promises);105}106/**107 * Permanently intern the given string. This is mainly used for the ping.type108 * strings that can be excessively duplicated in the _archivedPings map. Do not109 * pass large or temporary strings to this function.110 */111function internString(str) {112  return Symbol.keyFor(Symbol.for(str));113}114this.TelemetryStorage = {115  get pingDirectoryPath() {116    return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings");117  },118  /**119   * The maximum size a ping can have, in bytes.120   */121  get MAXIMUM_PING_SIZE() {122    return PING_FILE_MAXIMUM_SIZE_BYTES;123  },124  /**125   * Shutdown & block on any outstanding async activity in this module.126   *127   * @return {Promise} Promise that is resolved when shutdown is complete.128   */129  shutdown: function() {130    return TelemetryStorageImpl.shutdown();131  },132  /**133   * Save an archived ping to disk.134   *135   * @param {object} ping The ping data to archive.136   * @return {promise} Promise that is resolved when the ping is successfully archived.137   */138  saveArchivedPing: function(ping) {139    return TelemetryStorageImpl.saveArchivedPing(ping);140  },141  /**142   * Load an archived ping from disk.143   *144   * @param {string} id The pings id.145   * @return {promise<object>} Promise that is resolved with the ping data.146   */147  loadArchivedPing: function(id) {148    return TelemetryStorageImpl.loadArchivedPing(id);149  },150  /**151   * Get a list of info on the archived pings.152   * This will scan the archive directory and grab basic data about the existing153   * pings out of their filename.154   *155   * @return {promise<sequence<object>>}156   */157  loadArchivedPingList: function() {158    return TelemetryStorageImpl.loadArchivedPingList();159  },160  /**161   * Clean the pings archive by removing old pings.162   * This will scan the archive directory.163   *164   * @return {Promise} Resolved when the cleanup task completes.165   */166  runCleanPingArchiveTask: function() {167    return TelemetryStorageImpl.runCleanPingArchiveTask();168  },169  /**170   * Run the task to enforce the pending pings quota.171   *172   * @return {Promise} Resolved when the cleanup task completes.173   */174  runEnforcePendingPingsQuotaTask: function() {175    return TelemetryStorageImpl.runEnforcePendingPingsQuotaTask();176  },177  /**178   * Run the task to remove all the pending pings (except the deletion ping).179   *180   * @return {Promise} Resolved when the pings are removed.181   */182  runRemovePendingPingsTask: function() {183    return TelemetryStorageImpl.runRemovePendingPingsTask();184  },185  /**186   * Reset the storage state in tests.187   */188  reset: function() {189    return TelemetryStorageImpl.reset();190  },191  /**192   * Test method that allows waiting on the archive clean task to finish.193   */194  testCleanupTaskPromise: function() {195    return (TelemetryStorageImpl._cleanArchiveTask || Promise.resolve());196  },197  /**198   * Test method that allows waiting on the pending pings quota task to finish.199   */200  testPendingQuotaTaskPromise: function() {201    return (TelemetryStorageImpl._enforcePendingPingsQuotaTask || Promise.resolve());202  },203  /**204   * Save a pending - outgoing - ping to disk and track it.205   *206   * @param {Object} ping The ping data.207   * @return {Promise} Resolved when the ping was saved.208   */209  savePendingPing: function(ping) {210    return TelemetryStorageImpl.savePendingPing(ping);211  },212  /**213   * Saves session data to disk.214   * @param {Object}  sessionData The session data.215   * @return {Promise} Resolved when the data was saved.216   */217  saveSessionData: function(sessionData) {218    return TelemetryStorageImpl.saveSessionData(sessionData);219  },220  /**221   * Loads session data from a session data file.222   * @return {Promise<object>} Resolved with the session data in object form.223   */224  loadSessionData: function() {225    return TelemetryStorageImpl.loadSessionData();226  },227  /**228   * Load a pending ping from disk by id.229   *230   * @param {String} id The pings id.231   * @return {Promise} Resolved with the loaded ping data.232   */233  loadPendingPing: function(id) {234    return TelemetryStorageImpl.loadPendingPing(id);235  },236  /**237   * Remove a pending ping from disk by id.238   *239   * @param {String} id The pings id.240   * @return {Promise} Resolved when the ping was removed.241   */242  removePendingPing: function(id) {243    return TelemetryStorageImpl.removePendingPing(id);244  },245  /**246   * Returns a list of the currently pending pings in the format:247   * {248   *   id: <string>, // The pings UUID.249   *   lastModificationDate: <number>, // Timestamp of the pings last modification.250   * }251   * This populates the list by scanning the disk.252   *253   * @return {Promise<sequence>} Resolved with the ping list.254   */255  loadPendingPingList: function() {256    return TelemetryStorageImpl.loadPendingPingList();257   },258  /**259   * Returns a list of the currently pending pings in the format:260   * {261   *   id: <string>, // The pings UUID.262   *   lastModificationDate: <number>, // Timestamp of the pings last modification.263   * }264   * This does not scan pending pings on disk.265   *266   * @return {sequence} The current pending ping list.267   */268  getPendingPingList: function() {269    return TelemetryStorageImpl.getPendingPingList();270   },271  /**272   * Save an aborted-session ping to disk. This goes to a special location so273   * it is not picked up as a pending ping.274   *275   * @param {object} ping The ping data to save.276   * @return {promise} Promise that is resolved when the ping is successfully saved.277   */278  saveAbortedSessionPing: function(ping) {279    return TelemetryStorageImpl.saveAbortedSessionPing(ping);280  },281  /**282   * Load the aborted-session ping from disk if present.283   *284   * @return {promise<object>} Promise that is resolved with the ping data if found.285   *                           Otherwise returns null.286   */287  loadAbortedSessionPing: function() {288    return TelemetryStorageImpl.loadAbortedSessionPing();289  },290  /**291   * Save the deletion ping.292   * @param ping The deletion ping.293   * @return {Promise} A promise resolved when the ping is saved.294   */295  saveDeletionPing: function(ping) {296    return TelemetryStorageImpl.saveDeletionPing(ping);297  },298  /**299   * Remove the deletion ping.300   * @return {Promise} Resolved when the ping is deleted from the disk.301   */302  removeDeletionPing: function() {303    return TelemetryStorageImpl.removeDeletionPing();304  },305  /**306   * Check if the ping id identifies a deletion ping.307   */308  isDeletionPing: function(aPingId) {309    return TelemetryStorageImpl.isDeletionPing(aPingId);310  },311  /**312   * Remove the aborted-session ping if present.313   *314   * @return {promise} Promise that is resolved once the ping is removed.315   */316  removeAbortedSessionPing: function() {317    return TelemetryStorageImpl.removeAbortedSessionPing();318  },319  /**320   * Save a single ping to a file.321   *322   * @param {object} ping The content of the ping to save.323   * @param {string} file The destination file.324   * @param {bool} overwrite If |true|, the file will be overwritten if it exists,325   * if |false| the file will not be overwritten and no error will be reported if326   * the file exists.327   * @returns {promise}328   */329  savePingToFile: function(ping, file, overwrite) {330    return TelemetryStorageImpl.savePingToFile(ping, file, overwrite);331  },332  /**333   * Save a ping to its file.334   *335   * @param {object} ping The content of the ping to save.336   * @param {bool} overwrite If |true|, the file will be overwritten337   * if it exists.338   * @returns {promise}339   */340  savePing: function(ping, overwrite) {341    return TelemetryStorageImpl.savePing(ping, overwrite);342  },343  /**344   * Add a ping to the saved pings directory so that it gets saved345   * and sent along with other pings.346   *347   * @param {Object} pingData The ping object.348   * @return {Promise} A promise resolved when the ping is saved to the pings directory.349   */350  addPendingPing: function(pingData) {351    return TelemetryStorageImpl.addPendingPing(pingData);352  },353  /**354   * Remove the file for a ping355   *356   * @param {object} ping The ping.357   * @returns {promise}358   */359  cleanupPingFile: function(ping) {360    return TelemetryStorageImpl.cleanupPingFile(ping);361  },362  /**363   * The number of pending pings on disk.364   */365  get pendingPingCount() {366    return TelemetryStorageImpl.pendingPingCount;367  },368  /**369   * Loads a ping file.370   * @param {String} aFilePath The path of the ping file.371   * @return {Promise<Object>} A promise resolved with the ping content or rejected if the372   *                           ping contains invalid data.373   */374  loadPingFile: Task.async(function* (aFilePath) {375    return TelemetryStorageImpl.loadPingFile(aFilePath);376  }),377  /**378   * Remove FHR database files. This is temporary and will be dropped in379   * the future.380   * @return {Promise} Resolved when the database files are deleted.381   */382  removeFHRDatabase: function() {383    return TelemetryStorageImpl.removeFHRDatabase();384  },385  /**386   * Only used in tests, builds an archived ping path from the ping metadata.387   * @param {String} aPingId The ping id.388   * @param {Object} aDate The ping creation date.389   * @param {String} aType The ping type.390   * @return {String} The full path to the archived ping.391   */392  _testGetArchivedPingPath: function(aPingId, aDate, aType) {393    return getArchivedPingPath(aPingId, aDate, aType);394  },395  /**396   * Only used in tests, this helper extracts ping metadata from a given filename.397   *398   * @param fileName {String} The filename.399   * @return {Object} Null if the filename didn't match the expected form.400   *                  Otherwise an object with the extracted data in the form:401   *                  { timestamp: <number>,402   *                    id: <string>,403   *                    type: <string> }404   */405  _testGetArchivedPingDataFromFileName: function(aFileName) {406    return TelemetryStorageImpl._getArchivedPingDataFromFileName(aFileName);407  },408  /**409   * Only used in tests, this helper allows cleaning up the pending ping storage.410   */411  testClearPendingPings: function() {412    return TelemetryStorageImpl.runRemovePendingPingsTask();413  }414};415/**416 * This object allows the serialisation of asynchronous tasks. This is particularly417 * useful to serialise write access to the disk in order to prevent race conditions418 * to corrupt the data being written.419 * We are using this to synchronize saving to the file that TelemetrySession persists420 * its state in.421 */422function SaveSerializer() {423  this._queuedOperations = [];424  this._queuedInProgress = false;425  this._log = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);426}427SaveSerializer.prototype = {428  /**429   * Enqueues an operation to a list to serialise their execution in order to prevent race430   * conditions. Useful to serialise access to disk.431   *432   * @param {Function} aFunction The task function to enqueue. It must return a promise.433   * @return {Promise} A promise resolved when the enqueued task completes.434   */435  enqueueTask: function (aFunction) {436    let promise = new Promise((resolve, reject) =>437      this._queuedOperations.push([aFunction, resolve, reject]));438    if (this._queuedOperations.length == 1) {439      this._popAndPerformQueuedOperation();440    }441    return promise;442  },443  /**444   * Make sure to flush all the pending operations.445   * @return {Promise} A promise resolved when all the pending operations have completed.446   */447  flushTasks: function () {448    let dummyTask = () => new Promise(resolve => resolve());449    return this.enqueueTask(dummyTask);450  },451  /**452   * Pop a task from the queue, executes it and continue to the next one.453   * This function recursively pops all the tasks.454   */455  _popAndPerformQueuedOperation: function () {456    if (!this._queuedOperations.length || this._queuedInProgress) {457      return;458    }459    this._log.trace("_popAndPerformQueuedOperation - Performing queued operation.");460    let [func, resolve, reject] = this._queuedOperations.shift();461    let promise;462    try {463      this._queuedInProgress = true;464      promise = func();465    } catch (ex) {466      this._log.warn("_popAndPerformQueuedOperation - Queued operation threw during execution. ",467                     ex);468      this._queuedInProgress = false;469      reject(ex);470      this._popAndPerformQueuedOperation();471      return;472    }473    if (!promise || typeof(promise.then) != "function") {474      let msg = "Queued operation did not return a promise: " + func;475      this._log.warn("_popAndPerformQueuedOperation - " + msg);476      this._queuedInProgress = false;477      reject(new Error(msg));478      this._popAndPerformQueuedOperation();479      return;480    }481    promise.then(result => {482        this._queuedInProgress = false;483        resolve(result);484        this._popAndPerformQueuedOperation();485      },486      error => {487        this._log.warn("_popAndPerformQueuedOperation - Failure when performing queued operation.",488                       error);489        this._queuedInProgress = false;490        reject(error);491        this._popAndPerformQueuedOperation();492      });493  },494};495var TelemetryStorageImpl = {496  _logger: null,497  // Used to serialize aborted session ping writes to disk.498  _abortedSessionSerializer: new SaveSerializer(),499  // Used to serialize deletion ping writes to disk.500  _deletionPingSerializer: new SaveSerializer(),501  // Used to serialize session state writes to disk.502  _stateSaveSerializer: new SaveSerializer(),503  // Tracks the archived pings in a Map of (id -> {timestampCreated, type}).504  // We use this to cache info on archived pings to avoid scanning the disk more than once.505  _archivedPings: new Map(),506  // A set of promises for pings currently being archived507  _activelyArchiving: new Set(),508  // Track the archive loading task to prevent multiple tasks from being executed.509  _scanArchiveTask: null,510  // Track the archive cleanup task.511  _cleanArchiveTask: null,512  // Whether we already scanned the archived pings on disk.513  _scannedArchiveDirectory: false,514  // Track the pending ping removal task.515  _removePendingPingsTask: null,516  // This tracks all the pending async ping save activity.517  _activePendingPingSaves: new Set(),518  // Tracks the pending pings in a Map of (id -> {timestampCreated, type}).519  // We use this to cache info on pending pings to avoid scanning the disk more than once.520  _pendingPings: new Map(),521  // Track the pending pings enforce quota task.522  _enforcePendingPingsQuotaTask: null,523  // Track the shutdown process to bail out of the clean up task quickly.524  _shutdown: false,525  get _log() {526    if (!this._logger) {527      this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);528    }529    return this._logger;530  },531  /**532   * Shutdown & block on any outstanding async activity in this module.533   *534   * @return {Promise} Promise that is resolved when shutdown is complete.535   */536  shutdown: Task.async(function*() {537    this._shutdown = true;538    // If the following tasks are still running, block on them. They will bail out as soon539    // as possible.540    yield this._abortedSessionSerializer.flushTasks().catch(ex => {541      this._log.error("shutdown - failed to flush aborted-session writes", ex);542    });543    yield this._deletionPingSerializer.flushTasks().catch(ex => {544      this._log.error("shutdown - failed to flush deletion ping writes", ex);545    });546    if (this._cleanArchiveTask) {547      yield this._cleanArchiveTask.catch(ex => {548        this._log.error("shutdown - the archive cleaning task failed", ex);549      });550    }551    if (this._enforcePendingPingsQuotaTask) {552      yield this._enforcePendingPingsQuotaTask.catch(ex => {553        this._log.error("shutdown - the pending pings quota task failed", ex);554      });555    }556    if (this._removePendingPingsTask) {557      yield this._removePendingPingsTask.catch(ex => {558        this._log.error("shutdown - the pending pings removal task failed", ex);559      });560    }561    // Wait on pending pings still being saved. While OS.File should have shutdown562    // blockers in place, we a) have seen weird errors being reported that might563    // indicate a bad shutdown path and b) might have completion handlers hanging564    // off the save operations that don't expect to be late in shutdown.565    yield this.promisePendingPingSaves();566  }),567  /**568   * Save an archived ping to disk.569   *570   * @param {object} ping The ping data to archive.571   * @return {promise} Promise that is resolved when the ping is successfully archived.572   */573  saveArchivedPing: function(ping) {574    let promise = this._saveArchivedPingTask(ping);575    this._activelyArchiving.add(promise);576    promise.then((r) => { this._activelyArchiving.delete(promise); },577                 (e) => { this._activelyArchiving.delete(promise); });578    return promise;579  },580  _saveArchivedPingTask: Task.async(function*(ping) {581    const creationDate = new Date(ping.creationDate);582    if (this._archivedPings.has(ping.id)) {583      const data = this._archivedPings.get(ping.id);584      if (data.timestampCreated > creationDate.getTime()) {585        this._log.error("saveArchivedPing - trying to overwrite newer ping with the same id");586        return Promise.reject(new Error("trying to overwrite newer ping with the same id"));587      }588      this._log.warn("saveArchivedPing - overwriting older ping with the same id");589    }590    // Get the archived ping path and append the lz4 suffix to it (so we have 'jsonlz4').591    const filePath = getArchivedPingPath(ping.id, creationDate, ping.type) + "lz4";592    yield OS.File.makeDir(OS.Path.dirname(filePath), { ignoreExisting: true,593                                                       from: OS.Constants.Path.profileDir });594    yield this.savePingToFile(ping, filePath, /* overwrite*/ true, /* compressed*/ true);595    this._archivedPings.set(ping.id, {596      timestampCreated: creationDate.getTime(),597      type: internString(ping.type),598    });599    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SESSION_PING_COUNT").add();600    return undefined;601  }),602  /**603   * Load an archived ping from disk.604   *605   * @param {string} id The pings id.606   * @return {promise<object>} Promise that is resolved with the ping data.607   */608  loadArchivedPing: Task.async(function*(id) {609    this._log.trace("loadArchivedPing - id: " + id);610    const data = this._archivedPings.get(id);611    if (!data) {612      this._log.trace("loadArchivedPing - no ping with id: " + id);613      return Promise.reject(new Error("TelemetryStorage.loadArchivedPing - no ping with id " + id));614    }615    const path = getArchivedPingPath(id, new Date(data.timestampCreated), data.type);616    const pathCompressed = path + "lz4";617    // Purge pings which are too big.618    let checkSize = function*(path) {619      const fileSize = (yield OS.File.stat(path)).size;620      if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {621        Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB")622                 .add(Math.floor(fileSize / 1024 / 1024));623        Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add();624        yield OS.File.remove(path, {ignoreAbsent: true});625        throw new Error("loadArchivedPing - exceeded the maximum ping size: " + fileSize);626      }627    };628    try {629      // Try to load a compressed version of the archived ping first.630      this._log.trace("loadArchivedPing - loading ping from: " + pathCompressed);631      yield* checkSize(pathCompressed);632      return yield this.loadPingFile(pathCompressed, /* compressed*/ true);633    } catch (ex) {634      if (!ex.becauseNoSuchFile) {635        throw ex;636      }637      // If that fails, look for the uncompressed version.638      this._log.trace("loadArchivedPing - compressed ping not found, loading: " + path);639      yield* checkSize(path);640      return yield this.loadPingFile(path, /* compressed*/ false);641    }642  }),643  /**644   * Saves session data to disk.645   */646  saveSessionData: function(sessionData) {647    return this._stateSaveSerializer.enqueueTask(() => this._saveSessionData(sessionData));648  },649  _saveSessionData: Task.async(function* (sessionData) {650    let dataDir = OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR);651    yield OS.File.makeDir(dataDir);652    let filePath = OS.Path.join(gDataReportingDir, SESSION_STATE_FILE_NAME);653    try {654      yield CommonUtils.writeJSON(sessionData, filePath);655    } catch (e) {656      this._log.error("_saveSessionData - Failed to write session data to " + filePath, e);657      Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_SAVE").add(1);658    }659  }),660  /**661   * Loads session data from the session data file.662   * @return {Promise<Object>} A promise resolved with an object on success,663   *                           with null otherwise.664   */665  loadSessionData: function() {666    return this._stateSaveSerializer.enqueueTask(() => this._loadSessionData());667  },668  _loadSessionData: Task.async(function* () {669    const dataFile = OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR,670                                  SESSION_STATE_FILE_NAME);671    let content;672    try {673      content = yield OS.File.read(dataFile, { encoding: "utf-8" });674    } catch (ex) {675      this._log.info("_loadSessionData - can not load session data file", ex);676      Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_LOAD").add(1);677      return null;678    }679    let data;680    try {681      data = JSON.parse(content);682    } catch (ex) {683      this._log.error("_loadSessionData - failed to parse session data", ex);684      Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_PARSE").add(1);685      return null;686    }687    return data;688  }),689  /**690   * Remove an archived ping from disk.691   *692   * @param {string} id The pings id.693   * @param {number} timestampCreated The pings creation timestamp.694   * @param {string} type The pings type.695   * @return {promise<object>} Promise that is resolved when the pings is removed.696   */697  _removeArchivedPing: Task.async(function*(id, timestampCreated, type) {698    this._log.trace("_removeArchivedPing - id: " + id + ", timestampCreated: " + timestampCreated + ", type: " + type);699    const path = getArchivedPingPath(id, new Date(timestampCreated), type);700    const pathCompressed = path + "lz4";701    this._log.trace("_removeArchivedPing - removing ping from: " + path);702    yield OS.File.remove(path, {ignoreAbsent: true});703    yield OS.File.remove(pathCompressed, {ignoreAbsent: true});704    // Remove the ping from the cache.705    this._archivedPings.delete(id);706  }),707  /**708   * Clean the pings archive by removing old pings.709   *710   * @return {Promise} Resolved when the cleanup task completes.711   */712  runCleanPingArchiveTask: function() {713    // If there's an archive cleaning task already running, return it.714    if (this._cleanArchiveTask) {715      return this._cleanArchiveTask;716    }717    // Make sure to clear |_cleanArchiveTask| once done.718    let clear = () => this._cleanArchiveTask = null;719    // Since there's no archive cleaning task running, start it.720    this._cleanArchiveTask = this._cleanArchive().then(clear, clear);721    return this._cleanArchiveTask;722  },723  /**724   * Removes pings which are too old from the pings archive.725   * @return {Promise} Resolved when the ping age check is complete.726   */727  _purgeOldPings: Task.async(function*() {728    this._log.trace("_purgeOldPings");729    const nowDate = Policy.now();730    const startTimeStamp = nowDate.getTime();731    let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);732    let subdirs = (yield dirIterator.nextBatch()).filter(e => e.isDir);733    dirIterator.close();734    // Keep track of the newest removed month to update the cache, if needed.735    let newestRemovedMonthTimestamp = null;736    let evictedDirsCount = 0;737    let maxDirAgeInMonths = 0;738    // Walk through the monthly subdirs of the form <YYYY-MM>/739    for (let dir of subdirs) {740      if (this._shutdown) {741        this._log.trace("_purgeOldPings - Terminating the clean up task due to shutdown");742        return;743      }744      if (!isValidArchiveDir(dir.name)) {745        this._log.warn("_purgeOldPings - skipping invalidly named subdirectory " + dir.path);746        continue;747      }748      const archiveDate = getDateFromArchiveDir(dir.name);749      if (!archiveDate) {750        this._log.warn("_purgeOldPings - skipping invalid subdirectory date " + dir.path);751        continue;752      }753      // If this archive directory is older than 180 days, remove it.754      if ((startTimeStamp - archiveDate.getTime()) > MAX_ARCHIVED_PINGS_RETENTION_MS) {755        try {756          yield OS.File.removeDir(dir.path);757          evictedDirsCount++;758          // Update the newest removed month.759          newestRemovedMonthTimestamp = Math.max(archiveDate, newestRemovedMonthTimestamp);760        } catch (ex) {761          this._log.error("_purgeOldPings - Unable to remove " + dir.path, ex);762        }763      } else {764        // We're not removing this directory, so record the age for the oldest directory.765        const dirAgeInMonths = Utils.getElapsedTimeInMonths(archiveDate, nowDate);766        maxDirAgeInMonths = Math.max(dirAgeInMonths, maxDirAgeInMonths);767      }768    }769    // Trigger scanning of the archived pings.770    yield this.loadArchivedPingList();771    // Refresh the cache: we could still skip this, but it's cheap enough to keep it772    // to avoid introducing task dependencies.773    if (newestRemovedMonthTimestamp) {774      // Scan the archive cache for pings older than the newest directory pruned above.775      for (let [id, info] of this._archivedPings) {776        const timestampCreated = new Date(info.timestampCreated);777        if (timestampCreated.getTime() > newestRemovedMonthTimestamp) {778          continue;779        }780        // Remove outdated pings from the cache.781        this._archivedPings.delete(id);782      }783    }784    const endTimeStamp = Policy.now().getTime();785    // Save the time it takes to evict old directories and the eviction count.786    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS")787             .add(evictedDirsCount);788    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_DIRS_MS")789             .add(Math.ceil(endTimeStamp - startTimeStamp));790    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE")791             .add(maxDirAgeInMonths);792  }),793  /**794   * Enforce a disk quota for the pings archive.795   * @return {Promise} Resolved when the quota check is complete.796   */797  _enforceArchiveQuota: Task.async(function*() {798    this._log.trace("_enforceArchiveQuota");799    let startTimeStamp = Policy.now().getTime();800    // Build an ordered list, from newer to older, of archived pings.801    let pingList = Array.from(this._archivedPings, p => ({802      id: p[0],803      timestampCreated: p[1].timestampCreated,804      type: p[1].type,805    }));806    pingList.sort((a, b) => b.timestampCreated - a.timestampCreated);807    // If our archive is too big, we should reduce it to reach 90% of the quota.808    const SAFE_QUOTA = Policy.getArchiveQuota() * 0.9;809    // The index of the last ping to keep. Pings older than this one will be deleted if810    // the archive exceeds the quota.811    let lastPingIndexToKeep = null;812    let archiveSizeInBytes = 0;813    // Find the disk size of the archive.814    for (let i = 0; i < pingList.length; i++) {815      if (this._shutdown) {816        this._log.trace("_enforceArchiveQuota - Terminating the clean up task due to shutdown");817        return;818      }819      let ping = pingList[i];820      // Get the size for this ping.821      const fileSize =822        yield getArchivedPingSize(ping.id, new Date(ping.timestampCreated), ping.type);823      if (!fileSize) {824        this._log.warn("_enforceArchiveQuota - Unable to find the size of ping " + ping.id);825        continue;826      }827      // Enforce a maximum file size limit on archived pings.828      if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {829        this._log.error("_enforceArchiveQuota - removing file exceeding size limit, size: " + fileSize);830        // We just remove the ping from the disk, we don't bother removing it from pingList831        // since it won't contribute to the quota.832        yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type)833                  .catch(e => this._log.error("_enforceArchiveQuota - failed to remove archived ping" + ping.id));834        Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB")835                 .add(Math.floor(fileSize / 1024 / 1024));836        Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add();837        continue;838      }839      archiveSizeInBytes += fileSize;840      if (archiveSizeInBytes < SAFE_QUOTA) {841        // We save the index of the last ping which is ok to keep in order to speed up ping842        // pruning.843        lastPingIndexToKeep = i;844      } else if (archiveSizeInBytes > Policy.getArchiveQuota()) {845        // Ouch, our ping archive is too big. Bail out and start pruning!846        break;847      }848    }849    // Save the time it takes to check if the archive is over-quota.850    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_CHECKING_OVER_QUOTA_MS")851             .add(Math.round(Policy.now().getTime() - startTimeStamp));852    let submitProbes = (sizeInMB, evictedPings, elapsedMs) => {853      Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").add(sizeInMB);854      Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").add(evictedPings);855      Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS").add(elapsedMs);856    };857    // Check if we're using too much space. If not, submit the archive size and bail out.858    if (archiveSizeInBytes < Policy.getArchiveQuota()) {859      submitProbes(Math.round(archiveSizeInBytes / 1024 / 1024), 0, 0);860      return;861    }862    this._log.info("_enforceArchiveQuota - archive size: " + archiveSizeInBytes + "bytes"863                   + ", safety quota: " + SAFE_QUOTA + "bytes");864    startTimeStamp = Policy.now().getTime();865    let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1);866    // Remove all the pings older than the last one which we are safe to keep.867    for (let ping of pingsToPurge) {868      if (this._shutdown) {869        this._log.trace("_enforceArchiveQuota - Terminating the clean up task due to shutdown");870        return;871      }872      // This list is guaranteed to be in order, so remove the pings at its873      // beginning (oldest).874      yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type);875    }876    const endTimeStamp = Policy.now().getTime();877    submitProbes(ARCHIVE_SIZE_PROBE_SPECIAL_VALUE, pingsToPurge.length,878                 Math.ceil(endTimeStamp - startTimeStamp));879  }),880  _cleanArchive: Task.async(function*() {881    this._log.trace("cleanArchiveTask");882    if (!(yield OS.File.exists(gPingsArchivePath))) {883      return;884    }885    // Remove pings older than 180 days.886    try {887      yield this._purgeOldPings();888    } catch (ex) {889      this._log.error("_cleanArchive - There was an error removing old directories", ex);890    }891    // Make sure we respect the archive disk quota.892    yield this._enforceArchiveQuota();893  }),894  /**895   * Run the task to enforce the pending pings quota.896   *897   * @return {Promise} Resolved when the cleanup task completes.898   */899  runEnforcePendingPingsQuotaTask: Task.async(function*() {900    // If there's a cleaning task already running, return it.901    if (this._enforcePendingPingsQuotaTask) {902      return this._enforcePendingPingsQuotaTask;903    }904    // Since there's no quota enforcing task running, start it.905    try {906      this._enforcePendingPingsQuotaTask = this._enforcePendingPingsQuota();907      yield this._enforcePendingPingsQuotaTask;908    } finally {909      this._enforcePendingPingsQuotaTask = null;910    }911    return undefined;912  }),913  /**914   * Enforce a disk quota for the pending pings.915   * @return {Promise} Resolved when the quota check is complete.916   */917  _enforcePendingPingsQuota: Task.async(function*() {918    this._log.trace("_enforcePendingPingsQuota");919    let startTimeStamp = Policy.now().getTime();920    // Build an ordered list, from newer to older, of pending pings.921    let pingList = Array.from(this._pendingPings, p => ({922      id: p[0],923      lastModificationDate: p[1].lastModificationDate,924    }));925    pingList.sort((a, b) => b.lastModificationDate - a.lastModificationDate);926    // If our pending pings directory is too big, we should reduce it to reach 90% of the quota.927    const SAFE_QUOTA = Policy.getPendingPingsQuota() * 0.9;928    // The index of the last ping to keep. Pings older than this one will be deleted if929    // the pending pings directory size exceeds the quota.930    let lastPingIndexToKeep = null;931    let pendingPingsSizeInBytes = 0;932    // Find the disk size of the pending pings directory.933    for (let i = 0; i < pingList.length; i++) {934      if (this._shutdown) {935        this._log.trace("_enforcePendingPingsQuota - Terminating the clean up task due to shutdown");936        return;937      }938      let ping = pingList[i];939      // Get the size for this ping.940      const fileSize = yield getPendingPingSize(ping.id);941      if (!fileSize) {942        this._log.warn("_enforcePendingPingsQuota - Unable to find the size of ping " + ping.id);943        continue;944      }945      pendingPingsSizeInBytes += fileSize;946      if (pendingPingsSizeInBytes < SAFE_QUOTA) {947        // We save the index of the last ping which is ok to keep in order to speed up ping948        // pruning.949        lastPingIndexToKeep = i;950      } else if (pendingPingsSizeInBytes > Policy.getPendingPingsQuota()) {951        // Ouch, our pending pings directory size is too big. Bail out and start pruning!952        break;953      }954    }955    // Save the time it takes to check if the pending pings are over-quota.956    Telemetry.getHistogramById("TELEMETRY_PENDING_CHECKING_OVER_QUOTA_MS")957             .add(Math.round(Policy.now().getTime() - startTimeStamp));958    let recordHistograms = (sizeInMB, evictedPings, elapsedMs) => {959      Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").add(sizeInMB);960      Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").add(evictedPings);961      Telemetry.getHistogramById("TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS").add(elapsedMs);962    };963    // Check if we're using too much space. If not, bail out.964    if (pendingPingsSizeInBytes < Policy.getPendingPingsQuota()) {965      recordHistograms(Math.round(pendingPingsSizeInBytes / 1024 / 1024), 0, 0);966      return;967    }968    this._log.info("_enforcePendingPingsQuota - size: " + pendingPingsSizeInBytes + "bytes"969                   + ", safety quota: " + SAFE_QUOTA + "bytes");970    startTimeStamp = Policy.now().getTime();971    let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1);972    // Remove all the pings older than the last one which we are safe to keep.973    for (let ping of pingsToPurge) {974      if (this._shutdown) {975        this._log.trace("_enforcePendingPingsQuota - Terminating the clean up task due to shutdown");976        return;977      }978      // This list is guaranteed to be in order, so remove the pings at its979      // beginning (oldest).980      yield this.removePendingPing(ping.id);981    }982    const endTimeStamp = Policy.now().getTime();983    // We don't know the size of the pending pings directory if we are above the quota,984    // since we stop scanning once we reach the quota. We use a special value to show985    // this condition.986    recordHistograms(PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE, pingsToPurge.length,987                 Math.ceil(endTimeStamp - startTimeStamp));988  }),989  /**990   * Reset the storage state in tests.991   */992  reset: function() {993    this._shutdown = false;994    this._scannedArchiveDirectory = false;995    this._archivedPings = new Map();996    this._scannedPendingDirectory = false;997    this._pendingPings = new Map();998  },999  /**1000   * Get a list of info on the archived pings.1001   * This will scan the archive directory and grab basic data about the existing1002   * pings out of their filename.1003   *1004   * @return {promise<sequence<object>>}1005   */1006  loadArchivedPingList: Task.async(function*() {1007    // If there's an archive loading task already running, return it.1008    if (this._scanArchiveTask) {1009      return this._scanArchiveTask;1010    }1011    yield waitForAll(this._activelyArchiving);1012    if (this._scannedArchiveDirectory) {1013      this._log.trace("loadArchivedPingList - Archive already scanned, hitting cache.");1014      return this._archivedPings;1015    }1016    // Since there's no archive loading task running, start it.1017    let result;1018    try {1019      this._scanArchiveTask = this._scanArchive();1020      result = yield this._scanArchiveTask;1021    } finally {1022      this._scanArchiveTask = null;1023    }1024    return result;1025  }),1026  _scanArchive: Task.async(function*() {1027    this._log.trace("_scanArchive");1028    let submitProbes = (pingCount, dirCount) => {1029      Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT")1030               .add(pingCount);1031      Telemetry.getHistogramById("TELEMETRY_ARCHIVE_DIRECTORIES_COUNT")1032               .add(dirCount);1033    };1034    if (!(yield OS.File.exists(gPingsArchivePath))) {1035      submitProbes(0, 0);1036      return new Map();1037    }1038    let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);1039    let subdirs =1040      (yield dirIterator.nextBatch()).filter(e => e.isDir).filter(e => isValidArchiveDir(e.name));1041    dirIterator.close();1042    // Walk through the monthly subdirs of the form <YYYY-MM>/1043    for (let dir of subdirs) {1044      this._log.trace("_scanArchive - checking in subdir: " + dir.path);1045      let pingIterator = new OS.File.DirectoryIterator(dir.path);1046      let pings = (yield pingIterator.nextBatch()).filter(e => !e.isDir);1047      pingIterator.close();1048      // Now process any ping files of the form "<timestamp>.<uuid>.<type>.[json|jsonlz4]".1049      for (let p of pings) {1050        // data may be null if the filename doesn't match the above format.1051        let data = this._getArchivedPingDataFromFileName(p.name);1052        if (!data) {1053          continue;1054        }1055        // In case of conflicts, overwrite only with newer pings.1056        if (this._archivedPings.has(data.id)) {1057          const overwrite = data.timestamp > this._archivedPings.get(data.id).timestampCreated;1058          this._log.warn("_scanArchive - have seen this id before: " + data.id +1059                         ", overwrite: " + overwrite);1060          if (!overwrite) {1061            continue;1062          }1063          yield this._removeArchivedPing(data.id, data.timestampCreated, data.type)1064                    .catch((e) => this._log.warn("_scanArchive - failed to remove ping", e));1065        }1066        this._archivedPings.set(data.id, {1067          timestampCreated: data.timestamp,1068          type: internString(data.type),1069        });1070      }1071    }1072    // Mark the archive as scanned, so we no longer hit the disk.1073    this._scannedArchiveDirectory = true;1074    // Update the ping and directories count histograms.1075    submitProbes(this._archivedPings.size, subdirs.length);1076    return this._archivedPings;1077  }),1078  /**1079   * Save a single ping to a file.1080   *1081   * @param {object} ping The content of the ping to save.1082   * @param {string} file The destination file.1083   * @param {bool} overwrite If |true|, the file will be overwritten if it exists,1084   * if |false| the file will not be overwritten and no error will be reported if1085   * the file exists.1086   * @param {bool} [compress=false] If |true|, the file will use lz4 compression. Otherwise no1087   * compression will be used.1088   * @returns {promise}1089   */1090  savePingToFile: Task.async(function*(ping, filePath, overwrite, compress = false) {1091    try {1092      this._log.trace("savePingToFile - path: " + filePath);1093      let pingString = JSON.stringify(ping);1094      let options = { tmpPath: filePath + ".tmp", noOverwrite: !overwrite };1095      if (compress) {1096        options.compression = "lz4";1097      }1098      yield OS.File.writeAtomic(filePath, pingString, options);1099    } catch (e) {1100      if (!e.becauseExists) {1101        throw e;1102      }1103    }1104  }),1105  /**1106   * Save a ping to its file.1107   *1108   * @param {object} ping The content of the ping to save.1109   * @param {bool} overwrite If |true|, the file will be overwritten1110   * if it exists.1111   * @returns {promise}1112   */1113  savePing: Task.async(function*(ping, overwrite) {1114    yield getPingDirectory();1115    let file = pingFilePath(ping);1116    yield this.savePingToFile(ping, file, overwrite);1117    return file;1118  }),1119  /**1120   * Add a ping to the saved pings directory so that it gets saved1121   * and sent along with other pings.1122   * Note: that the original ping file will not be modified.1123   *1124   * @param {Object} ping The ping object.1125   * @return {Promise} A promise resolved when the ping is saved to the pings directory.1126   */1127  addPendingPing: function(ping) {1128    return this.savePendingPing(ping);1129  },1130  /**1131   * Remove the file for a ping1132   *1133   * @param {object} ping The ping.1134   * @returns {promise}1135   */1136  cleanupPingFile: function(ping) {1137    return OS.File.remove(pingFilePath(ping));1138  },1139  savePendingPing: function(ping) {1140    let p = this.savePing(ping, true).then((path) => {1141      this._pendingPings.set(ping.id, {1142        path: path,1143        lastModificationDate: Policy.now().getTime(),1144      });1145      this._log.trace("savePendingPing - saved ping with id " + ping.id);1146    });1147    this._trackPendingPingSaveTask(p);1148    return p;1149  },1150  loadPendingPing: Task.async(function*(id) {1151    this._log.trace("loadPendingPing - id: " + id);1152    let info = this._pendingPings.get(id);1153    if (!info) {1154      this._log.trace("loadPendingPing - unknown id " + id);1155      throw new Error("TelemetryStorage.loadPendingPing - no ping with id " + id);1156    }1157    // Try to get the dimension of the ping. If that fails, update the histograms.1158    let fileSize = 0;1159    try {1160      fileSize = (yield OS.File.stat(info.path)).size;1161    } catch (e) {1162      if (!(e instanceof OS.File.Error) || !e.becauseNoSuchFile) {1163        throw e;1164      }1165      // Fall through and let |loadPingFile| report the error.1166    }1167    // Purge pings which are too big.1168    if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {1169      yield this.removePendingPing(id);1170      Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB")1171               .add(Math.floor(fileSize / 1024 / 1024));1172      Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();1173      throw new Error("loadPendingPing - exceeded the maximum ping size: " + fileSize);1174    }1175    // Try to load the ping file. Update the related histograms on failure.1176    let ping;1177    try {1178      ping = yield this.loadPingFile(info.path, false);1179    } catch (e) {1180      // If we failed to load the ping, check what happened and update the histogram.1181      if (e instanceof PingReadError) {1182        Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").add();1183      } else if (e instanceof PingParseError) {1184        Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").add();1185      }1186      // Remove the ping from the cache, so we don't try to load it again.1187      this._pendingPings.delete(id);1188      // Then propagate the rejection.1189      throw e;1190    }1191    return ping;1192  }),1193  removePendingPing: function(id) {1194    let info = this._pendingPings.get(id);1195    if (!info) {1196      this._log.trace("removePendingPing - unknown id " + id);1197      return Promise.resolve();1198    }1199    this._log.trace("removePendingPing - deleting ping with id: " + id +1200                    ", path: " + info.path);1201    this._pendingPings.delete(id);1202    return OS.File.remove(info.path).catch((ex) =>1203      this._log.error("removePendingPing - failed to remove ping", ex));1204  },1205  /**1206   * Track any pending ping save tasks through the promise passed here.1207   * This is needed to block on any outstanding ping save activity.1208   *1209   * @param {Object<Promise>} The save promise to track.1210   */1211  _trackPendingPingSaveTask: function (promise) {1212    let clear = () => this._activePendingPingSaves.delete(promise);1213    promise.then(clear, clear);1214    this._activePendingPingSaves.add(promise);1215  },1216  /**1217   * Return a promise that allows to wait on pending pings being saved.1218   * @return {Object<Promise>} A promise resolved when all the pending pings save promises1219   *         are resolved.1220   */1221  promisePendingPingSaves: function () {1222    // Make sure to wait for all the promises, even if they reject. We don't need to log1223    // the failures here, as they are already logged elsewhere.1224    return waitForAll(this._activePendingPingSaves);1225  },1226  /**1227   * Run the task to remove all the pending pings (except the deletion ping).1228   *1229   * @return {Promise} Resolved when the pings are removed.1230   */1231  runRemovePendingPingsTask: Task.async(function*() {1232    // If we already have a pending pings removal task active, return that.1233    if (this._removePendingPingsTask) {1234      return this._removePendingPingsTask;1235    }1236    // Start the task to remove all pending pings. Also make sure to clear the task once done.1237    try {1238      this._removePendingPingsTask = this.removePendingPings();1239      yield this._removePendingPingsTask;1240    } finally {1241      this._removePendingPingsTask = null;1242    }1243    return undefined;1244  }),1245  removePendingPings: Task.async(function*() {1246    this._log.trace("removePendingPings - removing all pending pings");1247    // Wait on pending pings still being saved, so so we don't miss removing them.1248    yield this.promisePendingPingSaves();1249    // Individually remove existing pings, so we don't interfere with operations expecting1250    // the pending pings directory to exist.1251    const directory = TelemetryStorage.pingDirectoryPath;1252    let iter = new OS.File.DirectoryIterator(directory);1253    try {1254      if (!(yield iter.exists())) {1255        this._log.trace("removePendingPings - the pending pings directory doesn't exist");1256        return;1257      }1258      let files = (yield iter.nextBatch()).filter(e => !e.isDir);1259      for (let file of files) {1260        try {1261          yield OS.File.remove(file.path);1262        } catch (ex) {1263          this._log.error("removePendingPings - failed to remove file " + file.path, ex);1264          continue;1265        }1266      }1267    } finally {1268      yield iter.close();1269    }1270  }),1271  loadPendingPingList: function() {1272    // If we already have a pending scanning task active, return that.1273    if (this._scanPendingPingsTask) {1274      return this._scanPendingPingsTask;1275    }1276    if (this._scannedPendingDirectory) {1277      this._log.trace("loadPendingPingList - Pending already scanned, hitting cache.");1278      return Promise.resolve(this._buildPingList());1279    }1280    // Since there's no pending pings scan task running, start it.1281    // Also make sure to clear the task once done.1282    this._scanPendingPingsTask = this._scanPendingPings().then(pings => {1283      this._scanPendingPingsTask = null;1284      return pings;1285    }, ex => {1286      this._scanPendingPingsTask = null;1287      throw ex;1288    });1289    return this._scanPendingPingsTask;1290  },1291  getPendingPingList: function() {1292    return this._buildPingList();1293  },1294  _scanPendingPings: Task.async(function*() {1295    this._log.trace("_scanPendingPings");1296    let directory = TelemetryStorage.pingDirectoryPath;1297    let iter = new OS.File.DirectoryIterator(directory);1298    let exists = yield iter.exists();1299    try {1300      if (!exists) {1301        return [];1302      }1303      let files = (yield iter.nextBatch()).filter(e => !e.isDir);1304      for (let file of files) {1305        if (this._shutdown) {1306          return [];1307        }1308        let info;1309        try {1310          info = yield OS.File.stat(file.path);1311        } catch (ex) {1312          this._log.error("_scanPendingPings - failed to stat file " + file.path, ex);1313          continue;1314        }1315        // Enforce a maximum file size limit on pending pings.1316        if (info.size > PING_FILE_MAXIMUM_SIZE_BYTES) {1317          this._log.error("_scanPendingPings - removing file exceeding size limit " + file.path);1318          try {1319            yield OS.File.remove(file.path);1320          } catch (ex) {1321            this._log.error("_scanPendingPings - failed to remove file " + file.path, ex);1322          } finally {1323            Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB")1324                     .add(Math.floor(info.size / 1024 / 1024));1325            Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();1326            continue;1327          }1328        }1329        let id = OS.Path.basename(file.path);1330        if (!UUID_REGEX.test(id)) {1331          this._log.trace("_scanPendingPings - filename is not a UUID: " + id);1332          id = Utils.generateUUID();1333        }1334        this._pendingPings.set(id, {1335          path: file.path,1336          lastModificationDate: info.lastModificationDate.getTime(),1337        });1338      }1339    } finally {1340      yield iter.close();1341    }1342    // Explicitly load the deletion ping from its known path, if it's there.1343    if (yield OS.File.exists(gDeletionPingFilePath)) {1344      this._log.trace("_scanPendingPings - Adding pending deletion ping.");1345      // We can't get the ping id or the last modification date without hitting the disk.1346      // Since deletion has a special handling, we don't really need those.1347      this._pendingPings.set(Utils.generateUUID(), {1348        path: gDeletionPingFilePath,1349        lastModificationDate: Date.now(),1350      });1351    }1352    this._scannedPendingDirectory = true;1353    return this._buildPingList();1354  }),1355  _buildPingList: function() {1356    const list = Array.from(this._pendingPings, p => ({1357      id: p[0],1358      lastModificationDate: p[1].lastModificationDate,1359    }));1360    list.sort((a, b) => b.lastModificationDate - a.lastModificationDate);1361    return list;1362  },1363  get pendingPingCount() {1364    return this._pendingPings.size;1365  },1366  /**1367   * Loads a ping file.1368   * @param {String} aFilePath The path of the ping file.1369   * @param {Boolean} [aCompressed=false] If |true|, expects the file to be compressed using lz4.1370   * @return {Promise<Object>} A promise resolved with the ping content or rejected if the1371   *                           ping contains invalid data.1372   * @throws {PingReadError} There was an error while reading the ping file from the disk.1373   * @throws {PingParseError} There was an error while parsing the JSON content of the ping file.1374   */1375  loadPingFile: Task.async(function* (aFilePath, aCompressed = false) {1376    let options = {};1377    if (aCompressed) {1378      options.compression = "lz4";1379    }1380    let array;1381    try {1382      array = yield OS.File.read(aFilePath, options);1383    } catch (e) {1384      this._log.trace("loadPingfile - unreadable ping " + aFilePath, e);1385      throw new PingReadError(e.message, e.becauseNoSuchFile);1386    }1387    let decoder = new TextDecoder();1388    let string = decoder.decode(array);1389    let ping;1390    try {1391      ping = JSON.parse(string);1392    } catch (e) {1393      this._log.trace("loadPingfile - unparseable ping " + aFilePath, e);1394      yield OS.File.remove(aFilePath).catch((ex) => {1395        this._log.error("loadPingFile - failed removing unparseable ping file", ex);1396      });1397      throw new PingParseError(e.message);1398    }1399    return ping;1400  }),1401  /**1402   * Archived pings are saved with file names of the form:1403   * "<timestamp>.<uuid>.<type>.[json|jsonlz4]"1404   * This helper extracts that data from a given filename.1405   *1406   * @param fileName {String} The filename.1407   * @return {Object} Null if the filename didn't match the expected form.1408   *                  Otherwise an object with the extracted data in the form:1409   *                  { timestamp: <number>,1410   *                    id: <string>,1411   *                    type: <string> }1412   */1413  _getArchivedPingDataFromFileName: function(fileName) {1414    // Extract the parts.1415    let parts = fileName.split(".");1416    if (parts.length != 4) {1417      this._log.trace("_getArchivedPingDataFromFileName - should have 4 parts");1418      return null;1419    }1420    let [timestamp, uuid, type, extension] = parts;1421    if (extension != "json" && extension != "jsonlz4") {1422      this._log.trace("_getArchivedPingDataFromFileName - should have 'json' or 'jsonlz4' extension");1423      return null;1424    }1425    // Check for a valid timestamp.1426    timestamp = parseInt(timestamp);1427    if (Number.isNaN(timestamp)) {1428      this._log.trace("_getArchivedPingDataFromFileName - should have a valid timestamp");1429      return null;1430    }1431    // Check for a valid UUID.1432    if (!UUID_REGEX.test(uuid)) {1433      this._log.trace("_getArchivedPingDataFromFileName - should have a valid id");1434      return null;1435    }1436    // Check for a valid type string.1437    const typeRegex = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i;1438    if (!typeRegex.test(type)) {1439      this._log.trace("_getArchivedPingDataFromFileName - should have a valid type");1440      return null;1441    }1442    return {1443      timestamp: timestamp,1444      id: uuid,1445      type: type,1446    };1447  },1448  saveAbortedSessionPing: Task.async(function*(ping) {1449    this._log.trace("saveAbortedSessionPing - ping path: " + gAbortedSessionFilePath);1450    yield OS.File.makeDir(gDataReportingDir, { ignoreExisting: true });1451    return this._abortedSessionSerializer.enqueueTask(() =>1452      this.savePingToFile(ping, gAbortedSessionFilePath, true));1453  }),1454  loadAbortedSessionPing: Task.async(function*() {1455    let ping = null;1456    try {1457      ping = yield this.loadPingFile(gAbortedSessionFilePath);1458    } catch (ex) {1459      if (ex.becauseNoSuchFile) {1460        this._log.trace("loadAbortedSessionPing - no such file");1461      } else {1462        this._log.error("loadAbortedSessionPing - error loading ping", ex)1463      }1464    }1465    return ping;1466  }),1467  removeAbortedSessionPing: function() {1468    return this._abortedSessionSerializer.enqueueTask(Task.async(function*() {1469      try {1470        yield OS.File.remove(gAbortedSessionFilePath, { ignoreAbsent: false });1471        this._log.trace("removeAbortedSessionPing - success");1472      } catch (ex) {1473        if (ex.becauseNoSuchFile) {1474          this._log.trace("removeAbortedSessionPing - no such file");1475        } else {1476          this._log.error("removeAbortedSessionPing - error removing ping", ex)1477        }1478      }1479    }.bind(this)));1480  },1481  /**1482   * Save the deletion ping.1483   * @param ping The deletion ping.1484   * @return {Promise} Resolved when the ping is saved.1485   */1486  saveDeletionPing: Task.async(function*(ping) {1487    this._log.trace("saveDeletionPing - ping path: " + gDeletionPingFilePath);1488    yield OS.File.makeDir(gDataReportingDir, { ignoreExisting: true });1489    let p = this._deletionPingSerializer.enqueueTask(() =>1490      this.savePingToFile(ping, gDeletionPingFilePath, true));1491    this._trackPendingPingSaveTask(p);1492    return p;1493  }),1494  /**1495   * Remove the deletion ping.1496   * @return {Promise} Resolved when the ping is deleted from the disk.1497   */1498  removeDeletionPing: Task.async(function*() {1499    return this._deletionPingSerializer.enqueueTask(Task.async(function*() {1500      try {1501        yield OS.File.remove(gDeletionPingFilePath, { ignoreAbsent: false });1502        this._log.trace("removeDeletionPing - success");1503      } catch (ex) {1504        if (ex.becauseNoSuchFile) {1505          this._log.trace("removeDeletionPing - no such file");1506        } else {1507          this._log.error("removeDeletionPing - error removing ping", ex)1508        }1509      }1510    }.bind(this)));1511  }),1512  isDeletionPing: function(aPingId) {1513    this._log.trace("isDeletionPing - id: " + aPingId);1514    let pingInfo = this._pendingPings.get(aPingId);1515    if (!pingInfo) {1516      return false;1517    }1518    if (pingInfo.path != gDeletionPingFilePath) {1519      return false;1520    }1521    return true;1522  },1523  /**1524   * Remove FHR database files. This is temporary and will be dropped in1525   * the future.1526   * @return {Promise} Resolved when the database files are deleted.1527   */1528  removeFHRDatabase: Task.async(function*() {1529    this._log.trace("removeFHRDatabase");1530    // Let's try to remove the FHR DB with the default filename first.1531    const FHR_DB_DEFAULT_FILENAME = "healthreport.sqlite";1532    // Even if it's uncommon, there may be 2 additional files: - a "write ahead log"1533    // (-wal) file and a "shared memory file" (-shm). We need to remove them as well.1534    let FILES_TO_REMOVE = [1535      OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME),1536      OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME + "-wal"),1537      OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME + "-shm"),1538    ];1539    // FHR could have used either the default DB file name or a custom one1540    // through this preference.1541    const FHR_DB_CUSTOM_FILENAME =1542      Preferences.get("datareporting.healthreport.dbName", undefined);1543    if (FHR_DB_CUSTOM_FILENAME) {1544      FILES_TO_REMOVE.push(1545        OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME),1546        OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME + "-wal"),1547        OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME + "-shm"));1548    }1549    for (let f of FILES_TO_REMOVE) {1550      yield OS.File.remove(f, {ignoreAbsent: true})1551                   .catch(e => this._log.error("removeFHRDatabase - failed to remove " + f, e));1552    }1553  }),1554};1555// Utility functions1556function pingFilePath(ping) {1557  // Support legacy ping formats, who don't have an "id" field, but a "slug" field.1558  let pingIdentifier = (ping.slug) ? ping.slug : ping.id;1559  return OS.Path.join(TelemetryStorage.pingDirectoryPath, pingIdentifier);1560}1561function getPingDirectory() {1562  return Task.spawn(function*() {1563    let directory = TelemetryStorage.pingDirectoryPath;1564    if (!(yield OS.File.exists(directory))) {1565      yield OS.File.makeDir(directory, { unixMode: OS.Constants.S_IRWXU });1566    }1567    return directory;1568  });1569}1570/**1571 * Build the path to the archived ping.1572 * @param {String} aPingId The ping id.1573 * @param {Object} aDate The ping creation date.1574 * @param {String} aType The ping type.1575 * @return {String} The full path to the archived ping.1576 */1577function getArchivedPingPath(aPingId, aDate, aType) {1578  // Helper to pad the month to 2 digits, if needed (e.g. "1" -> "01").1579  let addLeftPadding = value => (value < 10) ? ("0" + value) : value;1580  // Get the ping creation date and generate the archive directory to hold it. Note1581  // that getMonth returns a 0-based month, so we need to add an offset.1582  let archivedPingDir = OS.Path.join(gPingsArchivePath,1583    aDate.getFullYear() + '-' + addLeftPadding(aDate.getMonth() + 1));1584  // Generate the archived ping file path as YYYY-MM/<TIMESTAMP>.UUID.type.json1585  let fileName = [aDate.getTime(), aPingId, aType, "json"].join(".");1586  return OS.Path.join(archivedPingDir, fileName);1587}1588/**1589 * Get the size of the ping file on the disk.1590 * @return {Integer} The file size, in bytes, of the ping file or 0 on errors.1591 */1592var getArchivedPingSize = Task.async(function*(aPingId, aDate, aType) {1593  const path = getArchivedPingPath(aPingId, aDate, aType);1594  let filePaths = [ path + "lz4", path ];1595  for (let path of filePaths) {1596    try {1597      return (yield OS.File.stat(path)).size;1598    } catch (e) {}1599  }1600  // That's odd, this ping doesn't seem to exist.1601  return 0;1602});1603/**1604 * Get the size of the pending ping file on the disk.1605 * @return {Integer} The file size, in bytes, of the ping file or 0 on errors.1606 */1607var getPendingPingSize = Task.async(function*(aPingId) {1608  const path = OS.Path.join(TelemetryStorage.pingDirectoryPath, aPingId)1609  try {1610    return (yield OS.File.stat(path)).size;1611  } catch (e) {}1612  // That's odd, this ping doesn't seem to exist.1613  return 0;1614});1615/**1616 * Check if a directory name is in the "YYYY-MM" format.1617 * @param {String} aDirName The name of the pings archive directory.1618 * @return {Boolean} True if the directory name is in the right format, false otherwise.1619 */1620function isValidArchiveDir(aDirName) {1621  const dirRegEx = /^[0-9]{4}-[0-9]{2}$/;1622  return dirRegEx.test(aDirName);1623}1624/**1625 * Gets a date object from an archive directory name.1626 * @param {String} aDirName The name of the pings archive directory. Must be in the YYYY-MM1627 *        format.1628 * @return {Object} A Date object or null if the dir name is not valid.1629 */1630function getDateFromArchiveDir(aDirName) {1631  let [year, month] = aDirName.split("-");1632  year = parseInt(year);1633  month = parseInt(month);1634  // Make sure to have sane numbers.1635  if (!Number.isFinite(month) || !Number.isFinite(year) || month < 1 || month > 12) {1636    return null;1637  }1638  return new Date(year, month - 1, 1, 0, 0, 0);...

test_TelemetrySendOldPings.js

Source:test_TelemetrySendOldPings.js

1/* Any copyright is dedicated to the Public Domain.2   http://creativecommons.org/publicdomain/zero/1.0/3/**4 * This test case populates the profile with some fake stored5 * pings, and checks that pending pings are immediatlely sent6 * after delayed init.7 */8"use strict"9Cu.import("resource://gre/modules/osfile.jsm", this);10Cu.import("resource://gre/modules/Services.jsm", this);11Cu.import("resource://gre/modules/Promise.jsm", this);12Cu.import("resource://gre/modules/TelemetryStorage.jsm", this);13Cu.import("resource://gre/modules/TelemetryController.jsm", this);14Cu.import("resource://gre/modules/TelemetrySend.jsm", this);15Cu.import("resource://gre/modules/Task.jsm", this);16Cu.import("resource://gre/modules/XPCOMUtils.jsm");17var {OS: {File, Path, Constants}} = Cu.import("resource://gre/modules/osfile.jsm", {});18// We increment TelemetryStorage's MAX_PING_FILE_AGE and19// OVERDUE_PING_FILE_AGE by 1 minute so that our test pings exceed20// those points in time, even taking into account file system imprecision.21const ONE_MINUTE_MS = 60 * 1000;22const OVERDUE_PING_FILE_AGE = TelemetrySend.OVERDUE_PING_FILE_AGE + ONE_MINUTE_MS;23const PING_SAVE_FOLDER = "saved-telemetry-pings";24const PING_TIMEOUT_LENGTH = 5000;25const OVERDUE_PINGS = 6;26const OLD_FORMAT_PINGS = 4;27const RECENT_PINGS = 4;28const TOTAL_EXPECTED_PINGS = OVERDUE_PINGS + RECENT_PINGS + OLD_FORMAT_PINGS;29const PREF_FHR_UPLOAD = "datareporting.healthreport.uploadEnabled";30var gCreatedPings = 0;31var gSeenPings = 0;32/**33 * Creates some Telemetry pings for the and saves them to disk. Each ping gets a34 * unique ID based on an incrementor.35 *36 * @param {Array} aPingInfos An array of ping type objects. Each entry must be an37 *                object containing a "num" field for the number of pings to create and38 *                an "age" field. The latter representing the age in milliseconds to offset39 *                from now. A value of 10 would make the ping 10ms older than now, for40 *                example.41 * @returns Promise42 * @resolve an Array with the created pings ids.43 */44var createSavedPings = Task.async(function* (aPingInfos) {45  let pingIds = [];46  let now = Date.now();47  for (let type in aPingInfos) {48    let num = aPingInfos[type].num;49    let age = now - (aPingInfos[type].age || 0);50    for (let i = 0; i < num; ++i) {51      let pingId = yield TelemetryController.addPendingPing("test-ping", {}, { overwrite: true });52      if (aPingInfos[type].age) {53        // savePing writes to the file synchronously, so we're good to54        // modify the lastModifedTime now.55        let filePath = getSavePathForPingId(pingId);56        yield File.setDates(filePath, null, age);57      }58      gCreatedPings++;59      pingIds.push(pingId);60    }61  }62  return pingIds;63});64/**65 * Deletes locally saved pings if they exist.66 *67 * @param aPingIds an Array of ping ids to delete.68 * @returns Promise69 */70var clearPings = Task.async(function* (aPingIds) {71  for (let pingId of aPingIds) {72    yield TelemetryStorage.removePendingPing(pingId);73  }74});75/**76 * Fakes the pending pings storage quota.77 * @param {Integer} aPendingQuota The new quota, in bytes.78 */79function fakePendingPingsQuota(aPendingQuota) {80  let storage = Cu.import("resource://gre/modules/TelemetryStorage.jsm");81  storage.Policy.getPendingPingsQuota = () => aPendingQuota;82}83/**84 * Returns a handle for the file that a ping should be85 * stored in locally.86 *87 * @returns path88 */89function getSavePathForPingId(aPingId) {90  return Path.join(Constants.Path.profileDir, PING_SAVE_FOLDER, aPingId);91}92/**93 * Check if the number of Telemetry pings received by the HttpServer is not equal94 * to aExpectedNum.95 *96 * @param aExpectedNum the number of pings we expect to receive.97 */98function assertReceivedPings(aExpectedNum) {99  do_check_eq(gSeenPings, aExpectedNum);100}101/**102 * Throws if any pings with the id in aPingIds is saved locally.103 *104 * @param aPingIds an Array of pings ids to check.105 * @returns Promise106 */107var assertNotSaved = Task.async(function* (aPingIds) {108  let saved = 0;109  for (let id of aPingIds) {110    let filePath = getSavePathForPingId(id);111    if (yield File.exists(filePath)) {112      saved++;113    }114  }115  if (saved > 0) {116    do_throw("Found " + saved + " unexpected saved pings.");117  }118});119/**120 * Our handler function for the HttpServer that simply121 * increments the gSeenPings global when it successfully122 * receives and decodes a Telemetry payload.123 *124 * @param aRequest the HTTP request sent from HttpServer.125 */126function pingHandler(aRequest) {127  gSeenPings++;128}129add_task(function* test_setup() {130  PingServer.start();131  PingServer.registerPingHandler(pingHandler);132  do_get_profile();133  loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");134  // Make sure we don't generate unexpected pings due to pref changes.135  yield setEmptyPrefWatchlist();136  Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);137  Services.prefs.setCharPref(TelemetryController.Constants.PREF_SERVER,138                             "http://localhost:" + PingServer.port);139});140/**141 * Setup the tests by making sure the ping storage directory is available, otherwise142 * |TelemetryController.testSaveDirectoryToFile| could fail.143 */144add_task(function* setupEnvironment() {145  // The following tests assume this pref to be true by default.146  Services.prefs.setBoolPref(PREF_FHR_UPLOAD, true);147  yield TelemetryController.testSetup();148  let directory = TelemetryStorage.pingDirectoryPath;149  yield File.makeDir(directory, { ignoreExisting: true, unixMode: OS.Constants.S_IRWXU });150  yield TelemetryStorage.testClearPendingPings();151});152/**153 * Test that really recent pings are sent on Telemetry initialization.154 */155add_task(function* test_recent_pings_sent() {156  let pingTypes = [{ num: RECENT_PINGS }];157  yield createSavedPings(pingTypes);158  yield TelemetryController.testReset();159  yield TelemetrySend.testWaitOnOutgoingPings();160  assertReceivedPings(RECENT_PINGS);161  yield TelemetryStorage.testClearPendingPings();162});163/**164 * Create an overdue ping in the old format and try to send it.165 */166add_task(function* test_overdue_old_format() {167  // A test ping in the old, standard format.168  const PING_OLD_FORMAT = {169    slug: "1234567abcd",170    reason: "test-ping",171    payload: {172      info: {173        reason: "test-ping",174        OS: "XPCShell",175        appID: "SomeId",176        appVersion: "1.0",177        appName: "XPCShell",178        appBuildID: "123456789",179        appUpdateChannel: "Test",180        platformBuildID: "987654321",181      },182    },183  };184  // A ping with no info section, but with a slug.185  const PING_NO_INFO = {186    slug: "1234-no-info-ping",187    reason: "test-ping",188    payload: {}189  };190  // A ping with no payload.191  const PING_NO_PAYLOAD = {192    slug: "5678-no-payload",193    reason: "test-ping",194  };195  // A ping with no info and no slug.196  const PING_NO_SLUG = {197    reason: "test-ping",198    payload: {}199  };200  const PING_FILES_PATHS = [201    getSavePathForPingId(PING_OLD_FORMAT.slug),202    getSavePathForPingId(PING_NO_INFO.slug),203    getSavePathForPingId(PING_NO_PAYLOAD.slug),204    getSavePathForPingId("no-slug-file"),205  ];206  // Write the ping to file and make it overdue.207  yield TelemetryStorage.savePing(PING_OLD_FORMAT, true);208  yield TelemetryStorage.savePing(PING_NO_INFO, true);209  yield TelemetryStorage.savePing(PING_NO_PAYLOAD, true);210  yield TelemetryStorage.savePingToFile(PING_NO_SLUG, PING_FILES_PATHS[3], true);211  for (let f in PING_FILES_PATHS) {212    yield File.setDates(PING_FILES_PATHS[f], null, Date.now() - OVERDUE_PING_FILE_AGE);213  }214  gSeenPings = 0;215  yield TelemetryController.testReset();216  yield TelemetrySend.testWaitOnOutgoingPings();217  assertReceivedPings(OLD_FORMAT_PINGS);218  // |TelemetryStorage.cleanup| doesn't know how to remove a ping with no slug or id,219  // so remove it manually so that the next test doesn't fail.220  yield OS.File.remove(PING_FILES_PATHS[3]);221  yield TelemetryStorage.testClearPendingPings();222});223add_task(function* test_corrupted_pending_pings() {224  const TEST_TYPE = "test_corrupted";225  Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").clear();226  Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").clear();227  // Save a pending ping and get its id.228  let pendingPingId = yield TelemetryController.addPendingPing(TEST_TYPE, {}, {});229  // Try to load it: there should be no error.230  yield TelemetryStorage.loadPendingPing(pendingPingId);231  let h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").snapshot();232  Assert.equal(h.sum, 0, "Telemetry must not report a pending ping load failure");233  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").snapshot();234  Assert.equal(h.sum, 0, "Telemetry must not report a pending ping parse failure");235  // Delete it from the disk, so that its id will be kept in the cache but it will236  // fail loading the file.237  yield OS.File.remove(getSavePathForPingId(pendingPingId));238  // Try to load a pending ping which isn't there anymore.239  yield Assert.rejects(TelemetryStorage.loadPendingPing(pendingPingId),240                       "Telemetry must fail loading a ping which isn't there");241  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").snapshot();242  Assert.equal(h.sum, 1, "Telemetry must report a pending ping load failure");243  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").snapshot();244  Assert.equal(h.sum, 0, "Telemetry must not report a pending ping parse failure");245  // Save a new ping, so that it gets in the pending pings cache.246  pendingPingId = yield TelemetryController.addPendingPing(TEST_TYPE, {}, {});247  // Overwrite it with a corrupted JSON file and then try to load it.248  const INVALID_JSON = "{ invalid,JSON { {1}";249  yield OS.File.writeAtomic(getSavePathForPingId(pendingPingId), INVALID_JSON, { encoding: "utf-8" });250  // Try to load the ping with the corrupted JSON content.251  yield Assert.rejects(TelemetryStorage.loadPendingPing(pendingPingId),252                       "Telemetry must fail loading a corrupted ping");253  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").snapshot();254  Assert.equal(h.sum, 1, "Telemetry must report a pending ping load failure");255  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").snapshot();256  Assert.equal(h.sum, 1, "Telemetry must report a pending ping parse failure");257  let exists = yield OS.File.exists(getSavePathForPingId(pendingPingId));258  Assert.ok(!exists, "The unparseable ping should have been removed");259  yield TelemetryStorage.testClearPendingPings();260});261/**262 * Create some recent and overdue pings and verify that they get sent.263 */264add_task(function* test_overdue_pings_trigger_send() {265  let pingTypes = [266    { num: RECENT_PINGS },267    { num: OVERDUE_PINGS, age: OVERDUE_PING_FILE_AGE },268  ];269  let pings = yield createSavedPings(pingTypes);270  let recentPings = pings.slice(0, RECENT_PINGS);271  let overduePings = pings.slice(-OVERDUE_PINGS);272  yield TelemetryController.testReset();273  yield TelemetrySend.testWaitOnOutgoingPings();274  assertReceivedPings(TOTAL_EXPECTED_PINGS);275  yield assertNotSaved(recentPings);276  yield assertNotSaved(overduePings);277  Assert.equal(TelemetrySend.overduePingsCount, overduePings.length,278               "Should have tracked the correct amount of overdue pings");279  yield TelemetryStorage.testClearPendingPings();280});281/**282 * Create a ping in the old format, send it, and make sure the request URL contains283 * the correct version query parameter.284 */285add_task(function* test_overdue_old_format() {286  // A test ping in the old, standard format.287  const PING_OLD_FORMAT = {288    slug: "1234567abcd",289    reason: "test-ping",290    payload: {291      info: {292        reason: "test-ping",293        OS: "XPCShell",294        appID: "SomeId",295        appVersion: "1.0",296        appName: "XPCShell",297        appBuildID: "123456789",298        appUpdateChannel: "Test",299        platformBuildID: "987654321",300      },301    },302  };303  const filePath =304    Path.join(Constants.Path.profileDir, PING_SAVE_FOLDER, PING_OLD_FORMAT.slug);305  // Write the ping to file and make it overdue.306  yield TelemetryStorage.savePing(PING_OLD_FORMAT, true);307  yield File.setDates(filePath, null, Date.now() - OVERDUE_PING_FILE_AGE);308  let receivedPings = 0;309  // Register a new prefix handler to validate the URL.310  PingServer.registerPingHandler(request => {311    // Check that we have a version query parameter in the URL.312    Assert.notEqual(request.queryString, "");313    // Make sure the version in the query string matches the old ping format version.314    let params = request.queryString.split("&");315    Assert.ok(params.find(p => p == "v=1"));316    receivedPings++;317  });318  yield TelemetryController.testReset();319  yield TelemetrySend.testWaitOnOutgoingPings();320  Assert.equal(receivedPings, 1, "We must receive a ping in the old format.");321  yield TelemetryStorage.testClearPendingPings();322  PingServer.resetPingHandler();323});324add_task(function* test_pendingPingsQuota() {325  const PING_TYPE = "foo";326  // Disable upload so pings don't get sent and removed from the pending pings directory.327  Services.prefs.setBoolPref(PREF_FHR_UPLOAD, false);328  // Remove all the pending pings then startup and wait for the cleanup task to complete.329  // There should be nothing to remove.330  yield TelemetryStorage.testClearPendingPings();331  yield TelemetryController.testReset();332  yield TelemetrySend.testWaitOnOutgoingPings();333  yield TelemetryStorage.testPendingQuotaTaskPromise();334  // Remove the pending deletion ping generated when flipping FHR upload off.335  yield TelemetryStorage.testClearPendingPings();336  let expectedPrunedPings = [];337  let expectedNotPrunedPings = [];338  let checkPendingPings = Task.async(function*() {339    // Check that the pruned pings are not on disk anymore.340    for (let prunedPingId of expectedPrunedPings) {341      yield Assert.rejects(TelemetryStorage.loadPendingPing(prunedPingId),342                           "Ping " + prunedPingId + " should have been pruned.");343      const pingPath = getSavePathForPingId(prunedPingId);344      Assert.ok(!(yield OS.File.exists(pingPath)), "The ping should not be on the disk anymore.");345    }346    // Check that the expected pings are there.347    for (let expectedPingId of expectedNotPrunedPings) {348      Assert.ok((yield TelemetryStorage.loadPendingPing(expectedPingId)),349                "Ping" + expectedPingId + " should be among the pending pings.");350    }351  });352  let pendingPingsInfo = [];353  let pingsSizeInBytes = 0;354  // Create 10 pings to test the pending pings quota.355  for (let days = 1; days < 11; days++) {356    const date = fakeNow(2010, 1, days, 1, 1, 0);357    const pingId = yield TelemetryController.addPendingPing(PING_TYPE, {}, {});358    // Find the size of the ping.359    const pingFilePath = getSavePathForPingId(pingId);360    const pingSize = (yield OS.File.stat(pingFilePath)).size;361    // Add the info at the beginning of the array, so that most recent pings come first.362    pendingPingsInfo.unshift({id: pingId, size: pingSize, timestamp: date.getTime() });363    // Set the last modification date.364    yield OS.File.setDates(pingFilePath, null, date.getTime());365    // Add it to the pending ping directory size.366    pingsSizeInBytes += pingSize;367  }368  // We need to test the pending pings size before we hit the quota, otherwise a special369  // value is recorded.370  Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").clear();371  Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").clear();372  Telemetry.getHistogramById("TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS").clear();373  yield TelemetryController.testReset();374  yield TelemetryStorage.testPendingQuotaTaskPromise();375  // Check that the correct values for quota probes are reported when no quota is hit.376  let h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").snapshot();377  Assert.equal(h.sum, Math.round(pingsSizeInBytes / 1024 / 1024),378               "Telemetry must report the correct pending pings directory size.");379  h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").snapshot();380  Assert.equal(h.sum, 0, "Telemetry must report 0 evictions if quota is not hit.");381  h = Telemetry.getHistogramById("TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS").snapshot();382  Assert.equal(h.sum, 0, "Telemetry must report a null elapsed time if quota is not hit.");383  // Set the quota to 80% of the space.384  const testQuotaInBytes = pingsSizeInBytes * 0.8;385  fakePendingPingsQuota(testQuotaInBytes);386  // The storage prunes pending pings until we reach 90% of the requested storage quota.387  // Based on that, find how many pings should be kept.388  const safeQuotaSize = Math.round(testQuotaInBytes * 0.9);389  let sizeInBytes = 0;390  let pingsWithinQuota = [];391  let pingsOutsideQuota = [];392  for (let pingInfo of pendingPingsInfo) {393    sizeInBytes += pingInfo.size;394    if (sizeInBytes >= safeQuotaSize) {395      pingsOutsideQuota.push(pingInfo.id);396      continue;397    }398    pingsWithinQuota.push(pingInfo.id);399  }400  expectedNotPrunedPings = pingsWithinQuota;401  expectedPrunedPings = pingsOutsideQuota;402  // Reset TelemetryController to start the pending pings cleanup.403  yield TelemetryController.testReset();404  yield TelemetryStorage.testPendingQuotaTaskPromise();405  yield checkPendingPings();406  h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").snapshot();407  Assert.equal(h.sum, pingsOutsideQuota.length,408               "Telemetry must correctly report the over quota pings evicted from the pending pings directory.");409  h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").snapshot();410  Assert.equal(h.sum, 17, "Pending pings quota was hit, a special size must be reported.");411  // Trigger a cleanup again and make sure we're not removing anything.412  yield TelemetryController.testReset();413  yield TelemetryStorage.testPendingQuotaTaskPromise();414  yield checkPendingPings();415  const OVERSIZED_PING_ID = "9b21ec8f-f762-4d28-a2c1-44e1c4694f24";416  // Create a pending oversized ping.417  const OVERSIZED_PING = {418    id: OVERSIZED_PING_ID,419    type: PING_TYPE,420    creationDate: (new Date()).toISOString(),421    // Generate a 2MB string to use as the ping payload.422    payload: generateRandomString(2 * 1024 * 1024),423  };424  yield TelemetryStorage.savePendingPing(OVERSIZED_PING);425  // Reset the histograms.426  Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").clear();427  Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB").clear();428  // Try to manually load the oversized ping.429  yield Assert.rejects(TelemetryStorage.loadPendingPing(OVERSIZED_PING_ID),430                       "The oversized ping should have been pruned.");431  Assert.ok(!(yield OS.File.exists(getSavePathForPingId(OVERSIZED_PING_ID))),432            "The ping should not be on the disk anymore.");433  // Make sure we're correctly updating the related histograms.434  h = Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").snapshot();435  Assert.equal(h.sum, 1, "Telemetry must report 1 oversized ping in the pending pings directory.");436  h = Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB").snapshot();437  Assert.equal(h.counts[2], 1, "Telemetry must report a 2MB, oversized, ping.");438  // Save the ping again to check if it gets pruned when scanning the pings directory.439  yield TelemetryStorage.savePendingPing(OVERSIZED_PING);440  expectedPrunedPings.push(OVERSIZED_PING_ID);441  // Scan the pending pings directory.442  yield TelemetryController.testReset();443  yield TelemetryStorage.testPendingQuotaTaskPromise();444  yield checkPendingPings();445  // Make sure we're correctly updating the related histograms.446  h = Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").snapshot();447  Assert.equal(h.sum, 2, "Telemetry must report 1 oversized ping in the pending pings directory.");448  h = Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB").snapshot();449  Assert.equal(h.counts[2], 2, "Telemetry must report two 2MB, oversized, pings.");450  Services.prefs.setBoolPref(PREF_FHR_UPLOAD, true);451});452add_task(function* teardown() {453  yield PingServer.stop();...

test_TelemetryController.js

Source:test_TelemetryController.js

1/* Any copyright is dedicated to the Public Domain.2   http://creativecommons.org/publicdomain/zero/1.0/3*/4/* This testcase triggers two telemetry pings.5 *6 * Telemetry code keeps histograms of past telemetry pings. The first7 * ping populates these histograms. One of those histograms is then8 * checked in the second request.9 */10Cu.import("resource://gre/modules/ClientID.jsm");11Cu.import("resource://gre/modules/Services.jsm");12Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);13Cu.import("resource://gre/modules/TelemetryController.jsm", this);14Cu.import("resource://gre/modules/TelemetryStorage.jsm", this);15Cu.import("resource://gre/modules/TelemetrySend.jsm", this);16Cu.import("resource://gre/modules/TelemetryArchive.jsm", this);17Cu.import("resource://gre/modules/Task.jsm", this);18Cu.import("resource://gre/modules/Promise.jsm", this);19Cu.import("resource://gre/modules/Preferences.jsm");20const PING_FORMAT_VERSION = 4;21const DELETION_PING_TYPE = "deletion";22const TEST_PING_TYPE = "test-ping-type";23const PLATFORM_VERSION = "1.9.2";24const APP_VERSION = "1";25const APP_NAME = "XPCShell";26const PREF_BRANCH = "toolkit.telemetry.";27const PREF_ENABLED = PREF_BRANCH + "enabled";28const PREF_ARCHIVE_ENABLED = PREF_BRANCH + "archive.enabled";29const PREF_FHR_UPLOAD_ENABLED = "datareporting.healthreport.uploadEnabled";30const PREF_UNIFIED = PREF_BRANCH + "unified";31var gClientID = null;32function sendPing(aSendClientId, aSendEnvironment) {33  if (PingServer.started) {34    TelemetrySend.setServer("http://localhost:" + PingServer.port);35  } else {36    TelemetrySend.setServer("http://doesnotexist");37  }38  let options = {39    addClientId: aSendClientId,40    addEnvironment: aSendEnvironment,41  };42  return TelemetryController.submitExternalPing(TEST_PING_TYPE, {}, options);43}44function checkPingFormat(aPing, aType, aHasClientId, aHasEnvironment) {45  const MANDATORY_PING_FIELDS = [46    "type", "id", "creationDate", "version", "application", "payload"47  ];48  const APPLICATION_TEST_DATA = {49    buildId: gAppInfo.appBuildID,50    name: APP_NAME,51    version: APP_VERSION,52    displayVersion: AppConstants.MOZ_APP_VERSION_DISPLAY,53    vendor: "Mozilla",54    platformVersion: PLATFORM_VERSION,55    xpcomAbi: "noarch-spidermonkey",56  };57  // Check that the ping contains all the mandatory fields.58  for (let f of MANDATORY_PING_FIELDS) {59    Assert.ok(f in aPing, f + " must be available.");60  }61  Assert.equal(aPing.type, aType, "The ping must have the correct type.");62  Assert.equal(aPing.version, PING_FORMAT_VERSION, "The ping must have the correct version.");63  // Test the application section.64  for (let f in APPLICATION_TEST_DATA) {65    Assert.equal(aPing.application[f], APPLICATION_TEST_DATA[f],66                 f + " must have the correct value.");67  }68  // We can't check the values for channel and architecture. Just make69  // sure they are in.70  Assert.ok("architecture" in aPing.application,71            "The application section must have an architecture field.");72  Assert.ok("channel" in aPing.application,73            "The application section must have a channel field.");74  // Check the clientId and environment fields, as needed.75  Assert.equal("clientId" in aPing, aHasClientId);76  Assert.equal("environment" in aPing, aHasEnvironment);77}78add_task(function* test_setup() {79  // Addon manager needs a profile directory80  do_get_profile();81  loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");82  // Make sure we don't generate unexpected pings due to pref changes.83  yield setEmptyPrefWatchlist();84  Services.prefs.setBoolPref(PREF_ENABLED, true);85  Services.prefs.setBoolPref(PREF_FHR_UPLOAD_ENABLED, true);86  yield new Promise(resolve =>87    Telemetry.asyncFetchTelemetryData(wrapWithExceptionHandler(resolve)));88});89add_task(function* asyncSetup() {90  yield TelemetryController.testSetup();91});92// Ensure that not overwriting an existing file fails silently93add_task(function* test_overwritePing() {94  let ping = {id: "foo"};95  yield TelemetryStorage.savePing(ping, true);96  yield TelemetryStorage.savePing(ping, false);97  yield TelemetryStorage.cleanupPingFile(ping);98});99// Checks that a sent ping is correctly received by a dummy http server.100add_task(function* test_simplePing() {101  PingServer.start();102  // Update the Telemetry Server preference with the address of the local server.103  // Otherwise we might end up sending stuff to a non-existing server after104  // |TelemetryController.testReset| is called.105  Preferences.set(TelemetryController.Constants.PREF_SERVER, "http://localhost:" + PingServer.port);106  yield sendPing(false, false);107  let request = yield PingServer.promiseNextRequest();108  // Check that we have a version query parameter in the URL.109  Assert.notEqual(request.queryString, "");110  // Make sure the version in the query string matches the new ping format version.111  let params = request.queryString.split("&");112  Assert.ok(params.find(p => p == ("v=" + PING_FORMAT_VERSION)));113  let ping = decodeRequestPayload(request);114  checkPingFormat(ping, TEST_PING_TYPE, false, false);115});116add_task(function* test_disableDataUpload() {117  const isUnified = Preferences.get(PREF_UNIFIED, false);118  if (!isUnified) {119    // Skipping the test if unified telemetry is off, as no deletion ping will120    // be generated.121    return;122  }123  // Disable FHR upload: this should trigger a deletion ping.124  Preferences.set(PREF_FHR_UPLOAD_ENABLED, false);125  let ping = yield PingServer.promiseNextPing();126  checkPingFormat(ping, DELETION_PING_TYPE, true, false);127  // Wait on ping activity to settle.128  yield TelemetrySend.testWaitOnOutgoingPings();129  // Restore FHR Upload.130  Preferences.set(PREF_FHR_UPLOAD_ENABLED, true);131  // Simulate a failure in sending the deletion ping by disabling the HTTP server.132  yield PingServer.stop();133  // Try to send a ping. It will be saved as pending  and get deleted when disabling upload.134  TelemetryController.submitExternalPing(TEST_PING_TYPE, {});135  // Disable FHR upload to send a deletion ping again.136  Preferences.set(PREF_FHR_UPLOAD_ENABLED, false);137  // Wait on sending activity to settle, as |TelemetryController.testReset()| doesn't do that.138  yield TelemetrySend.testWaitOnOutgoingPings();139  // Wait for the pending pings to be deleted. Resetting TelemetryController doesn't140  // trigger the shutdown, so we need to call it ourselves.141  yield TelemetryStorage.shutdown();142  // Simulate a restart, and spin the send task.143  yield TelemetryController.testReset();144  // Disabling Telemetry upload must clear out all the pending pings.145  let pendingPings = yield TelemetryStorage.loadPendingPingList();146  Assert.equal(pendingPings.length, 1,147               "All the pending pings but the deletion ping should have been deleted");148  // Enable the ping server again.149  PingServer.start();150  // We set the new server using the pref, otherwise it would get reset with151  // |TelemetryController.testReset|.152  Preferences.set(TelemetryController.Constants.PREF_SERVER, "http://localhost:" + PingServer.port);153  // Stop the sending task and then start it again.154  yield TelemetrySend.shutdown();155  // Reset the controller to spin the ping sending task.156  yield TelemetryController.testReset();157  ping = yield PingServer.promiseNextPing();158  checkPingFormat(ping, DELETION_PING_TYPE, true, false);159  // Wait on ping activity to settle before moving on to the next test. If we were160  // to shut down telemetry, even though the PingServer caught the expected pings,161  // TelemetrySend could still be processing them (clearing pings would happen in162  // a couple of ticks). Shutting down would cancel the request and save them as163  // pending pings.164  yield TelemetrySend.testWaitOnOutgoingPings();165  // Restore FHR Upload.166  Preferences.set(PREF_FHR_UPLOAD_ENABLED, true);167});168add_task(function* test_pingHasClientId() {169  const PREF_CACHED_CLIENTID = "toolkit.telemetry.cachedClientID";170  // Make sure we have no cached client ID for this test: we'll try to send171  // a ping with it while Telemetry is being initialized.172  Preferences.reset(PREF_CACHED_CLIENTID);173  yield TelemetryController.testShutdown();174  yield ClientID._reset();175  yield TelemetryStorage.testClearPendingPings();176  // And also clear the counter histogram since we're here.177  let h = Telemetry.getHistogramById("TELEMETRY_PING_SUBMISSION_WAITING_CLIENTID");178  h.clear();179  // Init telemetry and try to send a ping with a client ID.180  let promisePingSetup = TelemetryController.testReset();181  yield sendPing(true, false);182  Assert.equal(h.snapshot().sum, 1,183               "We must have a ping waiting for the clientId early during startup.");184  // Wait until we are fully initialized. Pings will be assembled but won't get185  // sent before then.186  yield promisePingSetup;187  let ping = yield PingServer.promiseNextPing();188  // Fetch the client ID after initializing and fetching the the ping, so we189  // don't unintentionally trigger its loading. We'll still need the client ID190  // to see if the ping looks sane.191  gClientID = yield ClientID.getClientID();192  checkPingFormat(ping, TEST_PING_TYPE, true, false);193  Assert.equal(ping.clientId, gClientID, "The correct clientId must be reported.");194  // Shutdown Telemetry so we can safely restart it.195  yield TelemetryController.testShutdown();196  yield TelemetryStorage.testClearPendingPings();197  // We should have cached the client ID now. Lets confirm that by checking it before198  // the async ping setup is finished.199  h.clear();200  promisePingSetup = TelemetryController.testReset();201  yield sendPing(true, false);202  yield promisePingSetup;203  // Check that we received the cached client id.204  Assert.equal(h.snapshot().sum, 0, "We must have used the cached clientId.");205  ping = yield PingServer.promiseNextPing();206  checkPingFormat(ping, TEST_PING_TYPE, true, false);207  Assert.equal(ping.clientId, gClientID,208               "Telemetry should report the correct cached clientId.");209  // Check that sending a ping without relying on the cache, after the210  // initialization, still works.211  Preferences.reset(PREF_CACHED_CLIENTID);212  yield TelemetryController.testShutdown();213  yield TelemetryStorage.testClearPendingPings();214  yield TelemetryController.testReset();215  yield sendPing(true, false);216  ping = yield PingServer.promiseNextPing();217  checkPingFormat(ping, TEST_PING_TYPE, true, false);218  Assert.equal(ping.clientId, gClientID, "The correct clientId must be reported.");219  Assert.equal(h.snapshot().sum, 0, "No ping should have been waiting for a clientId.");220});221add_task(function* test_pingHasEnvironment() {222  // Send a ping with the environment data.223  yield sendPing(false, true);224  let ping = yield PingServer.promiseNextPing();225  checkPingFormat(ping, TEST_PING_TYPE, false, true);226  // Test a field in the environment build section.227  Assert.equal(ping.application.buildId, ping.environment.build.buildId);228});229add_task(function* test_pingHasEnvironmentAndClientId() {230  // Send a ping with the environment data and client id.231  yield sendPing(true, true);232  let ping = yield PingServer.promiseNextPing();233  checkPingFormat(ping, TEST_PING_TYPE, true, true);234  // Test a field in the environment build section.235  Assert.equal(ping.application.buildId, ping.environment.build.buildId);236  // Test that we have the correct clientId.237  Assert.equal(ping.clientId, gClientID, "The correct clientId must be reported.");238});239add_task(function* test_archivePings() {240  let now = new Date(2009, 10, 18, 12, 0, 0);241  fakeNow(now);242  // Disable ping upload so that pings don't get sent.243  // With unified telemetry the FHR upload pref controls this,244  // with non-unified telemetry the Telemetry enabled pref.245  const isUnified = Preferences.get(PREF_UNIFIED, false);246  const uploadPref = isUnified ? PREF_FHR_UPLOAD_ENABLED : PREF_ENABLED;247  Preferences.set(uploadPref, false);248  // If we're using unified telemetry, disabling ping upload will generate a "deletion"249  // ping. Catch it.250  if (isUnified) {251    let ping = yield PingServer.promiseNextPing();252    checkPingFormat(ping, DELETION_PING_TYPE, true, false);253  }254  // Register a new Ping Handler that asserts if a ping is received, then send a ping.255  PingServer.registerPingHandler(() => Assert.ok(false, "Telemetry must not send pings if not allowed to."));256  let pingId = yield sendPing(true, true);257  // Check that the ping was archived, even with upload disabled.258  let ping = yield TelemetryArchive.promiseArchivedPingById(pingId);259  Assert.equal(ping.id, pingId, "TelemetryController should still archive pings.");260  // Check that pings don't get archived if not allowed to.261  now = new Date(2010, 10, 18, 12, 0, 0);262  fakeNow(now);263  Preferences.set(PREF_ARCHIVE_ENABLED, false);264  pingId = yield sendPing(true, true);265  let promise = TelemetryArchive.promiseArchivedPingById(pingId);266  Assert.ok((yield promiseRejects(promise)),267    "TelemetryController should not archive pings if the archive pref is disabled.");268  // Enable archiving and the upload so that pings get sent and archived again.269  Preferences.set(uploadPref, true);270  Preferences.set(PREF_ARCHIVE_ENABLED, true);271  now = new Date(2014, 6, 18, 22, 0, 0);272  fakeNow(now);273  // Restore the non asserting ping handler.274  PingServer.resetPingHandler();275  pingId = yield sendPing(true, true);276  // Check that we archive pings when successfully sending them.277  yield PingServer.promiseNextPing();278  ping = yield TelemetryArchive.promiseArchivedPingById(pingId);279  Assert.equal(ping.id, pingId,280    "TelemetryController should still archive pings if ping upload is enabled.");281});282// Test that we fuzz the submission time around midnight properly283// to avoid overloading the telemetry servers.284add_task(function* test_midnightPingSendFuzzing() {285  const fuzzingDelay = 60 * 60 * 1000;286  fakeMidnightPingFuzzingDelay(fuzzingDelay);287  let now = new Date(2030, 5, 1, 11, 0, 0);288  fakeNow(now);289  let waitForTimer = () => new Promise(resolve => {290    fakePingSendTimer((callback, timeout) => {291      resolve([callback, timeout]);292    }, () => {});293  });294  PingServer.clearRequests();295  yield TelemetryController.testReset();296  // A ping after midnight within the fuzzing delay should not get sent.297  now = new Date(2030, 5, 2, 0, 40, 0);298  fakeNow(now);299  PingServer.registerPingHandler((req, res) => {300    Assert.ok(false, "No ping should be received yet.");301  });302  let timerPromise = waitForTimer();303  yield sendPing(true, true);304  let [timerCallback, timerTimeout] = yield timerPromise;305  Assert.ok(!!timerCallback);306  Assert.deepEqual(futureDate(now, timerTimeout), new Date(2030, 5, 2, 1, 0, 0));307  // A ping just before the end of the fuzzing delay should not get sent.308  now = new Date(2030, 5, 2, 0, 59, 59);309  fakeNow(now);310  timerPromise = waitForTimer();311  yield sendPing(true, true);312  [timerCallback, timerTimeout] = yield timerPromise;313  Assert.deepEqual(timerTimeout, 1 * 1000);314  // Restore the previous ping handler.315  PingServer.resetPingHandler();316  // Setting the clock to after the fuzzing delay, we should trigger the two ping sends317  // with the timer callback.318  now = futureDate(now, timerTimeout);319  fakeNow(now);320  yield timerCallback();321  const pings = yield PingServer.promiseNextPings(2);322  for (let ping of pings) {323    checkPingFormat(ping, TEST_PING_TYPE, true, true);324  }325  yield TelemetrySend.testWaitOnOutgoingPings();326  // Moving the clock further we should still send pings immediately.327  now = futureDate(now, 5 * 60 * 1000);328  yield sendPing(true, true);329  let ping = yield PingServer.promiseNextPing();330  checkPingFormat(ping, TEST_PING_TYPE, true, true);331  yield TelemetrySend.testWaitOnOutgoingPings();332  // Check that pings shortly before midnight are immediately sent.333  now = fakeNow(2030, 5, 3, 23, 59, 0);334  yield sendPing(true, true);335  ping = yield PingServer.promiseNextPing();336  checkPingFormat(ping, TEST_PING_TYPE, true, true);337  yield TelemetrySend.testWaitOnOutgoingPings();338  // Clean-up.339  fakeMidnightPingFuzzingDelay(0);340  fakePingSendTimer(() => {}, () => {});341});342add_task(function* test_changePingAfterSubmission() {343  // Submit a ping with a custom payload.344  let payload = { canary: "test" };345  let pingPromise = TelemetryController.submitExternalPing(TEST_PING_TYPE, payload, options);346  // Change the payload with a predefined value.347  payload.canary = "changed";348  // Wait for the ping to be archived.349  const pingId = yield pingPromise;350  // Make sure our changes didn't affect the submitted payload.351  let archivedCopy = yield TelemetryArchive.promiseArchivedPingById(pingId);352  Assert.equal(archivedCopy.payload.canary, "test",353               "The payload must not be changed after being submitted.");354});355add_task(function* test_telemetryEnabledUnexpectedValue() {356  // Remove the default value for toolkit.telemetry.enabled from the default prefs.357  // Otherwise, we wouldn't be able to set the pref to a string.358  let defaultPrefBranch = Services.prefs.getDefaultBranch(null);359  defaultPrefBranch.deleteBranch(PREF_ENABLED);360  // Set the preferences controlling the Telemetry status to a string.361  Preferences.set(PREF_ENABLED, "false");362  // Check that Telemetry is not enabled.363  yield TelemetryController.testReset();364  Assert.equal(Telemetry.canRecordExtended, false,365               "Invalid values must not enable Telemetry recording.");366  // Delete the pref again.367  defaultPrefBranch.deleteBranch(PREF_ENABLED);368  // Make sure that flipping it to true works.369  Preferences.set(PREF_ENABLED, true);370  yield TelemetryController.testReset();371  Assert.equal(Telemetry.canRecordExtended, true,372               "True must enable Telemetry recording.");373  // Also check that the false works as well.374  Preferences.set(PREF_ENABLED, false);375  yield TelemetryController.testReset();376  Assert.equal(Telemetry.canRecordExtended, false,377               "False must disable Telemetry recording.");378});379add_task(function* test_telemetryCleanFHRDatabase() {380  const FHR_DBNAME_PREF = "datareporting.healthreport.dbName";381  const CUSTOM_DB_NAME = "unlikely.to.be.used.sqlite";382  const DEFAULT_DB_NAME = "healthreport.sqlite";383  // Check that we're able to remove a FHR DB with a custom name.384  const CUSTOM_DB_PATHS = [385    OS.Path.join(OS.Constants.Path.profileDir, CUSTOM_DB_NAME),386    OS.Path.join(OS.Constants.Path.profileDir, CUSTOM_DB_NAME + "-wal"),387    OS.Path.join(OS.Constants.Path.profileDir, CUSTOM_DB_NAME + "-shm"),388  ];389  Preferences.set(FHR_DBNAME_PREF, CUSTOM_DB_NAME);390  // Write fake DB files to the profile directory.391  for (let dbFilePath of CUSTOM_DB_PATHS) {392    yield OS.File.writeAtomic(dbFilePath, "some data");393  }394  // Trigger the cleanup and check that the files were removed.395  yield TelemetryStorage.removeFHRDatabase();396  for (let dbFilePath of CUSTOM_DB_PATHS) {397    Assert.ok(!(yield OS.File.exists(dbFilePath)), "The DB must not be on the disk anymore: " + dbFilePath);398  }399  // We should not break anything if there's no DB file.400  yield TelemetryStorage.removeFHRDatabase();401  // Check that we're able to remove a FHR DB with the default name.402  Preferences.reset(FHR_DBNAME_PREF);403  const DEFAULT_DB_PATHS = [404    OS.Path.join(OS.Constants.Path.profileDir, DEFAULT_DB_NAME),405    OS.Path.join(OS.Constants.Path.profileDir, DEFAULT_DB_NAME + "-wal"),406    OS.Path.join(OS.Constants.Path.profileDir, DEFAULT_DB_NAME + "-shm"),407  ];408  // Write fake DB files to the profile directory.409  for (let dbFilePath of DEFAULT_DB_PATHS) {410    yield OS.File.writeAtomic(dbFilePath, "some data");411  }412  // Trigger the cleanup and check that the files were removed.413  yield TelemetryStorage.removeFHRDatabase();414  for (let dbFilePath of DEFAULT_DB_PATHS) {415    Assert.ok(!(yield OS.File.exists(dbFilePath)), "The DB must not be on the disk anymore: " + dbFilePath);416  }417});418add_task(function* stopServer() {419  yield PingServer.stop();...

test_tcping.py

Source:test_tcping.py

...15            tcp_ping = ping.TCPing(destination=args.destination,16                                   port=args.port,17                                   timeout=args.timeout,18                                   use_ipv6=args.use_ipv6)19            tcp_ping.do_ping()20        self.assertEqual(len(tcp_ping.measures), 1)21    def test_ping_domain_incorrect(self):22        with mock.patch('socket.socket') as mock_socket:23            mock_socket.return_value.recv.return_value = b""24            cmd_parser = tcping.create_cmd_parser()25            args = cmd_parser.parse_args(['google.csom'])26            tcp_ping = ping.TCPing(destination=args.destination,27                                   port=args.port,28                                   timeout=args.timeout,29                                   use_ipv6=args.use_ipv6)30            with self.assertRaises(errors.InvalidIpOrDomain):31                tcp_ping.do_ping()32    def test_ping_ip_standart(self):33        with mock.patch('socket.socket') as mock_socket:34            mock_socket.return_value.recv.return_value = b""35            cmd_parser = tcping.create_cmd_parser()36            args = cmd_parser.parse_args(['64.233.165.101', '-c', '10'])37            tcp_ping = ping.TCPing(destination=args.destination,38                                   port=args.port,39                                   timeout=args.timeout,40                                   use_ipv6=args.use_ipv6)41            tcp_ping.do_ping()42        self.assertEqual(len(tcp_ping.measures), 1)43    def test_ping_ip_incorrect(self):44        with mock.patch('socket.socket') as mock_socket:45            mock_socket.return_value.recv.return_value = b""46            cmd_parser = tcping.create_cmd_parser()47            args = cmd_parser.parse_args(['64.233.165.101.123.214'])48            tcp_ping = ping.TCPing(destination=args.destination,49                                   port=args.port,50                                   timeout=args.timeout,51                                   use_ipv6=args.use_ipv6)52            with self.assertRaises(errors.InvalidIpOrDomain):53                tcp_ping.do_ping()54class TestParsing(unittest.TestCase):55    def test_parsing_with_args(self):56        cmd_parser = tcping.create_cmd_parser()57        args = cmd_parser.parse_args(['google.com',58                                      '-t', '10',59                                      '-p', '443',60                                      '-6'])61        tcp_ping = ping.TCPing(destination=args.destination, port=args.port,62                               timeout=args.timeout, use_ipv6=args.use_ipv6)63        self.assertEqual(tcp_ping.use_ipv6, True)64        self.assertEqual(tcp_ping.timeout, 10)65        self.assertEqual(tcp_ping.port, 443)66    def test_parsing_without_args_from_cmd(self):67        cmd_parser = tcping.create_cmd_parser()...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.