How to use use_external_config method in prospector

Best Python code snippet using prospector_python
cros_chrome_sdk.py
Source:cros_chrome_sdk.py
1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.2# Use of this source code is governed by a BSD-style license that can be3# found in the LICENSE file.4"""The cros chrome-sdk command for the simple chrome workflow."""5from __future__ import print_function6import argparse7import collections8import contextlib9import glob10import json11import os12import distutils.version13from chromite.cli import command14from chromite.lib import cache15from chromite.lib import chrome_util16from chromite.lib import cros_build_lib17from chromite.lib import cros_logging as logging18from chromite.lib import git19from chromite.lib import gs20from chromite.lib import osutils21from chromite.lib import path_util22from chromite.lib import stats23from chromite.cbuildbot import archive_lib24from chromite.cbuildbot import config_lib25from chromite.cbuildbot import constants26from gn_helpers import gn_helpers27COMMAND_NAME = 'chrome-sdk'28CUSTOM_VERSION = 'custom'29def Log(*args, **kwargs):30  """Conditional logging.31  Args:32    silent: If set to True, then logs with level DEBUG.  logs with level INFO33      otherwise.  Defaults to False.34  """35  silent = kwargs.pop('silent', False)36  level = logging.DEBUG if silent else logging.INFO37  logging.log(level, *args, **kwargs)38class MissingSDK(Exception):39  """Error thrown when we cannot find an SDK."""40  def __init__(self, board, version=None):41    msg = 'Cannot find SDK for %r' % (board,)42    if version is not None:43      msg += ' with version %s' % (version,)44    Exception.__init__(self, msg)45class SDKFetcher(object):46  """Functionality for fetching an SDK environment.47  For the version of ChromeOS specified, the class downloads and caches48  SDK components.49  """50  SDK_BOARD_ENV = '%SDK_BOARD'51  SDK_PATH_ENV = '%SDK_PATH'52  SDK_VERSION_ENV = '%SDK_VERSION'53  SDKContext = collections.namedtuple(54      'SDKContext', ['version', 'target_tc', 'key_map'])55  TARBALL_CACHE = 'tarballs'56  MISC_CACHE = 'misc'57  TARGET_TOOLCHAIN_KEY = 'target_toolchain'58  def __init__(self, cache_dir, board, clear_cache=False, chrome_src=None,59               sdk_path=None, toolchain_path=None, silent=False,60               use_external_config=None):61    """Initialize the class.62    Args:63      cache_dir: The toplevel cache dir to use.64      board: The board to manage the SDK for.65      clear_cache: Clears the sdk cache during __init__.66      chrome_src: The location of the chrome checkout.  If unspecified, the67        cwd is presumed to be within a chrome checkout.68      sdk_path: The path (whether a local directory or a gs:// path) to fetch69        SDK components from.70      toolchain_path: The path (whether a local directory or a gs:// path) to71        fetch toolchain components from.72      silent: If set, the fetcher prints less output.73      use_external_config: When identifying the configuration for a board,74        force usage of the external configuration if both external and internal75        are available.76    """77    site_config = config_lib.GetConfig()78    self.cache_base = os.path.join(cache_dir, COMMAND_NAME)79    if clear_cache:80      logging.warning('Clearing the SDK cache.')81      osutils.RmDir(self.cache_base, ignore_missing=True)82    self.tarball_cache = cache.TarballCache(83        os.path.join(self.cache_base, self.TARBALL_CACHE))84    self.misc_cache = cache.DiskCache(85        os.path.join(self.cache_base, self.MISC_CACHE))86    self.board = board87    self.config = site_config.FindCanonicalConfigForBoard(88        board, allow_internal=not use_external_config)89    self.gs_base = archive_lib.GetBaseUploadURI(self.config)90    self.clear_cache = clear_cache91    self.chrome_src = chrome_src92    self.sdk_path = sdk_path93    self.toolchain_path = toolchain_path94    self.silent = silent95    # For external configs, there is no need to run 'gsutil config', because96    # the necessary files are all accessible to anonymous users.97    internal = self.config['internal']98    self.gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=internal)99    if self.sdk_path is None:100      self.sdk_path = os.environ.get(self.SDK_PATH_ENV)101    if self.toolchain_path is None:102      self.toolchain_path = 'gs://%s' % constants.SDK_GS_BUCKET103  def _UpdateTarball(self, url, ref):104    """Worker function to fetch tarballs"""105    with osutils.TempDir(base_dir=self.tarball_cache.staging_dir) as tempdir:106      local_path = os.path.join(tempdir, os.path.basename(url))107      Log('SDK: Fetching %s', url, silent=self.silent)108      self.gs_ctx.Copy(url, tempdir, debug_level=logging.DEBUG)109      ref.SetDefault(local_path, lock=True)110  def _GetMetadata(self, version):111    """Return metadata (in the form of a dict) for a given version."""112    raw_json = None113    version_base = self._GetVersionGSBase(version)114    with self.misc_cache.Lookup(115        self._GetCacheKeyForComponent(version, constants.METADATA_JSON)) as ref:116      if ref.Exists(lock=True):117        raw_json = osutils.ReadFile(ref.path)118      else:119        metadata_path = os.path.join(version_base, constants.METADATA_JSON)120        partial_metadata_path = os.path.join(version_base,121                                             constants.PARTIAL_METADATA_JSON)122        try:123          raw_json = self.gs_ctx.Cat(metadata_path,124                                     debug_level=logging.DEBUG)125        except gs.GSNoSuchKey:126          logging.info('Could not read %s, falling back to %s',127                       metadata_path, partial_metadata_path)128          raw_json = self.gs_ctx.Cat(partial_metadata_path,129                                     debug_level=logging.DEBUG)130        ref.AssignText(raw_json)131    return json.loads(raw_json)132  def _GetChromeLKGM(self, chrome_src_dir):133    """Get ChromeOS LKGM checked into the Chrome tree.134    Returns:135      Version number in format '3929.0.0'.136    """137    version = osutils.ReadFile(os.path.join(138        chrome_src_dir, constants.PATH_TO_CHROME_LKGM))139    return version140  def _GetRepoCheckoutVersion(self, repo_root):141    """Get the version specified in chromeos_version.sh.142    Returns:143      Version number in format '3929.0.0'.144    """145    chromeos_version_sh = os.path.join(repo_root, constants.VERSION_FILE)146    sourced_env = osutils.SourceEnvironment(147        chromeos_version_sh, ['CHROMEOS_VERSION_STRING'],148        env={'CHROMEOS_OFFICIAL': '1'})149    return sourced_env['CHROMEOS_VERSION_STRING']150  def _GetNewestFullVersion(self, version=None):151    """Gets the full version number of the latest build for the given |version|.152    Args:153      version: The version number or branch to look at. By default, look at154        builds on the current branch.155    Returns:156      Version number in the format 'R30-3929.0.0'.157    """158    if version is None:159      version = git.GetChromiteTrackingBranch()160    version_file = '%s/LATEST-%s' % (self.gs_base, version)161    try:162      full_version = self.gs_ctx.Cat(version_file)163      assert full_version.startswith('R')164      return full_version165    except gs.GSNoSuchKey:166      return None167  def _GetNewestManifestVersion(self):168    """Gets the latest uploaded SDK version.169    Returns:170      Version number in the format '3929.0.0'.171    """172    full_version = self._GetNewestFullVersion()173    return None if full_version is None else full_version.split('-')[1]174  def GetDefaultVersion(self):175    """Get the default SDK version to use.176    If we are in an existing SDK shell, the default version will just be177    the current version. Otherwise, we will try to calculate the178    appropriate version to use based on the checkout.179    """180    if os.environ.get(self.SDK_BOARD_ENV) == self.board:181      sdk_version = os.environ.get(self.SDK_VERSION_ENV)182      if sdk_version is not None:183        return sdk_version184    with self.misc_cache.Lookup((self.board, 'latest')) as ref:185      if ref.Exists(lock=True):186        version = osutils.ReadFile(ref.path).strip()187        # Deal with the old version format.188        if version.startswith('R'):189          version = version.split('-')[1]190        return version191      else:192        return None193  def _SetDefaultVersion(self, version):194    """Set the new default version."""195    with self.misc_cache.Lookup((self.board, 'latest')) as ref:196      ref.AssignText(version)197  def UpdateDefaultVersion(self):198    """Update the version that we default to using.199    Returns:200      A tuple of the form (version, updated), where |version| is the201      version number in the format '3929.0.0', and |updated| indicates202      whether the version was indeed updated.203    """204    checkout_dir = self.chrome_src if self.chrome_src else os.getcwd()205    checkout = path_util.DetermineCheckout(checkout_dir)206    current = self.GetDefaultVersion() or '0'207    if checkout.chrome_src_dir:208      target = self._GetChromeLKGM(checkout.chrome_src_dir)209    elif checkout.type == path_util.CHECKOUT_TYPE_REPO:210      target = self._GetRepoCheckoutVersion(checkout.root)211      if target != current:212        lv_cls = distutils.version.LooseVersion213        if lv_cls(target) > lv_cls(current):214          # Hit the network for the newest uploaded version for the branch.215          newest = self._GetNewestManifestVersion()216          # The SDK for the version of the checkout has not been uploaded yet,217          # so fall back to the latest uploaded SDK.218          if newest is not None and lv_cls(target) > lv_cls(newest):219            target = newest220    else:221      target = self._GetNewestManifestVersion()222    if target is None:223      raise MissingSDK(self.board)224    self._SetDefaultVersion(target)225    return target, target != current226  def GetFullVersion(self, version):227    """Add the release branch and build number to a ChromeOS platform version.228    This will specify where you can get the latest build for the given version229    for the current board.230    Args:231      version: A ChromeOS platform number of the form XXXX.XX.XX, i.e.,232        3918.0.0.233    Returns:234      The version with release branch and build number added, as needed. E.g.235      R28-3918.0.0-b1234.236    """237    assert not version.startswith('R')238    with self.misc_cache.Lookup(('full-version', self.board, version)) as ref:239      if ref.Exists(lock=True):240        return osutils.ReadFile(ref.path).strip()241      else:242        # Find out the newest version from the LATEST (or LATEST-%s) file.243        full_version = self._GetNewestFullVersion(version=version)244        if full_version is None:245          raise MissingSDK(self.board, version)246        ref.AssignText(full_version)247        return full_version248  def _GetVersionGSBase(self, version):249    """The base path of the SDK for a particular version."""250    if self.sdk_path is not None:251      return self.sdk_path252    full_version = self.GetFullVersion(version)253    return os.path.join(self.gs_base, full_version)254  def _GetCacheKeyForComponent(self, version, component):255    """Builds the cache key tuple for an SDK component."""256    version_section = version257    if self.sdk_path is not None:258      version_section = self.sdk_path.replace('/', '__').replace(':', '__')259    return (self.board, version_section, component)260  @contextlib.contextmanager261  def Prepare(self, components, version=None, target_tc=None,262              toolchain_url=None):263    """Ensures the components of an SDK exist and are read-locked.264    For a given SDK version, pulls down missing components, and provides a265    context where the components are read-locked, which prevents the cache from266    deleting them during its purge operations.267    If both target_tc and toolchain_url arguments are provided, then this268    does not download metadata.json for the given version. Otherwise, this269    function requires metadata.json for the given version to exist.270    Args:271      gs_ctx: GSContext object.272      components: A list of specific components(tarballs) to prepare.273      version: The version to prepare.  If not set, uses the version returned by274        GetDefaultVersion().  If there is no default version set (this is the275        first time we are being executed), then we update the default version.276      target_tc: Target toolchain name to use, e.g. x86_64-cros-linux-gnu277      toolchain_url: Format pattern for path to fetch toolchain from,278        e.g. 2014/04/%(target)s-2014.04.23.220740.tar.xz279    Yields:280      An SDKFetcher.SDKContext namedtuple object.  The attributes of the281      object are:282        version: The version that was prepared.283        target_tc: Target toolchain name.284        key_map: Dictionary that contains CacheReference objects for the SDK285          artifacts, indexed by cache key.286    """287    if version is None and self.sdk_path is None:288      version = self.GetDefaultVersion()289      if version is None:290        version, _ = self.UpdateDefaultVersion()291    components = list(components)292    key_map = {}293    fetch_urls = {}294    if not target_tc or not toolchain_url:295      metadata = self._GetMetadata(version)296      target_tc = target_tc or metadata['toolchain-tuple'][0]297      toolchain_url = toolchain_url or metadata['toolchain-url']298    # Fetch toolchains from separate location.299    if self.TARGET_TOOLCHAIN_KEY in components:300      fetch_urls[self.TARGET_TOOLCHAIN_KEY] = os.path.join(301          self.toolchain_path, toolchain_url % {'target': target_tc})302      components.remove(self.TARGET_TOOLCHAIN_KEY)303    version_base = self._GetVersionGSBase(version)304    fetch_urls.update((t, os.path.join(version_base, t)) for t in components)305    try:306      for key, url in fetch_urls.iteritems():307        cache_key = self._GetCacheKeyForComponent(version, key)308        ref = self.tarball_cache.Lookup(cache_key)309        key_map[key] = ref310        ref.Acquire()311        if not ref.Exists(lock=True):312          # TODO(rcui): Parallelize this.  Requires acquiring locks *before*313          # generating worker processes; therefore the functionality needs to314          # be moved into the DiskCache class itself -315          # i.e.,DiskCache.ParallelSetDefault().316          self._UpdateTarball(url, ref)317      ctx_version = version318      if self.sdk_path is not None:319        ctx_version = CUSTOM_VERSION320      yield self.SDKContext(ctx_version, target_tc, key_map)321    finally:322      # TODO(rcui): Move to using cros_build_lib.ContextManagerStack()323      cros_build_lib.SafeRun([ref.Release for ref in key_map.itervalues()])324class GomaError(Exception):325  """Indicates error with setting up Goma."""326@command.CommandDecorator(COMMAND_NAME)327class ChromeSDKCommand(command.CliCommand):328  """Set up an environment for building Chrome on Chrome OS.329  Pulls down SDK components for building and testing Chrome for Chrome OS,330  sets up the environment for building Chrome, and runs a command in the331  environment, starting a bash session if no command is specified.332  The bash session environment is set up by a user-configurable rc file located333  at ~/.chromite/chrome_sdk.bashrc.334  """335  # Note, this URL is not accessible outside of corp.336  _GOMA_URL = ('https://clients5.google.com/cxx-compiler-service/'337               'download/goma_ctl.py')338  _CHROME_CLANG_DIR = 'third_party/llvm-build/Release+Asserts/bin'339  _HOST_BINUTILS_DIR = 'third_party/binutils/Linux_x64/Release/bin/'340  EBUILD_ENV = (341      # Compiler tools.342      'CXX',343      'CC',344      'AR',345      'AS',346      'LD',347      'RANLIB',348      # Compiler flags.349      'CFLAGS',350      'CXXFLAGS',351      'CPPFLAGS',352      'LDFLAGS',353      # Misc settings.354      'GN_ARGS',355      'GOLD_SET',356      'GYP_DEFINES',357  )358  SDK_GOMA_PORT_ENV = 'SDK_GOMA_PORT'359  SDK_GOMA_DIR_ENV = 'SDK_GOMA_DIR'360  GOMACC_PORT_CMD = ['./gomacc', 'port']361  FETCH_GOMA_CMD = ['wget', _GOMA_URL]362  # Override base class property to enable stats upload.363  upload_stats = True364  # Override base class property to use cache related commandline options.365  use_caching_options = True366  @property367  def upload_stats_timeout(self):368    # Give a longer timeout for interactive SDK shell invocations, since the369    # user will not notice a longer wait because it's happening in the370    # background.371    if self.options.cmd:372      return super(ChromeSDKCommand, self).upload_stats_timeout373    else:374      return stats.StatsUploader.UPLOAD_TIMEOUT375  @staticmethod376  def ValidateVersion(version):377    if version.startswith('R') or len(version.split('.')) != 3:378      raise argparse.ArgumentTypeError(379          '--version should be in the format 3912.0.0')380    return version381  @classmethod382  def AddParser(cls, parser):383    super(ChromeSDKCommand, cls).AddParser(parser)384    parser.add_argument(385        '--board', required=True, help='The board SDK to use.')386    parser.add_argument(387        '--bashrc', type='path',388        default=constants.CHROME_SDK_BASHRC,389        help='A bashrc file used to set up the SDK shell environment. '390             'Defaults to %s.' % constants.CHROME_SDK_BASHRC)391    parser.add_argument(392        '--chroot', type='path',393        help='Path to a ChromeOS chroot to use. If set, '394             '<chroot>/build/<board> will be used as the sysroot that Chrome '395             'is built against. If chromeos-chrome was built, the build '396             'environment from the chroot will also be used. The version shown '397             'in the SDK shell prompt will have an asterisk prepended to it.')398    parser.add_argument(399        '--chrome-src', type='path',400        help='Specifies the location of a Chrome src/ directory.  Required if '401             'running with --clang if not running from a Chrome checkout.')402    parser.add_argument(403        '--clang', action='store_true', default=False,404        help='Sets up the environment for building with clang.')405    parser.add_argument(406        '--cwd', type='path',407        help='Specifies a directory to switch to after setting up the SDK '408             'shell.  Defaults to the current directory.')409    parser.add_argument(410        '--internal', action='store_true', default=False,411        help='Sets up SDK for building official (internal) Chrome '412             'Chrome, rather than Chromium.')413    parser.add_argument(414        '--component', action='store_true', default=False,415        help='Sets up SDK for building a componentized build of Chrome '416             '(component=shared_library in GYP).')417    parser.add_argument(418        '--fastbuild', action='store_true', default=False,419        help='Turn off debugging information for a faster build '420             '(fastbuild=1 in GYP).')421    parser.add_argument(422        '--use-external-config', action='store_true', default=False,423        help='Use the external configuration for the specified board, even if '424             'an internal configuration is avalable.')425    parser.add_argument(426        '--sdk-path', type='local_or_gs_path',427        help='Provides a path, whether a local directory or a gs:// path, to '428             'pull SDK components from.')429    parser.add_argument(430        '--toolchain-path', type='local_or_gs_path',431        help='Provides a path, whether a local directory or a gs:// path, to '432             'pull toolchain components from.')433    parser.add_argument(434        '--nogoma', action='store_false', default=True, dest='goma',435        help="Disables Goma in the shell by removing it from the PATH.")436    parser.add_argument(437        '--gomadir', type='path',438        help="Use the goma installation at the specified PATH.")439    parser.add_argument(440        '--version', default=None, type=cls.ValidateVersion,441        help="Specify version of SDK to use, in the format '3912.0.0'.  "442             "Defaults to determining version based on the type of checkout "443             "(Chrome or ChromeOS) you are executing from.")444    parser.add_argument(445        'cmd', nargs='*', default=None,446        help='The command to execute in the SDK environment.  Defaults to '447             'starting a bash shell.')448    parser.add_option_to_group(449        parser.caching_group, '--clear-sdk-cache', action='store_true',450        default=False,451        help='Removes everything in the SDK cache before starting.')452    group = parser.add_option_group(453        'Metadata Overrides (Advanced)',454        description='Provide all of these overrides in order to remove '455                    'dependencies on metadata.json existence.')456    parser.add_option_to_group(457        group, '--target-tc', action='store', default=None,458        help='Override target toolchain name, e.g. x86_64-cros-linux-gnu')459    parser.add_option_to_group(460        group, '--toolchain-url', action='store', default=None,461        help='Override toolchain url format pattern, e.g. '462             '2014/04/%%(target)s-2014.04.23.220740.tar.xz')463  def __init__(self, options):464    super(ChromeSDKCommand, self).__init__(options)465    self.board = options.board466    # Lazy initialized.467    self.sdk = None468    # Initialized later based on options passed in.469    self.silent = True470  @staticmethod471  def _CreatePS1(board, version, chroot=None):472    """Returns PS1 string that sets commandline and xterm window caption.473    If a chroot path is set, then indicate we are using the sysroot from there474    instead of the stock sysroot by prepending an asterisk to the version.475    Args:476      board: The SDK board.477      version: The SDK version.478      chroot: The path to the chroot, if set.479    """480    custom = '*' if chroot else ''481    current_ps1 = cros_build_lib.RunCommand(482        ['bash', '-l', '-c', 'echo "$PS1"'], print_cmd=False,483        capture_output=True).output.splitlines()484    if current_ps1:485      current_ps1 = current_ps1[-1]486    if not current_ps1:487      # Something went wrong, so use a fallback value.488      current_ps1 = r'\u@\h \w $ '489    return '(sdk %s %s%s) %s' % (board, custom, version, current_ps1)490  def _FixGoldPath(self, var_contents, toolchain_path):491    """Point to the gold linker in the toolchain tarball.492    Accepts an already set environment variable in the form of '<cmd>493    -B<gold_path>', and overrides the gold_path to the correct path in the494    extracted toolchain tarball.495    Args:496      var_contents: The contents of the environment variable.497      toolchain_path: Path to the extracted toolchain tarball contents.498    Returns:499      Environment string that has correct gold path.500    """501    cmd, _, gold_path = var_contents.partition(' -B')502    gold_path = os.path.join(toolchain_path, gold_path.lstrip('/'))503    return '%s -B%s' % (cmd, gold_path)504  def _SetupTCEnvironment(self, sdk_ctx, options, env):505    """Sets up toolchain-related environment variables."""506    target_tc_path = sdk_ctx.key_map[self.sdk.TARGET_TOOLCHAIN_KEY].path507    tc_bin_path = os.path.join(target_tc_path, 'bin')508    env['PATH'] = '%s:%s' % (tc_bin_path, os.environ['PATH'])509    for var in ('CXX', 'CC', 'LD'):510      env[var] = self._FixGoldPath(env[var], target_tc_path)511    chrome_clang_path = os.path.join(options.chrome_src, self._CHROME_CLANG_DIR)512    if options.clang:513      clang_flags = ['-Wno-unknown-warning-option']514      env['CC'] = ' '.join([sdk_ctx.target_tc + '-clang'] +515                           env['CC'].split()[1:] + clang_flags)516      env['CXX'] = ' '.join([sdk_ctx.target_tc + '-clang++'] +517                            env['CXX'].split()[1:] + clang_flags)518      env['LD'] = env['CXX']519    # For host compiler, we use the compiler that comes with Chrome520    # instead of the target compiler.521    env['CC_host'] = os.path.join(chrome_clang_path, 'clang')522    env['CXX_host'] = os.path.join(chrome_clang_path, 'clang++')523    env['LD_host'] = env['CXX_host']524    binutils_path = os.path.join(options.chrome_src, self._HOST_BINUTILS_DIR)525    env['AR_host'] = os.path.join(binutils_path, 'ar')526    if not options.fastbuild:527      # Enable debug fission for GYP. linux_use_debug_fission cannot be used528      # because it doesn't have effect on Release builds.529      # TODO: Get rid of this once we remove support for GYP.530      env['CFLAGS'] = env.get('CFLAGS', '') +  ' -gsplit-dwarf'531      env['CXXFLAGS'] = env.get('CXXFLAGS', '') + ' -gsplit-dwarf'532  def _SetupEnvironment(self, board, sdk_ctx, options, goma_dir=None,533                        goma_port=None):534    """Sets environment variables to export to the SDK shell."""535    if options.chroot:536      sysroot = os.path.join(options.chroot, 'build', board)537      if not os.path.isdir(sysroot) and not options.cmd:538        logging.warning("Because --chroot is set, expected a sysroot to be at "539                        "%s, but couldn't find one.", sysroot)540    else:541      sysroot = sdk_ctx.key_map[constants.CHROME_SYSROOT_TAR].path542    environment = os.path.join(sdk_ctx.key_map[constants.CHROME_ENV_TAR].path,543                               'environment')544    if options.chroot:545      # Override with the environment from the chroot if available (i.e.546      # build_packages or emerge chromeos-chrome has been run for |board|).547      env_path = os.path.join(sysroot, 'var', 'db', 'pkg', 'chromeos-base',548                              'chromeos-chrome-*')549      env_glob = glob.glob(env_path)550      if len(env_glob) != 1:551        logging.warning('Multiple Chrome versions in %s. This can be resolved'552                        ' by running "eclean-$BOARD -d packages". Using'553                        ' environment from: %s', env_path, environment)554      elif not os.path.isdir(env_glob[0]):555        logging.warning('Environment path not found: %s. Using enviroment from:'556                        ' %s.', env_path, environment)557      else:558        chroot_env_file = os.path.join(env_glob[0], 'environment.bz2')559        if os.path.isfile(chroot_env_file):560          # Log a warning here since this is new behavior that is not obvious.561          logging.notice('Environment fetched from: %s', chroot_env_file)562          # Uncompress enviornment.bz2 to pass to osutils.SourceEnvironment.563          chroot_cache = os.path.join(564              self.options.cache_dir, COMMAND_NAME, 'chroot')565          osutils.SafeMakedirs(chroot_cache)566          environment = os.path.join(chroot_cache, 'environment_%s' % board)567          cros_build_lib.UncompressFile(chroot_env_file, environment)568    env = osutils.SourceEnvironment(environment, self.EBUILD_ENV)569    self._SetupTCEnvironment(sdk_ctx, options, env)570    # Add managed components to the PATH.571    env['PATH'] = '%s:%s' % (constants.CHROMITE_BIN_DIR, env['PATH'])572    env['PATH'] = '%s:%s' % (os.path.dirname(self.sdk.gs_ctx.gsutil_bin),573                             env['PATH'])574    # Export internally referenced variables.575    os.environ[self.sdk.SDK_BOARD_ENV] = board576    if self.options.sdk_path:577      os.environ[self.sdk.SDK_PATH_ENV] = self.options.sdk_path578    os.environ[self.sdk.SDK_VERSION_ENV] = sdk_ctx.version579    # Export the board/version info in a more accessible way, so developers can580    # reference them in their chrome_sdk.bashrc files, as well as within the581    # chrome-sdk shell.582    for var in [self.sdk.SDK_VERSION_ENV, self.sdk.SDK_BOARD_ENV]:583      env[var.lstrip('%')] = os.environ[var]584    # Export Goma information.585    if goma_dir:586      env[self.SDK_GOMA_DIR_ENV] = goma_dir587      env[self.SDK_GOMA_PORT_ENV] = goma_port588    # SYSROOT is necessary for Goma and the sysroot wrapper.589    env['SYSROOT'] = sysroot590    gyp_dict = chrome_util.ProcessGypDefines(env['GYP_DEFINES'])591    gn_args = gn_helpers.FromGNArgs(env['GN_ARGS'])592    gyp_dict['sysroot'] = sysroot593    gn_args['target_sysroot'] = sysroot594    gyp_dict.pop('pkg-config', None)595    gn_args.pop('pkg_config', None)596    if options.clang:597      gyp_dict['clang'] = 1598      gn_args['is_clang'] = True599    if options.internal:600      gyp_dict['branding'] = 'Chrome'601      gn_args['is_chrome_branded'] = True602      gyp_dict['buildtype'] = 'Official'603      gn_args['is_official_build'] = True604    else:605      gyp_dict.pop('branding', None)606      gn_args.pop('is_chrome_branded', None)607      gyp_dict.pop('buildtype', None)608      gn_args.pop('is_official_build', None)609      gyp_dict.pop('internal_gles2_conform_tests', None)610      gn_args.pop('internal_gles2_conform_tests', None)611    if options.component:612      gyp_dict['component'] = 'shared_library'613      gn_args['is_component_build'] = True614    if options.fastbuild:615      gyp_dict['fastbuild'] = 1616      gyp_dict.pop('release_extra_cflags', None)617      # symbol_level corresponds to GYP's fastbuild (https://goo.gl/ZC4fUO).618      gn_args['symbol_level'] = 1619    else:620      # Enable debug fission for GN.621      gn_args['use_debug_fission'] = True622    # For SimpleChrome, we use the binutils that comes bundled within Chrome.623    # We should not use the binutils from the host system.624    gn_args['linux_use_bundled_binutils'] = True625    gyp_dict['host_clang'] = 1626    # Need to reset these after the env vars have been fixed by627    # _SetupTCEnvironment.628    gn_args['cros_host_is_clang'] = True629    # v8 snapshot is built on the host, so we need to set this.630    # See crosbug/618346.631    gn_args['cros_v8_snapshot_is_clang'] = True632    #633    gn_args['cros_target_cc'] = env['CC']634    gn_args['cros_target_cxx'] = env['CXX']635    gn_args['cros_target_ld'] = env['LD']636    # We need to reset extra C/CXX flags to remove references to637    # EBUILD_CFLAGS, EBUILD_CXXFLAGS. We also need to remove redundant638    # -gsplit-dwarf (only needed while we still support GYP).639    gn_args['cros_target_extra_cflags'] = env['CFLAGS'].replace(640        '-gsplit-dwarf', '')641    gn_args['cros_target_extra_cxxflags'] = env['CXXFLAGS'].replace(642        '-gsplit-dwarf', '')643    gn_args['cros_host_cc'] = env['CC_host']644    gn_args['cros_host_cxx'] = env['CXX_host']645    gn_args['cros_host_ld'] = env['LD_host']646    gn_args['cros_host_ar'] = env['AR_host']647    gn_args['cros_v8_snapshot_cc'] = env['CC_host']648    gn_args['cros_v8_snapshot_cxx'] = env['CXX_host']649    gn_args['cros_v8_snapshot_ld'] = env['LD_host']650    gn_args['cros_v8_snapshot_ar'] = env['AR_host']651    # No need to adjust CFLAGS and CXXFLAGS for GN since the only652    # adjustment made in _SetupTCEnvironment is for split debug which653    # is done with 'use_debug_fission'.654    # Enable goma if requested.655    if goma_dir:656      gyp_dict['use_goma'] = 1657      gn_args['use_goma'] = True658      gyp_dict['gomadir'] = goma_dir659      gn_args['goma_dir'] = goma_dir660    gn_args.pop('internal_khronos_glcts_tests', None)  # crbug.com/588080661    env['GYP_DEFINES'] = chrome_util.DictToGypDefines(gyp_dict)662    env['GN_ARGS'] = gn_helpers.ToGNString(gn_args)663    # PS1 sets the command line prompt and xterm window caption.664    full_version = sdk_ctx.version665    if full_version != CUSTOM_VERSION:666      full_version = self.sdk.GetFullVersion(sdk_ctx.version)667    env['PS1'] = self._CreatePS1(self.board, full_version,668                                 chroot=options.chroot)669    out_dir = 'out_%s' % self.board670    env['builddir_name'] = out_dir671    env['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir672    env['GYP_CROSSCOMPILE'] = '1'673    # SimpleChrome now only supports GN by default. deploy_chrome relies on674    # the 'gn' USE flag to locate .so (and potentially other) files.675    useflags = set(os.environ.get('USE', '').split() + ['gn'])676    useflags.discard('-gn')677    env['USE'] = ' '.join(useflags)678    return env679  @staticmethod680  def _VerifyGoma(user_rc):681    """Verify that the user has no goma installations set up in user_rc.682    If the user does have a goma installation set up, verify that it's for683    ChromeOS.684    Args:685      user_rc: User-supplied rc file.686    """687    user_env = osutils.SourceEnvironment(user_rc, ['PATH'])688    goma_ctl = osutils.Which('goma_ctl.py', user_env.get('PATH'))689    if goma_ctl is not None:690      logging.warning(691          '%s is adding Goma to the PATH.  Using that Goma instead of the '692          'managed Goma install.', user_rc)693  @staticmethod694  def _VerifyChromiteBin(user_rc):695    """Verify that the user has not set a chromite bin/ dir in user_rc.696    Args:697      user_rc: User-supplied rc file.698    """699    user_env = osutils.SourceEnvironment(user_rc, ['PATH'])700    chromite_bin = osutils.Which('parallel_emerge', user_env.get('PATH'))701    if chromite_bin is not None:702      logging.warning(703          '%s is adding chromite/bin to the PATH.  Remove it from the PATH to '704          'use the the default Chromite.', user_rc)705  @contextlib.contextmanager706  def _GetRCFile(self, env, user_rc):707    """Returns path to dynamically created bashrc file.708    The bashrc file sets the environment variables contained in |env|, as well709    as sources the user-editable chrome_sdk.bashrc file in the user's home710    directory.  That rc file is created if it doesn't already exist.711    Args:712      env: A dictionary of environment variables that will be set by the rc713        file.714      user_rc: User-supplied rc file.715    """716    if not os.path.exists(user_rc):717      osutils.Touch(user_rc, makedirs=True)718    self._VerifyGoma(user_rc)719    self._VerifyChromiteBin(user_rc)720    # We need a temporary rc file to 'wrap' the user configuration file,721    # because running with '--rcfile' causes bash to ignore bash special722    # variables passed through subprocess.Popen, such as PS1.  So we set them723    # here.724    #725    # Having a wrapper rc file will also allow us to inject bash functions into726    # the environment, not just variables.727    with osutils.TempDir() as tempdir:728      # Only source the user's ~/.bashrc if running in interactive mode.729      contents = [730          '[[ -e ~/.bashrc && $- == *i* ]] && . ~/.bashrc\n',731      ]732      for key, value in env.iteritems():733        contents.append("export %s='%s'\n" % (key, value))734      contents.append('. "%s"\n' % user_rc)735      rc_file = os.path.join(tempdir, 'rcfile')736      osutils.WriteFile(rc_file, contents)737      yield rc_file738  def _GomaPort(self, goma_dir):739    """Returns current active Goma port."""740    port = cros_build_lib.RunCommand(741        self.GOMACC_PORT_CMD, cwd=goma_dir, debug_level=logging.DEBUG,742        error_code_ok=True, capture_output=True).output.strip()743    return port744  def _FetchGoma(self):745    """Fetch, install, and start Goma, using cached version if it exists.746    Returns:747      A tuple (dir, port) containing the path to the cached goma/ dir and the748      Goma port.749    """750    common_path = os.path.join(self.options.cache_dir, constants.COMMON_CACHE)751    common_cache = cache.DiskCache(common_path)752    goma_dir = self.options.gomadir753    if not goma_dir:754      ref = common_cache.Lookup(('goma', '2'))755      if not ref.Exists():756        Log('Installing Goma.', silent=self.silent)757        with osutils.TempDir() as tempdir:758          goma_dir = os.path.join(tempdir, 'goma')759          os.mkdir(goma_dir)760          result = cros_build_lib.DebugRunCommand(761              self.FETCH_GOMA_CMD, cwd=goma_dir, error_code_ok=True)762          if result.returncode:763            raise GomaError('Failed to fetch Goma')764         # Update to latest version of goma. We choose the outside-chroot765         # version ('goobuntu') over the chroot version ('chromeos') by766         # supplying input='1' to the following prompt:767         #768         # What is your platform?769         #  1. Goobuntu  2. Precise (32bit)  3. Lucid (32bit)  4. Debian770         #  5. Chrome OS  6. MacOS ? -->771          cros_build_lib.DebugRunCommand(772              ['python2', 'goma_ctl.py', 'update'], cwd=goma_dir, input='1\n')773          ref.SetDefault(goma_dir)774      goma_dir = ref.path775    Log('Starting Goma.', silent=self.silent)776    cros_build_lib.DebugRunCommand(777        ['python2', 'goma_ctl.py', 'ensure_start'], cwd=goma_dir)778    port = self._GomaPort(goma_dir)779    Log('Goma is started on port %s', port, silent=self.silent)780    if not port:781      raise GomaError('No Goma port detected')782    return goma_dir, port783  def Run(self):784    """Perform the command."""785    if os.environ.get(SDKFetcher.SDK_VERSION_ENV) is not None:786      cros_build_lib.Die('Already in an SDK shell.')787    src_path = self.options.chrome_src or os.getcwd()788    checkout = path_util.DetermineCheckout(src_path)789    if not checkout.chrome_src_dir:790      cros_build_lib.Die('Chrome checkout not found at %s', src_path)791    self.options.chrome_src = checkout.chrome_src_dir792    if self.options.clang and not self.options.chrome_src:793      cros_build_lib.Die('--clang requires --chrome-src to be set.')794    if self.options.version and self.options.sdk_path:795      cros_build_lib.Die('Cannot specify both --version and --sdk-path.')796    self.silent = bool(self.options.cmd)797    # Lazy initialize because SDKFetcher creates a GSContext() object in its798    # constructor, which may block on user input.799    self.sdk = SDKFetcher(self.options.cache_dir, self.options.board,800                          clear_cache=self.options.clear_sdk_cache,801                          chrome_src=self.options.chrome_src,802                          sdk_path=self.options.sdk_path,803                          toolchain_path=self.options.toolchain_path,804                          silent=self.silent,805                          use_external_config=self.options.use_external_config)806    prepare_version = self.options.version807    if not prepare_version and not self.options.sdk_path:808      prepare_version, _ = self.sdk.UpdateDefaultVersion()809    components = [self.sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR]810    if not self.options.chroot:811      components.append(constants.CHROME_SYSROOT_TAR)812    goma_dir = None813    goma_port = None814    if self.options.goma:815      try:816        goma_dir, goma_port = self._FetchGoma()817      except GomaError as e:818        logging.error('Goma: %s.  Bypass by running with --nogoma.', e)819    with self.sdk.Prepare(components, version=prepare_version,820                          target_tc=self.options.target_tc,821                          toolchain_url=self.options.toolchain_url) as ctx:822      env = self._SetupEnvironment(self.options.board, ctx, self.options,823                                   goma_dir=goma_dir, goma_port=goma_port)824      with self._GetRCFile(env, self.options.bashrc) as rcfile:825        bash_cmd = ['/bin/bash']826        extra_env = None827        if not self.options.cmd:828          bash_cmd.extend(['--rcfile', rcfile, '-i'])829        else:830          # The '"$@"' expands out to the properly quoted positional args831          # coming after the '--'.832          bash_cmd.extend(['-c', '"$@"', '--'])833          bash_cmd.extend(self.options.cmd)834          # When run in noninteractive mode, bash sources the rc file set in835          # BASH_ENV, and ignores the --rcfile flag.836          extra_env = {'BASH_ENV': rcfile}837        # Bash behaves differently when it detects that it's being launched by838        # sshd - it ignores the BASH_ENV variable.  So prevent ssh-related839        # environment variables from being passed through.840        os.environ.pop('SSH_CLIENT', None)841        os.environ.pop('SSH_CONNECTION', None)842        os.environ.pop('SSH_TTY', None)843        cmd_result = cros_build_lib.RunCommand(844            bash_cmd, print_cmd=False, debug_level=logging.CRITICAL,845            error_code_ok=True, extra_env=extra_env, cwd=self.options.cwd)846        if self.options.cmd:...
Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.