How to use _collect_artifacts method in yandex-tank

Best Python code snippet using yandex-tank

tulsi_aspects.bzl

Source:tulsi_aspects.bzl

...181      src=f.src,182      root=root_path,183      is_dir=new_is_dir184  )185def _collect_artifacts(obj, attr_path):186  """Returns a list of Artifact objects for the attr_path in obj."""187  return [f for src in _getattr_as_list(obj, attr_path)188          for f in _get_opt_attr(src, 'files')]189def _collect_files(obj, attr_path):190  """Returns a list of artifact_location's for the attr_path in obj."""191  return [_file_metadata(f) for f in _collect_artifacts(obj, attr_path)]192def _collect_first_file(obj, attr_path):193  """Returns a the first artifact_location for the attr_path in obj."""194  files = _collect_files(obj, attr_path)195  if not files:196    return None197  return files[0]198def _collect_supporting_files(rule_attr):199  """Extracts 'supporting' files from the given rule attributes."""200  all_files = []201  for attr in _SUPPORTING_FILE_ATTRIBUTES:202    all_files += _collect_files(rule_attr, attr)203  return all_files204def _collect_bundle_paths(rule_attr, bundle_attributes, bundle_ext):205  """Extracts subpaths with the given bundle_ext for the given attributes."""206  discovered_paths = set()207  bundles = []208  if not bundle_ext.endswith('/'):209    bundle_ext += '/'210  bundle_ext_len = len(bundle_ext) - 1211  for attr in bundle_attributes:212    for f in _collect_files(rule_attr, attr):213      end = f.path.find(bundle_ext)214      if end < 0:215        continue216      end += bundle_ext_len217      path = f.path[:end]218      root_path = _get_opt_attr(f, 'rootPath')219      full_path = str(root_path) + ':' + path220      if full_path in discovered_paths:221        continue222      discovered_paths += [full_path]223      # Generally Xcode treats bundles as special files so they should not be224      # flagged as directories.225      bundles.append(_file_metadata_by_replacing_path(f, path, False))226  return bundles227def _collect_asset_catalogs(rule_attr):228  """Extracts xcassets directories from the given rule attributes."""229  return _collect_bundle_paths(rule_attr,230                               ['app_asset_catalogs', 'asset_catalogs'],231                               '.xcassets')232def _collect_bundle_imports(rule_attr):233  """Extracts bundle directories from the given rule attributes."""234  return _collect_bundle_paths(rule_attr,235                               ['bundle_imports', 'settings_bundle'],236                               '.bundle')237def _collect_framework_imports(rule_attr):238  """Extracts framework directories from the given rule attributes."""239  return _collect_bundle_paths(rule_attr,240                               ['framework_imports'],241                               '.framework')242def _collect_xcdatamodeld_files(obj, attr_path):243  """Returns artifact_location's for xcdatamodeld's for attr_path in obj."""244  files = _collect_files(obj, attr_path)245  if not files:246    return []247  discovered_paths = set()248  datamodelds = []249  for f in files:250    end = f.path.find('.xcdatamodel/')251    if end < 0:252      continue253    end += 12254    path = f.path[:end]255    root_path = _get_opt_attr(f, 'rootPath')256    full_path = str(root_path) + ':' + path257    if full_path in discovered_paths:258      continue259    discovered_paths += [full_path]260    datamodelds.append(_file_metadata_by_replacing_path(f, path, False))261  return datamodelds262def _collect_dependency_labels(rule, attr_list):263  """Collects Bazel labels for a list of dependency attributes.264  Args:265    rule: The Bazel rule whose dependencies should be collected.266    attr_list: List of attribute names potentially containing Bazel labels for267        dependencies of the given rule.268  Returns:269    A list of the Bazel labels of dependencies of the given rule.270  """271  rule_attrs = rule.attr272  deps = [dep273          for attribute in attr_list274          for dep in _getattr_as_list(rule_attrs, attribute)]275  return [dep.label for dep in deps if hasattr(dep, 'label')]276def _get_opt_attr(obj, attr_path):277  """Returns the value at attr_path on the given object if it is set."""278  attr_path = attr_path.split('.')279  for a in attr_path:280    if not obj or not hasattr(obj, a):281      return None282    obj = getattr(obj, a)283  return obj284def _get_label_attr(obj, attr_path):285  """Returns the value at attr_path as a label string if it is set."""286  label = _get_opt_attr(obj, attr_path)287  return str(label) if label else None288def _getattr_as_list(obj, attr_path):289  """Returns the value at attr_path as a list.290  This handles normalization of attributes containing a single value for use in291  methods expecting a list of values.292  Args:293    obj: The struct whose attributes should be parsed.294    attr_path: Dotted path of attributes whose value should be returned in295        list form.296  Returns:297    A list of values for obj at attr_path or [] if the struct has298    no such attribute.299  """300  val = _get_opt_attr(obj, attr_path)301  if not val:302    return []303  if type(val) == 'list':304    return val305  return [val]306def _extract_defines_from_option_list(lst):307  """Extracts preprocessor defines from a list of -D strings."""308  defines = []309  for item in lst:310    if item.startswith('-D'):311      defines.append(item[2:])312  return defines313def _extract_compiler_defines(ctx):314  """Extracts preprocessor defines from compiler fragments."""315  defines = []316  cpp_fragment = _get_opt_attr(ctx.fragments, 'cpp')317  if cpp_fragment:318    c_options = _get_opt_attr(cpp_fragment, 'c_options')319    defines += _extract_defines_from_option_list(c_options)320    compiler_options = cpp_fragment.compiler_options([])321    defines += _extract_defines_from_option_list(compiler_options)322    unfiltered = cpp_fragment.unfiltered_compiler_options([])323    defines += _extract_defines_from_option_list(unfiltered)324    cxx = cpp_fragment.cxx_options([])325    defines += _extract_defines_from_option_list(cxx)326  objc_fragment = _get_opt_attr(ctx.fragments, 'objc')327  if objc_fragment:328    objc_copts = _get_opt_attr(objc_fragment, 'copts')329    defines += _extract_defines_from_option_list(objc_copts)330  return defines331def _collect_secondary_artifacts(target, ctx):332  """Returns a list of file metadatas for implicit outputs of 'rule'."""333  artifacts = []334  rule = ctx.rule335  if rule.kind in _MERGEDINFOPLIST_GENERATING_RULES:336    bin_dir = _convert_outpath_to_symlink_path(ctx.bin_dir.path)337    package = target.label.package338    basename = target.label.name339    artifacts.append(_struct_omitting_none(340        path='%s/%s-MergedInfo.plist' % (package, basename),341        src=False,342        root=bin_dir343    ))344  return artifacts345def _extract_generated_sources_and_includes(target):346  """Returns (source_metadatas, includes) generated by the given target."""347  file_metadatas = []348  includes = []349  objc_provider = _get_opt_attr(target, 'objc')350  if hasattr(objc_provider, 'source') and hasattr(objc_provider, 'header'):351    all_files = set(objc_provider.source)352    all_files += objc_provider.header353    file_metadatas = [_file_metadata(f) for f in all_files]354  if hasattr(objc_provider, 'include'):355    includes = [_convert_outpath_to_symlink_path(x, use_tulsi_symlink=True)356                for x in objc_provider.include]357  return file_metadatas, includes358def _extract_minimum_os_for_platform(ctx, platform):359  """Extracts the minimum OS version for the given apple_common.platform."""360  apple_frag = _get_opt_attr(ctx.fragments, 'apple')361  current_platform_str = _get_opt_attr(ctx, 'rule.attr.platform_type')362  if not current_platform_str:363    current_platform_str = str(apple_frag.single_arch_platform.platform_type)364  # Bazel is changing its API to only provide minimum OS for the current365  # configuration platform type, so return none if the requested platform366  # does not match the current platform.367  if current_platform_str != str(platform):368    return None369  min_os = apple_frag.minimum_os_for_platform_type(platform)370  if not min_os:371    return None372  # Convert the DottedVersion to a string suitable for inclusion in a struct.373  return str(min_os)374def _extract_swift_language_version(ctx):375  """Returns the Swift version set by the xcode_toolchain option for ctx."""376  swift_toolchain = _get_opt_attr(ctx, 'fragments.apple.xcode_toolchain')377  if swift_toolchain == 'com.apple.dt.toolchain.Swift_2_3':378    return ('2.3', swift_toolchain)379  elif swift_toolchain:380    # TODO(abaire): Adjust as necessary once versions > 3.0 come out.381    return ('3.0', swift_toolchain)382  # TODO(abaire): Remove the fallback check for swift_library once383  #               xcode_toolchain is available everywhere.384  if ctx.rule.kind == 'swift_library':385    return ('3.0', 'com.apple.dt.toolchain.XcodeDefault')386  return (None, None)387def _collect_swift_modules(target):388  """Returns a depset of Swift modules found on the given target."""389  swift_modules = depset()390  for modules in _getattr_as_list(target, 'swift.transitive_modules'):391    if type(modules) == 'depset':392      swift_modules += modules393    else:394      # TODO(b/37660812): Older version of swift_library used lists for395      # transitive modules. This branch is here for backwards compatibility.396      swift_modules += [modules]397  return swift_modules398def _collect_module_maps(target):399  """Returns a depset of Clang module maps found on the given target."""400  maps = depset()401  if hasattr(target, 'swift'):402    for module_maps in _getattr_as_list(target, 'objc.module_map'):403      maps += module_maps404  return maps405def _tulsi_sources_aspect(target, ctx):406  """Extracts information from a given rule, emitting it as a JSON struct."""407  rule = ctx.rule408  target_kind = rule.kind409  rule_attr = _get_opt_attr(rule, 'attr')410  tulsi_info_files = set()411  transitive_attributes = dict()412  for attr_name in _TULSI_COMPILE_DEPS:413    deps = _getattr_as_list(rule_attr, attr_name)414    for dep in deps:415      if hasattr(dep, 'tulsi_info_files'):416        tulsi_info_files += dep.tulsi_info_files417      if hasattr(dep, 'transitive_attributes'):418        transitive_attributes += dep.transitive_attributes419  artifacts = _get_opt_attr(target, 'files')420  if artifacts:421    # Ignore any generated Xcode projects as they are not useful to Tulsi.422    artifacts = [_file_metadata(f)423                 for f in artifacts424                 if not f.short_path.endswith('project.pbxproj')]425  else:426    # artifacts may be an empty set type, in which case it must be explicitly427    # set to None to allow Skylark's serialization to work.428    artifacts = None429  srcs = (_collect_files(rule, 'attr.srcs') +430          _collect_files(rule, 'attr.hdrs') +431          _collect_files(rule, 'attr.textual_hdrs'))432  generated_files = []433  generated_non_arc_files = []434  generated_includes = []435  if target_kind in _SOURCE_GENERATING_RULES:436    generated_files, generated_includes = (437        _extract_generated_sources_and_includes(target))438  elif target_kind in _NON_ARC_SOURCE_GENERATING_RULES:439    generated_non_arc_files, generated_includes = (440        _extract_generated_sources_and_includes(target))441  swift_transitive_modules = depset(442      [_file_metadata(f, use_tulsi_symlink=True)443       for f in _collect_swift_modules(target)])444  # Collect ObjC module maps dependencies for Swift targets.445  objc_module_maps = depset(446      [_file_metadata(f, use_tulsi_symlink=True)447       for f in _collect_module_maps(target)])448  # Collect the dependencies of this rule, dropping any .jar files (which may be449  # created as artifacts of java/j2objc rules).450  dep_labels = _collect_dependency_labels(rule, _TULSI_COMPILE_DEPS)451  compile_deps = [str(l) for l in dep_labels if not l.name.endswith('.jar')]452  binary_rule = _get_opt_attr(rule_attr, 'binary')453  if binary_rule and type(binary_rule) == 'list':454    binary_rule = binary_rule[0]455  supporting_files = (_collect_supporting_files(rule_attr) +456                      _collect_asset_catalogs(rule_attr) +457                      _collect_bundle_imports(rule_attr))458  # Keys for attribute and inheritable_attributes keys must be kept in sync459  # with defines in Tulsi's RuleEntry.460  attributes = _dict_omitting_none(461      binary=_get_label_attr(binary_rule, 'label'),462      copts=_get_opt_attr(rule_attr, 'copts'),463      datamodels=_collect_xcdatamodeld_files(rule_attr, 'datamodels'),464      supporting_files=supporting_files,465      xctest=_get_opt_attr(rule_attr, 'xctest'),466      xctest_app=_get_label_attr(rule_attr, 'xctest_app.label'),467      test_host=_get_label_attr(rule_attr, 'test_host.label'),468      test_bundle=_get_label_attr(rule_attr, 'test_bundle.label'),469  )470  # Inheritable attributes are pulled up through dependencies of type 'binary'471  # to simplify handling in Tulsi (so it appears as though bridging_header is472  # defined on an ios_application rather than its associated objc_binary, for473  # example).474  inheritable_attributes = _dict_omitting_none(475      bridging_header=_collect_first_file(rule_attr, 'bridging_header'),476      compiler_defines=_extract_compiler_defines(ctx),477      defines=_getattr_as_list(rule_attr, 'defines'),478      enable_modules=_get_opt_attr(rule_attr, 'enable_modules'),479      includes=_getattr_as_list(rule_attr, 'includes'),480      launch_storyboard=_collect_first_file(rule_attr, 'launch_storyboard'),481      pch=_collect_first_file(rule_attr, 'pch'),482  )483  # Merge any attributes on the "binary" dependency into this container rule.484  binary_attributes = _get_opt_attr(binary_rule, 'inheritable_attributes')485  if binary_attributes:486    inheritable_attributes = binary_attributes + inheritable_attributes487  # TODO(b/35322727): Remove this logic when outputs are discovered using488  # tulsi_outputs_aspect489  ipa_output_label = None490  if target_kind == 'apple_watch2_extension':491    # watch2 extensions need to use the IPA produced for the app_name attribute.492    ipa_name = _get_opt_attr(rule_attr, 'app_name') + '.ipa'493    ipa_output_label = '//' + target.label.package + ':' + ipa_name494  elif target_kind in _IPA_GENERATING_RULES:495    ipa_output_label = str(target.label) + '.ipa'496  extensions = [str(t.label) for t in _getattr_as_list(rule_attr, 'extensions')]497  bundle_id = _get_opt_attr(rule_attr, 'bundle_id')498  if not bundle_id:499    bundle_id = _get_opt_attr(rule_attr, 'app_bundle_id')500  # Build up any local transitive attributes and apply them.501  swift_language_version, swift_toolchain = _extract_swift_language_version(ctx)502  if swift_language_version:503    transitive_attributes['swift_language_version'] = swift_language_version504    transitive_attributes['has_swift_dependency'] = True505  if swift_toolchain:506    transitive_attributes['swift_toolchain'] = swift_toolchain507    transitive_attributes['has_swift_dependency'] = True508  # Collect Info.plist files from an extension to figure out its type.509  infoplist = None510  # Only Skylark versions of ios_extension have the 'apple_bundle' provider.511  if target_kind == 'ios_extension' and hasattr(target, 'apple_bundle'):512    infoplist = target.apple_bundle.infoplist513  all_attributes = attributes + inheritable_attributes + transitive_attributes514  info = _struct_omitting_none(515      artifacts=artifacts,516      attr=_struct_omitting_none(**all_attributes),517      build_file=ctx.build_file_path,518      bundle_id=bundle_id,519      deps=compile_deps,520      ext_bundle_id=_get_opt_attr(rule_attr, 'ext_bundle_id'),521      extensions=extensions,522      framework_imports=_collect_framework_imports(rule_attr),523      generated_files=generated_files,524      generated_non_arc_files=generated_non_arc_files,525      generated_includes=generated_includes,526      ipa_output_label=ipa_output_label,527      iphoneos_deployment_target=_extract_minimum_os_for_platform(528          ctx, apple_common.platform_type.ios),529      # TODO(abaire): Uncomment if/when Bazel supports macOS.530      # macos_deployment_target=_extract_minimum_os_for_platform(531      #     ctx, apple_common.platform_type.macosx),532      tvos_deployment_target=_extract_minimum_os_for_platform(533          ctx, apple_common.platform_type.tvos),534      watchos_deployment_target=_extract_minimum_os_for_platform(535          ctx, apple_common.platform_type.watchos),536      label=str(target.label),537      non_arc_srcs=_collect_files(rule, 'attr.non_arc_srcs'),538      secondary_product_artifacts=_collect_secondary_artifacts(target, ctx),539      srcs=srcs,540      swift_transitive_modules=swift_transitive_modules.to_list(),541      objc_module_maps=list(objc_module_maps),542      type=target_kind,543      infoplist=infoplist.basename if infoplist else None,544  )545  # Create an action to write out this target's info.546  output = ctx.new_file(target.label.name + '.tulsiinfo')547  ctx.file_action(output, info.to_json())548  tulsi_info_files += set([output])549  if infoplist:550    tulsi_info_files += [infoplist]551  return struct(552      # Matches the --output_groups on the bazel commandline.553      output_groups={554          'tulsi-info': tulsi_info_files,555      },556      # The file actions used to save this rule's info and that of all of its557      # transitive dependencies.558      tulsi_info_files=tulsi_info_files,559      # The inheritable attributes of this rule, expressed as a dict instead of560      # a struct to allow easy joining.561      inheritable_attributes=inheritable_attributes,562      # Transitive info that should be applied to every rule that depends on563      # this rule.564      transitive_attributes=transitive_attributes,565  )566def _tulsi_outputs_aspect(target, ctx):567  """Collects outputs of each build invocation."""568  rule = ctx.rule569  target_kind = rule.kind570  rule_attr = _get_opt_attr(rule, 'attr')571  tulsi_generated_files = depset()572  for attr_name in _TULSI_COMPILE_DEPS:573    deps = _getattr_as_list(rule_attr, attr_name)574    for dep in deps:575      if hasattr(dep, 'tulsi_generated_files'):576        tulsi_generated_files += dep.tulsi_generated_files577  # TODO(b/35322727): Move apple_watch2_extension into _IPA_GENERATING_RULES578  # when dynamic outputs is the default strategy and it does need to be579  # special-cased above.580  ipa_output_name = None581  if target_kind == 'apple_watch2_extension':582    # watch2 extensions need to use the IPA produced for the app_name attribute.583    ipa_output_name = _get_opt_attr(target, 'attr.app_name')584  elif target_kind in _IPA_GENERATING_RULES:585    ipa_output_name = target.label.name586  artifacts = [x.path for x in target.files]587  if ipa_output_name:588    # Some targets produce more than one IPA or ZIP (e.g. ios_test will generate589    # two IPAs for the test and host bundles), we want to filter only exact590    # matches to label name.591    # TODO(b/37244852): Use a defined provider to get outputs instead.592    output_ipa = '/%s.ipa' % ipa_output_name593    output_zip = '/%s.zip' % ipa_output_name594    artifacts = [x for x in artifacts if x.endswith(output_ipa) or x.endswith(output_zip)]595  # Collect generated files for bazel_build.py to copy under Tulsi root.596  all_files = depset()597  if target_kind in _SOURCE_GENERATING_RULES + _NON_ARC_SOURCE_GENERATING_RULES:598    objc_provider = _get_opt_attr(target, 'objc')599    if hasattr(objc_provider, 'source') and hasattr(objc_provider, 'header'):600      all_files += objc_provider.source601      all_files += objc_provider.header602  all_files += _collect_swift_modules(target)603  all_files += _collect_module_maps(target)604  all_files += (_collect_artifacts(rule, 'attr.srcs')605                + _collect_artifacts(rule, 'attr.hdrs')606                + _collect_artifacts(rule, 'attr.textual_hdrs'))607  tulsi_generated_files += depset(608      [x for x in all_files.to_list() if not x.is_source])609  info = _struct_omitting_none(610      artifacts=artifacts,611      generated_sources=[(x.path, x.short_path) for x in tulsi_generated_files])612  output = ctx.new_file(target.label.name + '.tulsiouts')613  ctx.file_action(output, info.to_json())614  return struct(615      output_groups={616          'tulsi-outputs': [output],617      },618      tulsi_generated_files=tulsi_generated_files,619  )620tulsi_sources_aspect = aspect(...

pytorch_lightning.py

Source:pytorch_lightning.py

1#!/usr/bin/python2#3# Copyright 2018-2022 Polyaxon, Inc.4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9#      http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16from argparse import Namespace17from typing import Any, Dict, List, Optional, Union18from polyaxon import tracking19from polyaxon.client import RunClient20from traceml.exceptions import TracemlException21try:22    from pytorch_lightning.loggers.base import LightningLoggerBase, rank_zero_experiment23    from pytorch_lightning.utilities import rank_zero_only24    from pytorch_lightning.utilities.logger import (25        _convert_params,26        _flatten_dict,27        _sanitize_callable_params,28    )29except ImportError:30    raise TracemlException("PytorchLightning is required to use the tracking Callback")31class Callback(LightningLoggerBase):32    def __init__(33        self,34        owner: str = None,35        project: str = None,36        run_uuid: str = None,37        client: RunClient = None,38        track_code: bool = True,39        track_env: bool = True,40        refresh_data: bool = False,41        artifacts_path: str = None,42        collect_artifacts: str = None,43        collect_resources: str = None,44        is_offline: bool = None,45        is_new: bool = None,46        name: str = None,47        description: str = None,48        tags: List[str] = None,49        end_on_finalize: bool = False,50        prefix: str = "",51    ):52        super().__init__()53        self._owner = owner54        self._project = project55        self._run_uuid = run_uuid56        self._client = client57        self._track_code = track_code58        self._track_env = track_env59        self._refresh_data = refresh_data60        self._artifacts_path = artifacts_path61        self._collect_artifacts = collect_artifacts62        self._collect_resources = collect_resources63        self._is_offline = is_offline64        self._is_new = is_new65        self._name = name66        self._description = description67        self._tags = tags68        self._end_on_finalize = end_on_finalize69        self._prefix = prefix70        self._experiment = None71    @property72    @rank_zero_experiment73    def experiment(self) -> tracking.Run:74        if self._experiment:75            return self._experiment76        tracking.init(77            owner=self._owner,78            project=self._project,79            run_uuid=self._run_uuid,80            client=self._client,81            track_code=self._track_code,82            track_env=self._track_env,83            refresh_data=self._refresh_data,84            artifacts_path=self._artifacts_path,85            collect_artifacts=self._collect_artifacts,86            collect_resources=self._collect_resources,87            is_offline=self._is_offline,88            is_new=self._is_new,89            name=self._name,90            description=self._description,91            tags=self._tags,92        )93        self._experiment = tracking.TRACKING_RUN94        return self._experiment95    @rank_zero_only96    def log_hyperparams(self, params: Union[Dict[str, Any], Namespace]) -> None:97        params = _convert_params(params)98        params = _flatten_dict(params)99        params = _sanitize_callable_params(params)100        self.experiment.log_inputs(**params)101    @rank_zero_only102    def log_metrics(103        self, metrics: Dict[str, float], step: Optional[int] = None104    ) -> None:105        assert rank_zero_only.rank == 0, "experiment tried to log from global_rank != 0"106        metrics = self._add_prefix(metrics)107        self.experiment.log_metrics(**metrics, step=step)108    @property109    def save_dir(self) -> Optional[str]:110        return self.experiment.get_outputs_path()111    @rank_zero_only112    def finalize(self, status: str) -> None:113        if self._end_on_finalize:114            self.experiment.end()115            self._experiment = None116    @property117    def name(self) -> str:118        return self.experiment.run_data.name119    @property120    def version(self) -> str:...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.