How to use _cacher method in yandex-tank

Best Python code snippet using yandex-tank

ceilometer.py

Source:ceilometer.py

1# -*- coding: utf-8 -*-2# Copyright 2014 Objectif Libre3#4#    Licensed under the Apache License, Version 2.0 (the "License"); you may5#    not use this file except in compliance with the License. You may obtain6#    a copy of the License at7#8#         http://www.apache.org/licenses/LICENSE-2.09#10#    Unless required by applicable law or agreed to in writing, software11#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT12#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the13#    License for the specific language governing permissions and limitations14#    under the License.15#16# @author: StÃ©phane Albert17#18import decimal19from ceilometerclient import client as cclient20from keystoneauth1 import loading as ks_loading21from oslo_config import cfg22from oslo_utils import units23from cloudkitty import collector24from cloudkitty import utils as ck_utils25CEILOMETER_COLLECTOR_OPTS = 'ceilometer_collector'26ceilometer_collector_opts = ks_loading.get_auth_common_conf_options()27cfg.CONF.register_opts(ceilometer_collector_opts, CEILOMETER_COLLECTOR_OPTS)28ks_loading.register_session_conf_options(29    cfg.CONF,30    CEILOMETER_COLLECTOR_OPTS)31ks_loading.register_auth_conf_options(32    cfg.CONF,33    CEILOMETER_COLLECTOR_OPTS)34CONF = cfg.CONF35class ResourceNotFound(Exception):36    """Raised when the resource doesn't exist."""37    def __init__(self, resource_type, resource_id):38        super(ResourceNotFound, self).__init__(39            "No such resource: %s, type: %s" % (resource_id, resource_type))40        self.resource_id = resource_id41        self.resource_type = resource_type42class CeilometerResourceCacher(object):43    def __init__(self):44        self._resource_cache = {}45    def add_resource_detail(self, resource_type, resource_id, resource_data):46        if resource_type not in self._resource_cache:47            self._resource_cache[resource_type] = {}48        self._resource_cache[resource_type][resource_id] = resource_data49        return self._resource_cache[resource_type][resource_id]50    def has_resource_detail(self, resource_type, resource_id):51        if resource_type in self._resource_cache:52            if resource_id in self._resource_cache[resource_type]:53                return True54        return False55    def get_resource_detail(self, resource_type, resource_id):56        try:57            resource = self._resource_cache[resource_type][resource_id]58            return resource59        except KeyError:60            raise ResourceNotFound(resource_type, resource_id)61class CeilometerCollector(collector.BaseCollector):62    collector_name = 'ceilometer'63    dependencies = ('CeilometerTransformer',64                    'CloudKittyFormatTransformer')65    units_mappings = {66        'compute': 'instance',67        'image': 'MB',68        'volume': 'GB',69        'network.bw.out': 'MB',70        'network.bw.in': 'MB',71        'network.floating': 'ip',72    }73    def __init__(self, transformers, **kwargs):74        super(CeilometerCollector, self).__init__(transformers, **kwargs)75        self.t_ceilometer = self.transformers['CeilometerTransformer']76        self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']77        self._cacher = CeilometerResourceCacher()78        self.auth = ks_loading.load_auth_from_conf_options(79            CONF,80            CEILOMETER_COLLECTOR_OPTS)81        self.session = ks_loading.load_session_from_conf_options(82            CONF,83            CEILOMETER_COLLECTOR_OPTS,84            auth=self.auth)85        self._conn = cclient.get_client(86            '2',87            session=self.session)88    @classmethod89    def get_metadata(cls, resource_name, transformers):90        info = super(CeilometerCollector, cls).get_metadata(resource_name,91                                                            transformers)92        try:93            info["metadata"].extend(transformers['CeilometerTransformer']94                                    .get_metadata(resource_name))95            info["unit"] = cls.units_mappings[resource_name]96        except KeyError:97            pass98        return info99    def gen_filter(self, op='eq', **kwargs):100        """Generate ceilometer filter from kwargs."""101        q_filter = []102        for kwarg in kwargs:103            q_filter.append({'field': kwarg, 'op': op, 'value': kwargs[kwarg]})104        return q_filter105    def prepend_filter(self, prepend, **kwargs):106        """Filter composer."""107        q_filter = {}108        for kwarg in kwargs:109            q_filter[prepend + kwarg] = kwargs[kwarg]110        return q_filter111    def user_metadata_filter(self, op='eq', **kwargs):112        """Create user_metadata filter from kwargs."""113        user_filter = {}114        for kwarg in kwargs:115            field = kwarg116            # Auto replace of . to _ to match ceilometer behaviour117            if '.' in field:118                field = field.replace('.', '_')119            user_filter[field] = kwargs[kwarg]120        user_filter = self.prepend_filter('user_metadata.', **user_filter)121        return self.metadata_filter(op, **user_filter)122    def metadata_filter(self, op='eq', **kwargs):123        """Create metadata filter from kwargs."""124        meta_filter = self.prepend_filter('metadata.', **kwargs)125        return self.gen_filter(op, **meta_filter)126    def resources_stats(self,127                        meter,128                        start,129                        end=None,130                        project_id=None,131                        q_filter=None):132        """Resources statistics during the timespan."""133        start_iso = ck_utils.ts2iso(start)134        req_filter = self.gen_filter(op='ge', timestamp=start_iso)135        if project_id:136            req_filter.extend(self.gen_filter(project=project_id))137        if end:138            end_iso = ck_utils.ts2iso(end)139            req_filter.extend(self.gen_filter(op='le', timestamp=end_iso))140        if isinstance(q_filter, list):141            req_filter.extend(q_filter)142        elif q_filter:143            req_filter.append(q_filter)144        resources_stats = self._conn.statistics.list(meter_name=meter,145                                                     period=0, q=req_filter,146                                                     groupby=['resource_id'])147        return resources_stats148    def active_resources(self,149                         meter,150                         start,151                         end=None,152                         project_id=None,153                         q_filter=None):154        """Resources that were active during the timespan."""155        resources_stats = self.resources_stats(meter,156                                               start,157                                               end,158                                               project_id,159                                               q_filter)160        return [resource.groupby['resource_id']161                for resource in resources_stats]162    def get_compute(self, start, end=None, project_id=None, q_filter=None):163        active_instance_ids = self.active_resources('cpu', start, end,164                                                    project_id, q_filter)165        compute_data = []166        for instance_id in active_instance_ids:167            if not self._cacher.has_resource_detail('compute', instance_id):168                raw_resource = self._conn.resources.get(instance_id)169                instance = self.t_ceilometer.strip_resource_data('compute',170                                                                 raw_resource)171                self._cacher.add_resource_detail('compute',172                                                 instance_id,173                                                 instance)174            instance = self._cacher.get_resource_detail('compute',175                                                        instance_id)176            compute_data.append(177                self.t_cloudkitty.format_item(instance, self.units_mappings[178                    "compute"], 1))179        if not compute_data:180            raise collector.NoDataCollected(self.collector_name, 'compute')181        return self.t_cloudkitty.format_service('compute', compute_data)182    def get_image(self, start, end=None, project_id=None, q_filter=None):183        active_image_stats = self.resources_stats('image.size',184                                                  start,185                                                  end,186                                                  project_id,187                                                  q_filter)188        image_data = []189        for image_stats in active_image_stats:190            image_id = image_stats.groupby['resource_id']191            if not self._cacher.has_resource_detail('image', image_id):192                raw_resource = self._conn.resources.get(image_id)193                image = self.t_ceilometer.strip_resource_data('image',194                                                              raw_resource)195                self._cacher.add_resource_detail('image',196                                                 image_id,197                                                 image)198            image = self._cacher.get_resource_detail('image',199                                                     image_id)200            image_size_mb = decimal.Decimal(image_stats.max) / units.Mi201            image_data.append(202                self.t_cloudkitty.format_item(image, self.units_mappings[203                    "image"], image_size_mb))204        if not image_data:205            raise collector.NoDataCollected(self.collector_name, 'image')206        return self.t_cloudkitty.format_service('image', image_data)207    def get_volume(self, start, end=None, project_id=None, q_filter=None):208        active_volume_stats = self.resources_stats('volume.size',209                                                   start,210                                                   end,211                                                   project_id,212                                                   q_filter)213        volume_data = []214        for volume_stats in active_volume_stats:215            volume_id = volume_stats.groupby['resource_id']216            if not self._cacher.has_resource_detail('volume',217                                                    volume_id):218                raw_resource = self._conn.resources.get(volume_id)219                volume = self.t_ceilometer.strip_resource_data('volume',220                                                               raw_resource)221                self._cacher.add_resource_detail('volume',222                                                 volume_id,223                                                 volume)224            volume = self._cacher.get_resource_detail('volume',225                                                      volume_id)226            volume_data.append(227                self.t_cloudkitty.format_item(volume, self.units_mappings[228                    "volume"], volume_stats.max))229        if not volume_data:230            raise collector.NoDataCollected(self.collector_name, 'volume')231        return self.t_cloudkitty.format_service('volume', volume_data)232    def _get_network_bw(self,233                        direction,234                        start,235                        end=None,236                        project_id=None,237                        q_filter=None):238        if direction == 'in':239            resource_type = 'network.incoming.bytes'240        else:241            direction = 'out'242            resource_type = 'network.outgoing.bytes'243        active_tap_stats = self.resources_stats(resource_type,244                                                start,245                                                end,246                                                project_id,247                                                q_filter)248        bw_data = []249        for tap_stat in active_tap_stats:250            tap_id = tap_stat.groupby['resource_id']251            if not self._cacher.has_resource_detail('network.tap',252                                                    tap_id):253                raw_resource = self._conn.resources.get(tap_id)254                tap = self.t_ceilometer.strip_resource_data(255                    'network.tap',256                    raw_resource)257                self._cacher.add_resource_detail('network.tap',258                                                 tap_id,259                                                 tap)260            tap = self._cacher.get_resource_detail('network.tap',261                                                   tap_id)262            tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M263            bw_data.append(264                self.t_cloudkitty.format_item(tap, self.units_mappings[265                    "network.bw." + direction], tap_bw_mb))266        ck_res_name = 'network.bw.{}'.format(direction)267        if not bw_data:268            raise collector.NoDataCollected(self.collector_name,269                                            ck_res_name)270        return self.t_cloudkitty.format_service(ck_res_name,271                                                bw_data)272    def get_network_bw_out(self,273                           start,274                           end=None,275                           project_id=None,276                           q_filter=None):277        return self._get_network_bw('out', start, end, project_id, q_filter)278    def get_network_bw_in(self,279                          start,280                          end=None,281                          project_id=None,282                          q_filter=None):283        return self._get_network_bw('in', start, end, project_id, q_filter)284    def get_network_floating(self,285                             start,286                             end=None,287                             project_id=None,288                             q_filter=None):289        active_floating_ids = self.active_resources('ip.floating',290                                                    start,291                                                    end,292                                                    project_id,293                                                    q_filter)294        floating_data = []295        for floating_id in active_floating_ids:296            if not self._cacher.has_resource_detail('network.floating',297                                                    floating_id):298                raw_resource = self._conn.resources.get(floating_id)299                floating = self.t_ceilometer.strip_resource_data(300                    'network.floating',301                    raw_resource)302                self._cacher.add_resource_detail('network.floating',303                                                 floating_id,304                                                 floating)305            floating = self._cacher.get_resource_detail('network.floating',306                                                        floating_id)307            floating_data.append(308                self.t_cloudkitty.format_item(floating, self.units_mappings[309                    "network.floating"], 1))310        if not floating_data:311            raise collector.NoDataCollected(self.collector_name,312                                            'network.floating')313        return self.t_cloudkitty.format_service('network.floating',...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.