Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use swift method in tempest

Best Python code snippet using tempest_python

swift.py

Source:swift.py

1# Copyright 2010-2011 OpenStack Foundation2# All Rights Reserved.3#4#    Licensed under the Apache License, Version 2.0 (the "License"); you may5#    not use this file except in compliance with the License. You may obtain6#    a copy of the License at7#8#         http://www.apache.org/licenses/LICENSE-2.09#10#    Unless required by applicable law or agreed to in writing, software11#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT12#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the13#    License for the specific language governing permissions and limitations14#    under the License.15"""Storage backend for SWIFT"""16from __future__ import absolute_import17import hashlib18import httplib19import math20from oslo.config import cfg21import six.moves.urllib.parse as urlparse22from glance.common import auth23from glance.common import exception24from glance.openstack.common import excutils25import glance.openstack.common.log as logging26import glance.store27import glance.store.base28import glance.store.location29try:30    import swiftclient31except ImportError:32    pass33LOG = logging.getLogger(__name__)34DEFAULT_CONTAINER = 'glance'35DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024  # 5GB36DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200  # 200M37ONE_MB = 1000 * 102438swift_opts = [39    cfg.BoolOpt('swift_enable_snet', default=False,40                help=_('Whether to use ServiceNET to communicate with the '41                       'Swift storage servers.')),42    cfg.StrOpt('swift_store_auth_address',43               help=_('The address where the Swift authentication service '44                      'is listening.')),45    cfg.StrOpt('swift_store_user', secret=True,46               help=_('The user to authenticate against the Swift '47                      'authentication service.')),48    cfg.StrOpt('swift_store_key', secret=True,49               help=_('Auth key for the user authenticating against the '50                      'Swift authentication service.')),51    cfg.StrOpt('swift_store_auth_version', default='2',52               help=_('Version of the authentication service to use. '53                      'Valid versions are 2 for keystone and 1 for swauth '54                      'and rackspace.')),55    cfg.BoolOpt('swift_store_auth_insecure', default=False,56                help=_('If True, swiftclient won\'t check for a valid SSL '57                       'certificate when authenticating.')),58    cfg.StrOpt('swift_store_region',59               help=_('The region of the swift endpoint to be used for '60                      'single tenant. This setting is only necessary if the '61                      'tenant has multiple swift endpoints.')),62    cfg.StrOpt('swift_store_endpoint_type', default='publicURL',63               help=_('A string giving the endpoint type of the swift '64                      'service to use (publicURL, adminURL or internalURL). '65                      'This setting is only used if swift_store_auth_version '66                      'is 2.')),67    cfg.StrOpt('swift_store_service_type', default='object-store',68               help=_('A string giving the service type of the swift service '69                      'to use. This setting is only used if '70                      'swift_store_auth_version is 2.')),71    cfg.StrOpt('swift_store_container',72               default=DEFAULT_CONTAINER,73               help=_('Container within the account that the account should '74                      'use for storing images in Swift.')),75    cfg.IntOpt('swift_store_large_object_size',76               default=DEFAULT_LARGE_OBJECT_SIZE,77               help=_('The size, in MB, that Glance will start chunking image '78                      'files and do a large object manifest in Swift.')),79    cfg.IntOpt('swift_store_large_object_chunk_size',80               default=DEFAULT_LARGE_OBJECT_CHUNK_SIZE,81               help=_('The amount of data written to a temporary disk buffer '82                      'during the process of chunking the image file.')),83    cfg.BoolOpt('swift_store_create_container_on_put', default=False,84                help=_('A boolean value that determines if we create the '85                       'container if it does not exist.')),86    cfg.BoolOpt('swift_store_multi_tenant', default=False,87                help=_('If set to True, enables multi-tenant storage '88                       'mode which causes Glance images to be stored in '89                       'tenant specific Swift accounts.')),90    cfg.ListOpt('swift_store_admin_tenants', default=[],91                help=_('A list of tenants that will be granted read/write '92                       'access on all Swift containers created by Glance in '93                       'multi-tenant mode.')),94    cfg.BoolOpt('swift_store_ssl_compression', default=True,95                help=_('If set to False, disables SSL layer compression of '96                       'https swift requests. Setting to False may improve '97                       'performance for images which are already in a '98                       'compressed format, eg qcow2.')),99    cfg.IntOpt('swift_store_retry_get_count', default=0,100               help=_('The number of times a Swift download will be retried '101                      'before the request fails.'))102]103CONF = cfg.CONF104CONF.register_opts(swift_opts)105def swift_retry_iter(resp_iter, length, store, location):106    length = length if length else (resp_iter.len107                                    if hasattr(resp_iter, 'len') else 0)108    retries = 0109    bytes_read = 0110    while retries <= CONF.swift_store_retry_get_count:111        try:112            for chunk in resp_iter:113                yield chunk114                bytes_read += len(chunk)115        except swiftclient.ClientException as e:116            LOG.warn(_("Swift exception raised %s") % e)117        if bytes_read != length:118            if retries == CONF.swift_store_retry_get_count:119                # terminate silently and let higher level decide120                LOG.error(_("Stopping Swift retries after %d "121                            "attempts") % retries)122                break123            else:124                retries += 1125                LOG.info(_("Retrying Swift connection "126                           "(%(retries)d/%(max_retries)d) with "127                           "range=%(start)d-%(end)d") %128                         {'retries': retries,129                          'max_retries': CONF.swift_store_retry_get_count,130                          'start': bytes_read,131                          'end': length})132                (resp_headers, resp_iter) = store._get_object(location, None,133                                                              bytes_read)134        else:135            break136class StoreLocation(glance.store.location.StoreLocation):137    """138    Class describing a Swift URI. A Swift URI can look like any of139    the following:140        swift://user:pass@authurl.com/container/obj-id141        swift://account:user:pass@authurl.com/container/obj-id142        swift+http://user:pass@authurl.com/container/obj-id143        swift+https://user:pass@authurl.com/container/obj-id144    When using multi-tenant a URI might look like this (a storage URL):145        swift+https://example.com/container/obj-id146    The swift+http:// URIs indicate there is an HTTP authentication URL.147    The default for Swift is an HTTPS authentication URL, so swift:// and148    swift+https:// are the same...149    """150    def process_specs(self):151        self.scheme = self.specs.get('scheme', 'swift+https')152        self.user = self.specs.get('user')153        self.key = self.specs.get('key')154        self.auth_or_store_url = self.specs.get('auth_or_store_url')155        self.container = self.specs.get('container')156        self.obj = self.specs.get('obj')157    def _get_credstring(self):158        if self.user and self.key:159            return '%s:%s@' % (urlparse.quote(self.user),160                               urlparse.quote(self.key))161        return ''162    def get_uri(self):163        auth_or_store_url = self.auth_or_store_url164        if auth_or_store_url.startswith('http://'):165            auth_or_store_url = auth_or_store_url[len('http://'):]166        elif auth_or_store_url.startswith('https://'):167            auth_or_store_url = auth_or_store_url[len('https://'):]168        credstring = self._get_credstring()169        auth_or_store_url = auth_or_store_url.strip('/')170        container = self.container.strip('/')171        obj = self.obj.strip('/')172        return '%s://%s%s/%s/%s' % (self.scheme, credstring, auth_or_store_url,173                                    container, obj)174    def parse_uri(self, uri):175        """176        Parse URLs. This method fixes an issue where credentials specified177        in the URL are interpreted differently in Python 2.6.1+ than prior178        versions of Python. It also deals with the peculiarity that new-style179        Swift URIs have where a username can contain a ':', like so:180            swift://account:user:pass@authurl.com/container/obj181        """182        # Make sure that URIs that contain multiple schemes, such as:183        # swift://user:pass@http://authurl.com/v1/container/obj184        # are immediately rejected.185        if uri.count('://') != 1:186            reason = _("URI cannot contain more than one occurrence "187                       "of a scheme. If you have specified a URI like "188                       "swift://user:pass@http://authurl.com/v1/container/obj"189                       ", you need to change it to use the "190                       "swift+http:// scheme, like so: "191                       "swift+http://user:pass@authurl.com/v1/container/obj")192            LOG.debug(_("Invalid store URI: %(reason)s"), {'reason': reason})193            raise exception.BadStoreUri(message=reason)194        pieces = urlparse.urlparse(uri)195        assert pieces.scheme in ('swift', 'swift+http', 'swift+https')196        self.scheme = pieces.scheme197        netloc = pieces.netloc198        path = pieces.path.lstrip('/')199        if netloc != '':200            # > Python 2.6.1201            if '@' in netloc:202                creds, netloc = netloc.split('@')203            else:204                creds = None205        else:206            # Python 2.6.1 compat207            # see lp659445 and Python issue7904208            if '@' in path:209                creds, path = path.split('@')210            else:211                creds = None212            netloc = path[0:path.find('/')].strip('/')213            path = path[path.find('/'):].strip('/')214        if creds:215            cred_parts = creds.split(':')216            if len(cred_parts) != 2:217                reason = (_("Badly formed credentials in Swift URI."))218                LOG.debug(reason)219                raise exception.BadStoreUri()220            user, key = cred_parts221            self.user = urlparse.unquote(user)222            self.key = urlparse.unquote(key)223        else:224            self.user = None225            self.key = None226        path_parts = path.split('/')227        try:228            self.obj = path_parts.pop()229            self.container = path_parts.pop()230            if not netloc.startswith('http'):231                # push hostname back into the remaining to build full authurl232                path_parts.insert(0, netloc)233                self.auth_or_store_url = '/'.join(path_parts)234        except IndexError:235            reason = _("Badly formed Swift URI.")236            LOG.debug(reason)237            raise exception.BadStoreUri()238    @property239    def swift_url(self):240        """241        Creates a fully-qualified auth url that the Swift client library can242        use. The scheme for the auth_url is determined using the scheme243        included in the `location` field.244        HTTPS is assumed, unless 'swift+http' is specified.245        """246        if self.auth_or_store_url.startswith('http'):247            return self.auth_or_store_url248        else:249            if self.scheme in ('swift+https', 'swift'):250                auth_scheme = 'https://'251            else:252                auth_scheme = 'http://'253            return ''.join([auth_scheme, self.auth_or_store_url])254def Store(context=None, loc=None):255    if (CONF.swift_store_multi_tenant and256            (loc is None or loc.store_location.user is None)):257        return MultiTenantStore(context, loc)258    return SingleTenantStore(context, loc)259class BaseStore(glance.store.base.Store):260    CHUNKSIZE = 65536261    def get_schemes(self):262        return ('swift+https', 'swift', 'swift+http')263    def configure(self):264        _obj_size = self._option_get('swift_store_large_object_size')265        self.large_object_size = _obj_size * ONE_MB266        _chunk_size = self._option_get('swift_store_large_object_chunk_size')267        self.large_object_chunk_size = _chunk_size * ONE_MB268        self.admin_tenants = CONF.swift_store_admin_tenants269        self.region = CONF.swift_store_region270        self.service_type = CONF.swift_store_service_type271        self.endpoint_type = CONF.swift_store_endpoint_type272        self.snet = CONF.swift_enable_snet273        self.insecure = CONF.swift_store_auth_insecure274        self.ssl_compression = CONF.swift_store_ssl_compression275    def _get_object(self, location, connection=None, start=None):276        if not connection:277            connection = self.get_connection(location)278        headers = {}279        if start is not None:280            bytes_range = 'bytes=%d-' % start281            headers = {'Range': bytes_range}282        try:283            resp_headers, resp_body = connection.get_object(284                container=location.container, obj=location.obj,285                resp_chunk_size=self.CHUNKSIZE, headers=headers)286        except swiftclient.ClientException as e:287            if e.http_status == httplib.NOT_FOUND:288                msg = _("Swift could not find object %s.") % location.obj289                LOG.warn(msg)290                raise exception.NotFound(msg)291            else:292                raise293        return (resp_headers, resp_body)294    def get(self, location, connection=None):295        location = location.store_location296        (resp_headers, resp_body) = self._get_object(location, connection)297        class ResponseIndexable(glance.store.Indexable):298            def another(self):299                try:300                    return self.wrapped.next()301                except StopIteration:302                    return ''303        length = int(resp_headers.get('content-length', 0))304        if CONF.swift_store_retry_get_count > 0:305            resp_body = swift_retry_iter(resp_body, length, self, location)306        return (ResponseIndexable(resp_body, length), length)307    def get_size(self, location, connection=None):308        location = location.store_location309        if not connection:310            connection = self.get_connection(location)311        try:312            resp_headers = connection.head_object(313                container=location.container, obj=location.obj)314            return int(resp_headers.get('content-length', 0))315        except Exception:316            return 0317    def _option_get(self, param):318        result = getattr(CONF, param)319        if not result:320            reason = (_("Could not find %(param)s in configuration "321                        "options.") % {'param': param})322            LOG.error(reason)323            raise exception.BadStoreConfiguration(store_name="swift",324                                                  reason=reason)325        return result326    def _delete_stale_chunks(self, connection, container, chunk_list):327        for chunk in chunk_list:328            LOG.debug(_("Deleting chunk %s") % chunk)329            try:330                connection.delete_object(container, chunk)331            except Exception:332                msg = _("Failed to delete orphaned chunk "333                        "%(container)s/%(chunk)s")334                LOG.exception(msg % {'container': container,335                                     'chunk': chunk})336    def add(self, image_id, image_file, image_size, connection=None):337        location = self.create_location(image_id)338        if not connection:339            connection = self.get_connection(location)340        self._create_container_if_missing(location.container, connection)341        LOG.debug(_("Adding image object '%(obj_name)s' "342                    "to Swift") % dict(obj_name=location.obj))343        try:344            if image_size > 0 and image_size < self.large_object_size:345                # Image size is known, and is less than large_object_size.346                # Send to Swift with regular PUT.347                obj_etag = connection.put_object(location.container,348                                                 location.obj, image_file,349                                                 content_length=image_size)350            else:351                # Write the image into Swift in chunks.352                chunk_id = 1353                if image_size > 0:354                    total_chunks = str(int(355                        math.ceil(float(image_size) /356                                  float(self.large_object_chunk_size))))357                else:358                    # image_size == 0 is when we don't know the size359                    # of the image. This can occur with older clients360                    # that don't inspect the payload size.361                    LOG.debug(_("Cannot determine image size. Adding as a "362                                "segmented object to Swift."))363                    total_chunks = '?'364                checksum = hashlib.md5()365                written_chunks = []366                combined_chunks_size = 0367                while True:368                    chunk_size = self.large_object_chunk_size369                    if image_size == 0:370                        content_length = None371                    else:372                        left = image_size - combined_chunks_size373                        if left == 0:374                            break375                        if chunk_size > left:376                            chunk_size = left377                        content_length = chunk_size378                    chunk_name = "%s-%05d" % (location.obj, chunk_id)379                    reader = ChunkReader(image_file, checksum, chunk_size)380                    try:381                        chunk_etag = connection.put_object(382                            location.container, chunk_name, reader,383                            content_length=content_length)384                        written_chunks.append(chunk_name)385                    except Exception:386                        # Delete orphaned segments from swift backend387                        with excutils.save_and_reraise_exception():388                            LOG.exception(_("Error during chunked upload to "389                                            "backend, deleting stale chunks"))390                            self._delete_stale_chunks(connection,391                                                      location.container,392                                                      written_chunks)393                    bytes_read = reader.bytes_read394                    msg = (_("Wrote chunk %(chunk_name)s (%(chunk_id)d/"395                             "%(total_chunks)s) of length %(bytes_read)d "396                             "to Swift returning MD5 of content: "397                             "%(chunk_etag)s") %398                           {'chunk_name': chunk_name,399                            'chunk_id': chunk_id,400                            'total_chunks': total_chunks,401                            'bytes_read': bytes_read,402                            'chunk_etag': chunk_etag})403                    LOG.debug(msg)404                    if bytes_read == 0:405                        # Delete the last chunk, because it's of zero size.406                        # This will happen if size == 0.407                        LOG.debug(_("Deleting final zero-length chunk"))408                        connection.delete_object(location.container,409                                                 chunk_name)410                        break411                    chunk_id += 1412                    combined_chunks_size += bytes_read413                # In the case we have been given an unknown image size,414                # set the size to the total size of the combined chunks.415                if image_size == 0:416                    image_size = combined_chunks_size417                # Now we write the object manifest and return the418                # manifest's etag...419                manifest = "%s/%s-" % (location.container, location.obj)420                headers = {'ETag': hashlib.md5("").hexdigest(),421                           'X-Object-Manifest': manifest}422                # The ETag returned for the manifest is actually the423                # MD5 hash of the concatenated checksums of the strings424                # of each chunk...so we ignore this result in favour of425                # the MD5 of the entire image file contents, so that426                # users can verify the image file contents accordingly427                connection.put_object(location.container, location.obj,428                                      None, headers=headers)429                obj_etag = checksum.hexdigest()430            # NOTE: We return the user and key here! Have to because431            # location is used by the API server to return the actual432            # image data. We *really* should consider NOT returning433            # the location attribute from GET /images/<ID> and434            # GET /images/details435            return (location.get_uri(), image_size, obj_etag, {})436        except swiftclient.ClientException as e:437            if e.http_status == httplib.CONFLICT:438                raise exception.Duplicate(_("Swift already has an image at "439                                            "this location"))440            msg = (_("Failed to add object to Swift.\n"441                     "Got error from Swift: %(e)s") % {'e': e})442            LOG.error(msg)443            raise glance.store.BackendException(msg)444    def delete(self, location, connection=None):445        location = location.store_location446        if not connection:447            connection = self.get_connection(location)448        try:449            # We request the manifest for the object. If one exists,450            # that means the object was uploaded in chunks/segments,451            # and we need to delete all the chunks as well as the452            # manifest.453            manifest = None454            try:455                headers = connection.head_object(456                    location.container, location.obj)457                manifest = headers.get('x-object-manifest')458            except swiftclient.ClientException as e:459                if e.http_status != httplib.NOT_FOUND:460                    raise461            if manifest:462                # Delete all the chunks before the object manifest itself463                obj_container, obj_prefix = manifest.split('/', 1)464                segments = connection.get_container(465                    obj_container, prefix=obj_prefix)[1]466                for segment in segments:467                    # TODO(jaypipes): This would be an easy area to parallelize468                    # since we're simply sending off parallelizable requests469                    # to Swift to delete stuff. It's not like we're going to470                    # be hogging up network or file I/O here...471                    connection.delete_object(obj_container,472                                             segment['name'])473            # Delete object (or, in segmented case, the manifest)474            connection.delete_object(location.container, location.obj)475        except swiftclient.ClientException as e:476            if e.http_status == httplib.NOT_FOUND:477                msg = _("Swift could not find image at URI.")478                raise exception.NotFound(msg)479            else:480                raise481    def _create_container_if_missing(self, container, connection):482        """483        Creates a missing container in Swift if the484        ``swift_store_create_container_on_put`` option is set.485        :param container: Name of container to create486        :param connection: Connection to swift service487        """488        try:489            connection.head_container(container)490        except swiftclient.ClientException as e:491            if e.http_status == httplib.NOT_FOUND:492                if CONF.swift_store_create_container_on_put:493                    try:494                        connection.put_container(container)495                    except swiftclient.ClientException as e:496                        msg = (_("Failed to add container to Swift.\n"497                                 "Got error from Swift: %(e)s") % {'e': e})498                        raise glance.store.BackendException(msg)499                else:500                    msg = (_("The container %(container)s does not exist in "501                             "Swift. Please set the "502                             "swift_store_create_container_on_put option"503                             "to add container to Swift automatically.") %504                           {'container': container})505                    raise glance.store.BackendException(msg)506            else:507                raise508    def get_connection(self):509        raise NotImplemented()510    def create_location(self):511        raise NotImplemented()512class SingleTenantStore(BaseStore):513    EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>"514    def configure(self):515        super(SingleTenantStore, self).configure()516        self.auth_version = self._option_get('swift_store_auth_version')517    def configure_add(self):518        self.auth_address = self._option_get('swift_store_auth_address')519        if self.auth_address.startswith('http://'):520            self.scheme = 'swift+http'521        else:522            self.scheme = 'swift+https'523        self.container = CONF.swift_store_container524        self.user = self._option_get('swift_store_user')525        self.key = self._option_get('swift_store_key')526    def create_location(self, image_id):527        specs = {'scheme': self.scheme,528                 'container': self.container,529                 'obj': str(image_id),530                 'auth_or_store_url': self.auth_address,531                 'user': self.user,532                 'key': self.key}533        return StoreLocation(specs)534    def get_connection(self, location):535        if not location.user:536            reason = (_("Location is missing user:password information."))537            LOG.debug(reason)538            raise exception.BadStoreUri(message=reason)539        auth_url = location.swift_url540        if not auth_url.endswith('/'):541            auth_url += '/'542        if self.auth_version == '2':543            try:544                tenant_name, user = location.user.split(':')545            except ValueError:546                reason = (_("Badly formed tenant:user '%(user)s' in "547                            "Swift URI") % {'user': location.user})548                LOG.debug(reason)549                raise exception.BadStoreUri()550        else:551            tenant_name = None552            user = location.user553        os_options = {}554        if self.region:555            os_options['region_name'] = self.region556        os_options['endpoint_type'] = self.endpoint_type557        os_options['service_type'] = self.service_type558        return swiftclient.Connection(559            auth_url, user, location.key, insecure=self.insecure,560            tenant_name=tenant_name, snet=self.snet,561            auth_version=self.auth_version, os_options=os_options,562            ssl_compression=self.ssl_compression)563class MultiTenantStore(BaseStore):564    EXAMPLE_URL = "swift://<SWIFT_URL>/<CONTAINER>/<FILE>"565    def configure_add(self):566        self.container = CONF.swift_store_container567        if self.context is None:568            reason = _("Multi-tenant Swift storage requires a context.")569            raise exception.BadStoreConfiguration(store_name="swift",570                                                  reason=reason)571        if self.context.service_catalog is None:572            reason = _("Multi-tenant Swift storage requires "573                       "a service catalog.")574            raise exception.BadStoreConfiguration(store_name="swift",575                                                  reason=reason)576        self.storage_url = auth.get_endpoint(577            self.context.service_catalog, service_type=self.service_type,578            endpoint_region=self.region, endpoint_type=self.endpoint_type)579        if self.storage_url.startswith('http://'):580            self.scheme = 'swift+http'581        else:582            self.scheme = 'swift+https'583    def delete(self, location, connection=None):584        if not connection:585            connection = self.get_connection(location.store_location)586        super(MultiTenantStore, self).delete(location, connection)587        connection.delete_container(location.store_location.container)588    def set_acls(self, location, public=False, read_tenants=None,589                 write_tenants=None, connection=None):590        location = location.store_location591        if not connection:592            connection = self.get_connection(location)593        if read_tenants is None:594            read_tenants = []595        if write_tenants is None:596            write_tenants = []597        headers = {}598        if public:599            headers['X-Container-Read'] = ".r:*,.rlistings"600        elif read_tenants:601            headers['X-Container-Read'] = ','.join('%s:*' % i602                                                   for i in read_tenants)603        else:604            headers['X-Container-Read'] = ''605        write_tenants.extend(self.admin_tenants)606        if write_tenants:607            headers['X-Container-Write'] = ','.join('%s:*' % i608                                                    for i in write_tenants)609        else:610            headers['X-Container-Write'] = ''611        try:612            connection.post_container(location.container, headers=headers)613        except swiftclient.ClientException as e:614            if e.http_status == httplib.NOT_FOUND:615                msg = _("Swift could not find image at URI.")616                raise exception.NotFound(msg)617            else:618                raise619    def create_location(self, image_id):620        specs = {'scheme': self.scheme,621                 'container': self.container + '_' + str(image_id),622                 'obj': str(image_id),623                 'auth_or_store_url': self.storage_url}624        return StoreLocation(specs)625    def get_connection(self, location):626        return swiftclient.Connection(627            None, self.context.user, None,628            preauthurl=location.swift_url,629            preauthtoken=self.context.auth_tok,630            tenant_name=self.context.tenant,631            auth_version='2', snet=self.snet, insecure=self.insecure,632            ssl_compression=self.ssl_compression)633class ChunkReader(object):634    def __init__(self, fd, checksum, total):635        self.fd = fd636        self.checksum = checksum637        self.total = total638        self.bytes_read = 0639    def read(self, i):640        left = self.total - self.bytes_read641        if i > left:642            i = left643        result = self.fd.read(i)644        self.bytes_read += len(result)645        self.checksum.update(result)...

compiler_swift.py

Source:compiler_swift.py

1import re2import string3import os.path4from waflib import Utils5from distutils.version import StrictVersion6def __run(cmd):7    try:8        output = Utils.subprocess.check_output(cmd, stderr=Utils.subprocess.STDOUT, universal_newlines=True)9        return output.strip()10    except Exception:11        return ""12def __add_swift_flags(ctx):13    ctx.env.SWIFT_FLAGS = [14        "-frontend", "-c", "-sdk", ctx.env.MACOS_SDK,15        "-enable-objc-interop", "-emit-objc-header", "-parse-as-library",16        "-target", "x86_64-apple-macosx10.10"17    ]18    verRe = re.compile("(?i)version\s?([\d.]+)")19    ctx.env.SWIFT_VERSION = verRe.search(__run([ctx.env.SWIFT, '-version'])).group(1)20    # prevent possible breakages with future swift versions21    if StrictVersion(ctx.env.SWIFT_VERSION) >= StrictVersion("6.0"):22        ctx.env.SWIFT_FLAGS.extend(["-swift-version", "5"])23    if ctx.is_debug_build():24        ctx.env.SWIFT_FLAGS.append("-g")25    if ctx.is_optimization():26        ctx.env.SWIFT_FLAGS.append("-O")27def __add_static_swift_library_linking_flags(ctx, swift_library):28    ctx.env.append_value('LINKFLAGS', [29        '-L%s' % swift_library,30        '-Xlinker', '-force_load_swift_libs', '-lc++',31    ])32def __add_dynamic_swift_library_linking_flags(ctx, swift_library):33    ctx.env.append_value('LINKFLAGS', ['-L%s' % swift_library])34    # ABI compatibility35    if StrictVersion(ctx.env.SWIFT_VERSION) >= StrictVersion("5.0"):36        ctx.env.append_value('LINKFLAGS', [37            '-Xlinker', '-rpath', '-Xlinker', '/usr/lib/swift',38            '-L/usr/lib/swift',39        ])40    ctx.env.append_value('LINKFLAGS', [41        '-Xlinker', '-rpath', '-Xlinker', swift_library,42    ])43def __find_swift_library(ctx):44    swift_libraries = {}45    # look for set lib paths in passed environment variables46    if 'SWIFT_LIB_DYNAMIC' in ctx.environ:47        swift_libraries['SWIFT_LIB_DYNAMIC'] = ctx.environ['SWIFT_LIB_DYNAMIC']48    if 'SWIFT_LIB_STATIC' in ctx.environ:49        swift_libraries['SWIFT_LIB_STATIC'] = ctx.environ['SWIFT_LIB_STATIC']50    # search for swift libs relative to the swift compiler executable51    swift_library_relative_paths = {52        'SWIFT_LIB_DYNAMIC': '../../lib/swift/macosx',53        'SWIFT_LIB_STATIC': '../../lib/swift_static/macosx'54    }55    for lib_type, path in swift_library_relative_paths.items():56        if lib_type not in swift_libraries:57            lib_path = os.path.join(ctx.env.SWIFT, path)58            swift_library = ctx.root.find_dir(lib_path)59            if swift_library is not None:60                swift_libraries[lib_type] = swift_library.abspath()61    # fall back to xcode-select path62    swift_library_paths = {63        'SWIFT_LIB_DYNAMIC': [64            'Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx',65            'usr/lib/swift/macosx'66        ],67        'SWIFT_LIB_STATIC': [68            'Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static/macosx',69            'usr/lib/swift_static/macosx'70        ]71    }72    dev_path = __run(['xcode-select', '-p'])[1:]73    for lib_type, paths in swift_library_paths.items():74        for path in paths:75            if lib_type not in swift_libraries:76                swift_library = ctx.root.find_dir([dev_path, path])77                if swift_library is not None:78                    swift_libraries[lib_type] = swift_library.abspath()79                    break80            else:81                break82    # check if library paths were found83    ctx.start_msg('Checking for dynamic Swift Library')84    if 'SWIFT_LIB_DYNAMIC' in swift_libraries:85        ctx.end_msg(swift_libraries['SWIFT_LIB_DYNAMIC'])86    else:87        ctx.end_msg(False)88    ctx.start_msg('Checking for static Swift Library')89    if 'SWIFT_LIB_STATIC' in swift_libraries:90        ctx.end_msg(swift_libraries['SWIFT_LIB_STATIC'])91        ctx.env['SWIFT_LIB_STATIC'] = swift_libraries['SWIFT_LIB_STATIC']92    else:93        ctx.end_msg(False)94    enableStatic = getattr(ctx.options, 'enable_swift-static')95    if (enableStatic) and 'SWIFT_LIB_STATIC' in swift_libraries:96        __add_static_swift_library_linking_flags(ctx, swift_libraries['SWIFT_LIB_STATIC'])97    else:98        __add_dynamic_swift_library_linking_flags(ctx, swift_libraries['SWIFT_LIB_DYNAMIC'])99def __find_macos_sdk(ctx):100    ctx.start_msg('Checking for macOS SDK')101    sdk = None102    sdk_build_version = None103    sdk_version = None104    # look for set macOS SDK paths and version in passed environment variables105    if 'MACOS_SDK' in ctx.environ:106        sdk = ctx.environ['MACOS_SDK']107    if 'MACOS_SDK_VERSION' in ctx.environ:108        ctx.env.MACOS_SDK_VERSION = ctx.environ['MACOS_SDK_VERSION']109    # find macOS SDK paths and version110    if not sdk:111        sdk = __run(['xcrun', '--sdk', 'macosx', '--show-sdk-path'])112    if not ctx.env.MACOS_SDK_VERSION:113        # show-sdk-build-version: is not available on older command line tools, but return a build version (eg 17A360)114        # show-sdk-version: is always available, but on older dev tools it's only the major version115        sdk_build_version = __run(['xcrun', '--sdk', 'macosx', '--show-sdk-build-version'])116        sdk_version = __run(['xcrun', '--sdk', 'macosx', '--show-sdk-version'])117    if sdk:118        ctx.env.MACOS_SDK = sdk119        build_version = '10.10.0'120        if not ctx.env.MACOS_SDK_VERSION:121            # convert build version to a version string122            # first 2 two digits are the major version, starting with 15 which is 10.11 (offset of 4)123            # 1 char is the minor version, A => 0, B => 1 and ongoing124            # las digits are bugfix version, which are nor relevant for us125            # eg 16E185 => 10.12.4, 17A360 => 10.13, 18B71 => 10.14.1126            if sdk_build_version and isinstance(sdk_build_version, str):127                verRe = re.compile("(\d+)(\D+)(\d+)")128                version_parts = verRe.search(sdk_build_version)129                major = int(version_parts.group(1)) - 4130                minor = string.ascii_lowercase.index(version_parts.group(2).lower())131                build_version = '10.' + str(major) + '.' + str(minor)132            if not isinstance(sdk_version, str):133                sdk_version = '10.10.0'134            # pick the higher version, always pick sdk over build if newer135            if StrictVersion(build_version) > StrictVersion(sdk_version):136                ctx.env.MACOS_SDK_VERSION = build_version137            else:138                ctx.env.MACOS_SDK_VERSION = sdk_version139        ctx.end_msg(sdk + ' (version found: ' + ctx.env.MACOS_SDK_VERSION + ')')140    else:141        ctx.end_msg(False)142def __find_swift_compiler(ctx):143    ctx.start_msg('Checking for swift (Swift compiler)')144    swift = ''145    # look for set swift paths in passed environment variables146    if 'SWIFT' in ctx.environ:147        swift = ctx.environ['SWIFT']148    # find swift executable149    if not swift:150        swift = __run(['xcrun', '-find', 'swift'])151    if swift:152        ctx.end_msg(swift)153        ctx.env.SWIFT = swift154        __add_swift_flags(ctx)155        __find_swift_library(ctx)156    else:157        ctx.end_msg(False)158def configure(ctx):159    if ctx.env.DEST_OS == "darwin":160        __find_macos_sdk(ctx)161        if ctx.options.enable_swift is not False:...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.