How to use is_multipart_upload method in localstack

Best Python code snippet using localstack_python

shotgun.py

Source:shotgun.py

1#!/usr/bin/env python2"""3 -----------------------------------------------------------------------------4 Copyright (c) 2009-2017, Shotgun Software Inc.5 Redistribution and use in source and binary forms, with or without6 modification, are permitted provided that the following conditions are met:7  - Redistributions of source code must retain the above copyright notice, this8    list of conditions and the following disclaimer.9  - Redistributions in binary form must reproduce the above copyright notice,10    this list of conditions and the following disclaimer in the documentation11    and/or other materials provided with the distribution.12  - Neither the name of the Shotgun Software Inc nor the names of its13    contributors may be used to endorse or promote products derived from this14    software without specific prior written permission.15 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"16 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE17 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE18 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE19 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL20 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR21 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER22 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,23 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE24 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.25"""26import base6427import cookielib    # used for attachment upload28import cStringIO    # used for attachment upload29import datetime30import logging31import mimetools    # used for attachment upload32import os33import re34import copy35import stat         # used for attachment upload36import sys37import time38import types39import urllib40import urllib2      # used for image upload41import urlparse42import shutil       # used for attachment download43# use relative import for versions >=2.5 and package import for python versions <2.544if (sys.version_info[0] > 2) or (sys.version_info[0] == 2 and sys.version_info[1] >= 6):45    from sg_26 import *46elif (sys.version_info[0] > 2) or (sys.version_info[0] == 2 and sys.version_info[1] >= 5):47    from sg_25 import *48else:49    from sg_24 import *50# mimetypes imported in version specific imports51mimetypes.add_type('video/webm','.webm') # webm and mp4 seem to be missing52mimetypes.add_type('video/mp4', '.mp4')  # from some OS/distros53LOG = logging.getLogger("shotgun_api3")54"""55Logging instance for shotgun_api356Provides a logging instance where log messages are sent during execution. This instance has no57handler associated with it.58.. seealso:: :ref:`logging`59"""60LOG.setLevel(logging.WARN)61SG_TIMEZONE = SgTimezone()62NO_SSL_VALIDATION = False63"""64Turns off hostname matching validation for SSL certificates65Sometimes there are cases where certificate validation should be disabled. For example, if you66have a self-signed internal certificate that isn't included in our certificate bundle, you may67not require the added security provided by enforcing this.68"""69try:70    import ssl71except ImportError, e:72    if "SHOTGUN_FORCE_CERTIFICATE_VALIDATION" in os.environ:73        raise ImportError("%s. SHOTGUN_FORCE_CERTIFICATE_VALIDATION environment variable prevents "74                          "disabling SSL certificate validation." % e)75    LOG.debug("ssl not found, disabling certificate validation")76    NO_SSL_VALIDATION = True77# ----------------------------------------------------------------------------78# Version79__version__ = "3.0.35"80# ----------------------------------------------------------------------------81# Errors82class ShotgunError(Exception):83    """84    Base for all Shotgun API Errors.85    """86    pass87class ShotgunFileDownloadError(ShotgunError):88    """89    Exception for file download-related errors.90    """91    pass92class Fault(ShotgunError):93    """94    Exception when server-side exception detected.95    """96    pass97class AuthenticationFault(Fault):98    """99    Exception when the server side reports an error related to authentication.100    """101    pass102class MissingTwoFactorAuthenticationFault(Fault):103    """104    Exception when the server side reports an error related to missing two-factor authentication105    credentials.106    """107    pass108class UserCredentialsNotAllowedForSSOAuthenticationFault(Fault):109    """110    Exception when the server is configured to use SSO. It is not possible to use111    a username/password pair to authenticate on such server.112    """113    pass114# ----------------------------------------------------------------------------115# API116class ServerCapabilities(object):117    """118    Container for the servers capabilities, such as version enabled features.119    .. warning::120        This class is part of the internal API and its interfaces may change at any time in121        the future. Therefore, usage of this class is discouraged.122    """123    def __init__(self, host, meta):124        """125        ServerCapabilities.__init__126        :param str host: Host name for the server excluding protocol.127        :param dict meta: dict of meta data for the server returned from the info() api method.128        :ivar str host:129        :ivar dict server_info:130        :ivar tuple version: Simple version of the Shotgun server. ``(major, minor, rev)``131        :ivar bool is_dev: ``True`` if server is running a development version of the Shotgun132            codebase.133        """134        # Server host name135        self.host = host136        self.server_info = meta137        # Version from server is major.minor.rev or major.minor.rev."Dev"138        # Store version as tuple and check dev flag139        try:140            self.version = meta.get("version", None)141        except AttributeError:142            self.version = None143        if not self.version:144            raise ShotgunError("The Shotgun Server didn't respond with a version number. "145                               "This may be because you are running an older version of "146                               "Shotgun against a more recent version of the Shotgun API. "147                               "For more information, please contact Shotgun Support.")148        if len(self.version) > 3 and self.version[3] == "Dev":149            self.is_dev = True150        else:151            self.is_dev = False152        self.version = tuple(self.version[:3])153        self._ensure_json_supported()154    def _ensure_support(self, feature, raise_hell=True):155        """156        Checks the server version supports a given feature, raises an exception if it does not.157        :param dict feature: dict where **version** key contains a 3 integer tuple indicating the158            supported server version and **label** key contains a human-readable label str::159                { 'version': (5, 4, 4), 'label': 'project parameter }160        :param bool raise_hell: Whether to raise an exception if the feature is not supported.161            Defaults to ``True``162        :raises: :class:`ShotgunError` if the current server version does not support ``feature``163        """164        if not self.version or self.version < feature['version']:165            if raise_hell:166                raise ShotgunError(167                    "%s requires server version %s or higher, "\168                    "server is %s" % (feature['label'], _version_str(feature['version']), _version_str(self.version))169                )170            return False171        else:172            return True173    def _ensure_json_supported(self):174        """175        Ensures server has support for JSON API endpoint added in v2.4.0.176        """177        self._ensure_support({178            'version': (2, 4, 0),179            'label': 'JSON API'180        })181    def ensure_include_archived_projects(self):182        """183        Ensures server has support for archived Projects feature added in v5.3.14.184        """185        self._ensure_support({186            'version': (5, 3, 14),187            'label': 'include_archived_projects parameter'188        })189    def ensure_per_project_customization(self):190        """191        Ensures server has support for per-project customization feature added in v5.4.4.192        """193        return self._ensure_support({194            'version': (5, 4, 4),195            'label': 'project parameter'196        }, True)197    def ensure_support_for_additional_filter_presets(self):198        """199        Ensures server has support for additional filter presets feature added in v7.0.0.200        """201        return self._ensure_support({202            'version': (7, 0, 0),203            'label': 'additional_filter_presets parameter'204        }, True)205    def ensure_user_following_support(self):206        """207        Ensures server has support for listing items a user is following, added in v7.0.12.208        """209        return self._ensure_support({210            'version': (7, 0, 12),211            'label': 'user_following parameter'212        }, True)213    def ensure_paging_info_without_counts_support(self):214        """215        Ensures server has support for optimized pagination, added in v7.4.0.216        """217        return self._ensure_support({218            'version': (7, 4, 0),219            'label': 'optimized pagination'220        }, False)221    def ensure_return_image_urls_support(self):222        """223        Ensures server has support for returning thumbnail URLs without additional round-trips, added in v3.3.0.224        """225        return self._ensure_support({226            'version': (3, 3, 0),227            'label': 'return thumbnail URLs'228        }, False)229    def __str__(self):230        return "ServerCapabilities: host %s, version %s, is_dev %s"\231                 % (self.host, self.version, self.is_dev)232class ClientCapabilities(object):233    """234    Container for the client capabilities.235    .. warning::236        This class is part of the internal API and its interfaces may change at any time in237        the future. Therefore, usage of this class is discouraged.238    :ivar str platform: The current client platform. Valid values are ``mac``, ``linux``,239        ``windows``, or ``None`` (if the current platform couldn't be determined).240    :ivar str local_path_field: The SG field used for local file paths. This is calculated using241        the value of ``platform``. Ex. ``local_path_mac``.242    :ivar str py_version: Simple version of Python executable as a string. Eg. ``2.7``.243    :ivar str ssl_version: Version of OpenSSL installed. Eg. ``OpenSSL 1.0.2g  1 Mar 2016``. This244        info is only available in Python 2.7+ if the ssl module was imported successfully.245        Defaults to ``unknown``246    """247    def __init__(self):248        system = sys.platform.lower()249        if system == 'darwin':250            self.platform = "mac"251        elif system.startswith('linux'):252            self.platform = 'linux'253        elif system == 'win32':254            self.platform = 'windows'255        else:256            self.platform = None257        if self.platform:258            self.local_path_field = "local_path_%s" % (self.platform)259        else:260            self.local_path_field = None261        self.py_version = ".".join(str(x) for x in sys.version_info[:2])262        # extract the OpenSSL version if we can. The version is only available in Python 2.7 and263        # only if we successfully imported ssl264        self.ssl_version = "unknown"265        try:266            self.ssl_version = ssl.OPENSSL_VERSION267        except (AttributeError, NameError):268            pass269    def __str__(self):270        return "ClientCapabilities: platform %s, local_path_field %s, "\271            "py_verison %s, ssl version %s" % (self.platform, self.local_path_field,272            self.py_version, self.ssl_version)273class _Config(object):274    """275    Container for the client configuration.276    """277    def __init__(self):278        self.max_rpc_attempts = 3279        # From http://docs.python.org/2.6/library/httplib.html:280        # If the optional timeout parameter is given, blocking operations281        # (like connection attempts) will timeout after that many seconds282        # (if it is not given, the global default timeout setting is used)283        self.timeout_secs = None284        self.api_ver = 'api3'285        self.convert_datetimes_to_utc = True286        self.records_per_page = 500287        self.api_key = None288        self.script_name = None289        self.user_login = None290        self.user_password = None291        self.auth_token = None292        self.sudo_as_login = None293        # Authentication parameters to be folded into final auth_params dict294        self.extra_auth_params = None295        # uuid as a string296        self.session_uuid = None297        self.scheme = None298        self.server = None299        self.api_path = None300        # The raw_http_proxy reflects the exact string passed in301        # to the Shotgun constructor. This can be useful if you302        # need to construct a Shotgun API instance based on303        # another Shotgun API instance.304        self.raw_http_proxy = None305        # if a proxy server is being used, the proxy_handler306        # below will contain a urllib2.ProxyHandler instance307        # which can be used whenever a request needs to be made.308        self.proxy_handler = None309        self.proxy_server = None310        self.proxy_port = 8080311        self.proxy_user = None312        self.proxy_pass = None313        self.session_token = None314        self.authorization = None315        self.no_ssl_validation = False316class Shotgun(object):317    """318    Shotgun Client connection.319    """320    # reg ex from321    # http://underground.infovark.com/2008/07/22/iso-date-validation-regex/322    # Note a length check is done before checking the reg ex323    _DATE_PATTERN = re.compile(324        "^(\d{4})\D?(0[1-9]|1[0-2])\D?([12]\d|0[1-9]|3[01])$")325    _DATE_TIME_PATTERN = re.compile(326        "^(\d{4})\D?(0[1-9]|1[0-2])\D?([12]\d|0[1-9]|3[01])"\327        "(\D?([01]\d|2[0-3])\D?([0-5]\d)\D?([0-5]\d)?\D?(\d{3})?)?$")328    _MULTIPART_UPLOAD_CHUNK_SIZE = 20000000329    def __init__(self,330                 base_url,331                 script_name=None,332                 api_key=None,333                 convert_datetimes_to_utc=True,334                 http_proxy=None,335                 ensure_ascii=True,336                 connect=True,337                 ca_certs=None,338                 login=None,339                 password=None,340                 sudo_as_login=None,341                 session_token=None,342                 auth_token=None):343        """344        Initializes a new instance of the Shotgun client.345        :param str base_url: http or https url of the Shotgun server. Do not include the trailing346            slash::347                https://example.shotgunstudio.com348        :param str script_name: name of the Script entity used to authenticate to the server.349            If provided, then ``api_key`` must be as well, and neither ``login`` nor ``password``350            can be provided.351            .. seealso:: :ref:`authentication`352        :param str api_key: API key for the provided ``script_name``. Used to authenticate to the353            server.  If provided, then ``script_name`` must be as well, and neither ``login`` nor354            ``password`` can be provided.355            .. seealso:: :ref:`authentication`356        :param bool convert_datetimes_to_utc: (optional) When ``True``, datetime values are converted357            from local time to UTC time before being sent to the server. Datetimes received from358            the server are then converted back to local time. When ``False`` the client should use359            UTC date time values. Default is ``True``.360        :param str http_proxy: (optional) URL for a proxy server to use for all connections. The361            expected str format is ``[username:password@]111.222.333.444[:8080]``. Examples::362                192.168.0.1363                192.168.0.1:8888364                joe:user@192.168.0.1:8888365        :param bool connect: (optional) When ``True``, as soon as the :class:`~shotgun_api3.Shotgun`366            instance is created, a connection will be made to the Shotgun server to determine the367            server capabilities and confirm this version of the client is compatible with the server368            version. This is mostly used for testing. Default is ``True``.369        :param str ca_certs: (optional) path to an external SSL certificates file. By default, the370            Shotgun API will use its own built-in certificates file which stores root certificates371            for the most common Certificate Authorities (CAs). If you are using a corporate or372            internal CA, or are packaging an application into an executable, it may be necessary to373            point to your own certificates file. You can do this by passing in the full path to the374            file via this parameter or by setting the environment variable ``SHOTGUN_API_CACERTS``.375            In the case both are set, this parameter will take precedence.376        :param str login: The user login str to use to authenticate to the server when using user-based377            authentication. If provided, then ``password`` must be as well, and neither378            ``script_name`` nor ``api_key`` can be provided.379            .. seealso:: :ref:`authentication`380        :param str password: The password str to use to authenticate to the server when using user-based381            authentication. If provided, then ``login`` must be as well and neither ``script_name``382            nor ``api_key`` can be provided.383            See :ref:`authentication` for more info.384        :param str sudo_as_login: A user login string for the user whose permissions will be applied385            to all actions. Event log entries will be generated showing this user performing all386            actions with an additional extra meta-data parameter ``sudo_actual_user`` indicating the387            script or user that is actually authenticated.388        :param str session_token: The session token to use to authenticate to the server. This389            can be used as an alternative to authenticating with a script user or regular user.390            You can retrieve the session token by running the391            :meth:`~shotgun_api3.Shotgun.get_session_token()` method.392            .. todo: Add this info to the Authentication section of the docs393        :param str auth_token: The authentication token required to authenticate to a server with394            two-factor authentication turned on. If provided, then ``login`` and ``password`` must395            be provided as well, and neither ``script_name`` nor ``api_key`` can be provided.396            .. note:: These tokens can be short lived so a session is established right away if an397                ``auth_token`` is provided. A398                :class:`~shotgun_api3.MissingTwoFactorAuthenticationFault` will be raised if the399                ``auth_token`` is invalid.400            .. todo: Add this info to the Authentication section of the docs401        .. note:: A note about proxy connections: If you are using Python <= v2.6.2, HTTPS402            connections through a proxy server will not work due to a bug in the :mod:`urllib2`403            library (see http://bugs.python.org/issue1424152). This will affect upload and404            download-related methods in the Shotgun API (eg. :meth:`~shotgun_api3.Shotgun.upload`,405            :meth:`~shotgun_api3.Shotgun.upload_thumbnail`,406            :meth:`~shotgun_api3.Shotgun.upload_filmstrip_thumbnail`,407            :meth:`~shotgun_api3.Shotgun.download_attachment`. Normal CRUD methods for passing JSON408            data should still work fine. If you cannot upgrade your Python installation, you can see409            the patch merged into Python v2.6.3 (http://hg.python.org/cpython/rev/0f57b30a152f/) and410            try and hack it into your installation but YMMV. For older versions of Python there411            are other patches that were proposed in the bug report that may help you as well.412        """413        # verify authentication arguments414        if session_token is not None:415            if script_name is not None or api_key is not None:416                raise ValueError("cannot provide both session_token "417                                 "and script_name/api_key")418            if login is not None or password is not None:419                raise ValueError("cannot provide both session_token "420                                 "and login/password")421        if login is not None or password is not None:422            if script_name is not None or api_key is not None:423                raise ValueError("cannot provide both login/password "424                                 "and script_name/api_key")425            if login is None:426                raise ValueError("password provided without login")427            if password is None:428                raise ValueError("login provided without password")429        if script_name is not None or api_key is not None:430            if script_name is None:431                raise ValueError("api_key provided without script_name")432            if api_key is None:433                raise ValueError("script_name provided without api_key")434        if auth_token is not None:435            if login is None or password is None:436                raise ValueError("must provide a user login and password with an auth_token")437            if script_name is not None or api_key is not None:438                raise ValueError("cannot provide an auth_code with script_name/api_key")439        # Can't use 'all' with python 2.4440        if len([x for x in [session_token, script_name, api_key, login, password] if x]) == 0:441            if connect:442                raise ValueError("must provide login/password, session_token or script_name/api_key")443        self.config = _Config()444        self.config.api_key = api_key445        self.config.script_name = script_name446        self.config.user_login = login447        self.config.user_password = password448        self.config.auth_token = auth_token449        self.config.session_token = session_token450        self.config.sudo_as_login = sudo_as_login451        self.config.convert_datetimes_to_utc = convert_datetimes_to_utc452        self.config.no_ssl_validation = NO_SSL_VALIDATION453        self.config.raw_http_proxy = http_proxy454        self._connection = None455        if ca_certs is not None:456            self.__ca_certs = ca_certs457        else:458            self.__ca_certs = os.environ.get('SHOTGUN_API_CACERTS')459        self.base_url = (base_url or "").lower()460        self.config.scheme, self.config.server, api_base, _, _ = \461            urlparse.urlsplit(self.base_url)462        if self.config.scheme not in ("http", "https"):463            raise ValueError("base_url must use http or https got '%s'" %464                self.base_url)465        self.config.api_path = urlparse.urljoin(urlparse.urljoin(466            api_base or "/", self.config.api_ver + "/"), "json")467        # if the service contains user information strip it out468        # copied from the xmlrpclib which turned the user:password into469        # and auth header470        auth, self.config.server = urllib.splituser(urlparse.urlsplit(base_url).netloc)471        if auth:472            auth = base64.encodestring(urllib.unquote(auth))473            self.config.authorization = "Basic " + auth.strip()474        # foo:bar@123.456.789.012:3456475        if http_proxy:476            # check if we're using authentication. Start from the end since there might be477            # @ in the user's password.478            p = http_proxy.rsplit("@", 1)479            if len(p) > 1:480                self.config.proxy_user, self.config.proxy_pass = \481                    p[0].split(":", 1)482                proxy_server = p[1]483            else:484                proxy_server = http_proxy485            proxy_netloc_list = proxy_server.split(":", 1)486            self.config.proxy_server = proxy_netloc_list[0]487            if len(proxy_netloc_list) > 1:488                try:489                    self.config.proxy_port = int(proxy_netloc_list[1])490                except ValueError:491                    raise ValueError("Invalid http_proxy address '%s'. Valid " \492                        "format is '123.456.789.012' or '123.456.789.012:3456'"\493                        ". If no port is specified, a default of %d will be "\494                        "used." % (http_proxy, self.config.proxy_port))495            # now populate self.config.proxy_handler496            if self.config.proxy_user and self.config.proxy_pass:497                auth_string = "%s:%s@" % (self.config.proxy_user, self.config.proxy_pass)498            else:499                auth_string = ""500            proxy_addr = "http://%s%s:%d" % (auth_string, self.config.proxy_server, self.config.proxy_port)501            self.config.proxy_handler = urllib2.ProxyHandler({self.config.scheme : proxy_addr})502        if ensure_ascii:503            self._json_loads = self._json_loads_ascii504        self.client_caps = ClientCapabilities()505        # this relies on self.client_caps being set first506        self.reset_user_agent()507        self._server_caps = None508        # test to ensure the the server supports the json API509        # call to server will only be made once and will raise error510        if connect:511            self.server_caps512        # Check for api_max_entities_per_page in the server info and change the record per page value if it is supplied.513        self.config.records_per_page = self.server_info.get('api_max_entities_per_page') or self.config.records_per_page514        # When using auth_token in a 2FA scenario we need to switch to session-based515        # authentication because the auth token will no longer be valid after a first use.516        if self.config.auth_token is not None:517            self.config.session_token = self.get_session_token()518            self.config.user_login = None519            self.config.user_password = None520            self.config.auth_token = None521    # ========================================================================522    # API Functions523    @property524    def server_info(self):525        """526        Property containing server information.527        >>> sg.server_info528        {'full_version': [6, 3, 15, 0], 'version': [6, 3, 15], ...}529        .. note::530            Beyond ``full_version`` and ``version`` which differ by the inclusion of the bugfix number, you should expect531            these values to be unsupported and for internal use only.532        :returns: dict of server information from :class:`ServerCapabilities` object533        :rtype: dict534        """535        return self.server_caps.server_info536    @property537    def server_caps(self):538        """539        Property containing :class:`ServerCapabilities` object.540        >>> sg.server_caps541        <shotgun_api3.shotgun.ServerCapabilities object at 0x10120d350>542        :returns: :class:`ServerCapabilities` object that describe the server the client is543            connected to.544        :rtype: :class:`ServerCapabilities` object545        """546        if not self._server_caps or (547            self._server_caps.host != self.config.server):548            self._server_caps = ServerCapabilities(self.config.server,549                self.info())550        return self._server_caps551    def connect(self):552        """553        Connect client to the server if it is not already connected.554        .. note:: The client will automatically connect to the server on demand. You only need to555            call this function if you wish to confirm the client can connect.556        """557        self._get_connection()558        self.info()559        return560    def close(self):561        """562        Close the current connection to the server.563        If the client needs to connect again it will do so automatically.564        """565        self._close_connection()566        return567    def info(self):568        """569        Get API-related metadata from the Shotgun server.570        >>> sg.info()571        {'full_version': [6, 3, 15, 0], 'version': [6, 3, 15], ...}572        .. note::573            Beyond ``full_version`` and ``version`` which differ by the inclusion of the bugfix number, you should expect574            these values to be unsupported and for internal use only.575        :returns: dict of the server metadata.576        :rtype: dict577        """578        return self._call_rpc("info", None, include_auth_params=False)579    def find_one(self, entity_type, filters, fields=None, order=None,580        filter_operator=None, retired_only=False, include_archived_projects=True,581        additional_filter_presets=None):582        """583        Shortcut for :meth:`~shotgun_api3.Shotgun.find` with ``limit=1`` so it returns a single584        result.585            >>> sg.find_one("Asset", [["id", "is", 32]], ["id", "code", "sg_status_list"])586            {'code': 'Gopher', 'id': 32, 'sg_status_list': 'ip', 'type': 'Asset'}587        :param str entity_type: Shotgun entity type as a string to find.588        :param list filters: list of filters to apply to the query.589            .. seealso:: :ref:`filter_syntax`590        :param list fields: Optional list of fields to include in each entity record returned.591            Defaults to ``["id"]``.592        :param int order: Optional list of fields to order the results by. List has the format::593            [{'field_name':'foo', 'direction':'asc'}, {'field_name':'bar', 'direction':'desc'}]594            Defaults to sorting by ``id`` in ascending order.595        :param str filter_operator: Operator to apply to the filters. Supported values are ``"all"``596            and ``"any"``. These are just another way of defining if the query is an AND or OR597            query. Defaults to ``"all"``.598        :param bool retired_only: Optional boolean when ``True`` will return only entities that have599            been retried. Defaults to ``False`` which returns only entities which have not been600            retired. There is no option to return both retired and non-retired entities in the601            same query.602        :param bool include_archived_projects: Optional boolean flag to include entities whose projects603            have been archived. Defaults to ``True``.604        :param additional_filter_presets: Optional list of presets to further filter the result605            set, list has the form::606                [{"preset_name": <preset_name>, <optional_param1>: <optional_value1>, ... }]607            Note that these filters are ANDed together and ANDed with the 'filter'608            argument.609            For details on supported presets and the format of this parameter see610            :ref:`additional_filter_presets`611        :returns: Dictionary representing a single matching entity with the requested fields,612            and the defaults ``"id"`` and ``"type"`` which are always included.613        :rtype: dict614        """615        results = self.find(entity_type, filters, fields, order,616            filter_operator, 1, retired_only, include_archived_projects=include_archived_projects,617            additional_filter_presets=additional_filter_presets)618        if results:619            return results[0]620        return None621    def find(self, entity_type, filters, fields=None, order=None,622            filter_operator=None, limit=0, retired_only=False, page=0,623            include_archived_projects=True, additional_filter_presets=None):624        """625        Find entities matching the given filters.626            >>> # Find Character Assets in Sequence 100_FOO627            >>> # -------------628            >>> fields = ['id', 'code', 'sg_asset_type']629            >>> sequence_id = 2 # Sequence "100_FOO"630            >>> project_id = 4 # Demo Project631            >>> filters = [632            ...     ['project', 'is', {'type': 'Project', 'id': project_id}],633            ...     ['sg_asset_type', 'is', 'Character'],634            ...     ['sequences', 'is', {'type': 'Sequence', 'id': sequence_id}]635            ... ]636            >>> assets= sg.find("Asset",filters,fields)637            [{'code': 'Gopher', 'id': 32, 'sg_asset_type': 'Character', 'type': 'Asset'},638             {'code': 'Cow', 'id': 33, 'sg_asset_type': 'Character', 'type': 'Asset'},639             {'code': 'Bird_1', 'id': 35, 'sg_asset_type': 'Character', 'type': 'Asset'},640             {'code': 'Bird_2', 'id': 36, 'sg_asset_type': 'Character', 'type': 'Asset'},641             {'code': 'Bird_3', 'id': 37, 'sg_asset_type': 'Character', 'type': 'Asset'},642             {'code': 'Raccoon', 'id': 45, 'sg_asset_type': 'Character', 'type': 'Asset'},643             {'code': 'Wet Gopher', 'id': 149, 'sg_asset_type': 'Character', 'type': 'Asset'}]644        You can drill through single entity links to filter on fields or display linked fields.645        This is often called "deep linking" or using "dot syntax".646            .. seealso:: :ref:`filter_syntax`647            >>> # Find Versions created by Tasks in the Animation Pipeline Step648            >>> # -------------649            >>> fields = ['id', 'code']650            >>> pipeline_step_id = 2 # Animation Step ID651            >>> project_id = 4 # Demo Project652            >>> # you can drill through single-entity link fields653            >>> filters = [654            ...     ['project','is', {'type': 'Project','id': project_id}],655            ...     ['sg_task.Task.step.Step.id', 'is', pipeline_step_id]656            >>> ]657            >>> sg.find("Version", filters, fields)658            [{'code': 'scene_010_anim_v001', 'id': 42, 'type': 'Version'},659             {'code': 'scene_010_anim_v002', 'id': 134, 'type': 'Version'},660             {'code': 'bird_v001', 'id': 137, 'type': 'Version'},661             {'code': 'birdAltBlue_v002', 'id': 236, 'type': 'Version'}]662        :param str entity_type: Shotgun entity type to find.663        :param list filters: list of filters to apply to the query.664            .. seealso:: :ref:`filter_syntax`665        :param list fields: Optional list of fields to include in each entity record returned.666            Defaults to ``["id"]``.667        :param list order: Optional list of dictionaries defining how to order the results of the668            query. Each dictionary contains the ``field_name`` to order by and  the ``direction``669            to sort::670                [{'field_name':'foo', 'direction':'asc'}, {'field_name':'bar', 'direction':'desc'}]671            Defaults to sorting by ``id`` in ascending order.672        :param str filter_operator: Operator to apply to the filters. Supported values are ``"all"``673            and ``"any"``. These are just another way of defining if the query is an AND or OR674            query. Defaults to ``"all"``.675        :param int limit: Optional limit to the number of entities to return. Defaults to ``0`` which676            returns all entities that match.677        :param int page: Optional page of results to return. Use this together with the ``limit``678            parameter to control how your query results are paged. Defaults to ``0`` which returns679            all entities that match.680        :param bool retired_only: Optional boolean when ``True`` will return only entities that have681            been retried. Defaults to ``False`` which returns only entities which have not been682            retired. There is no option to return both retired and non-retired entities in the683            same query.684        :param bool include_archived_projects: Optional boolean flag to include entities whose projects685            have been archived. Defaults to ``True``.686        :param additional_filter_presets: Optional list of presets to further filter the result687            set, list has the form::688                [{"preset_name": <preset_name>, <optional_param1>: <optional_value1>, ... }]689            Note that these filters are ANDed together and ANDed with the 'filter'690            argument.691            For details on supported presets and the format of this parameter see692            :ref:`additional_filter_presets`693        :returns: list of dictionaries representing each entity with the requested fields, and the694            defaults ``"id"`` and ``"type"`` which are always included.695        :rtype: list696        """697        if not isinstance(limit, int) or limit < 0:698            raise ValueError("limit parameter must be a positive integer")699        if not isinstance(page, int) or page < 0:700            raise ValueError("page parameter must be a positive integer")701        if isinstance(filters, (list, tuple)):702            filters = _translate_filters(filters, filter_operator)703        elif filter_operator:704            # TODO: Not sure if this test is correct, replicated from prev api705            raise ShotgunError("Deprecated: Use of filter_operator for find()"706                " is not valid any more. See the documentation on find()")707        if not include_archived_projects:708            # This defaults to True on the server (no argument is sent)709            # So we only need to check the server version if it is False710            self.server_caps.ensure_include_archived_projects()711        if additional_filter_presets:712            self.server_caps.ensure_support_for_additional_filter_presets()713        params = self._construct_read_parameters(entity_type,714                                                 fields,715                                                 filters,716                                                 retired_only,717                                                 order,718                                                 include_archived_projects,719                                                 additional_filter_presets)720        if self.server_caps.ensure_return_image_urls_support():721            params['api_return_image_urls'] = True722        if self.server_caps.ensure_paging_info_without_counts_support():723            paging_info_param = "return_paging_info_without_counts"724        else:725            paging_info_param = "return_paging_info"726        params[paging_info_param] = False727        if limit and limit <= self.config.records_per_page:728            params["paging"]["entities_per_page"] = limit729            # If page isn't set and the limit doesn't require pagination,730            # then trigger the faster code path.731            if page == 0:732                page = 1733        # if page is specified, then only return the page of records requested734        if page != 0:735            params["paging"]["current_page"] = page736            records = self._call_rpc("read", params).get("entities", [])737            return self._parse_records(records)738        params[paging_info_param] = True739        records = []740        if self.server_caps.ensure_paging_info_without_counts_support():741            has_next_page = True742            while has_next_page:743                result = self._call_rpc("read", params)744                records.extend(result.get("entities"))745                if limit and len(records) >= limit:746                    records = records[:limit]747                    break748                has_next_page = result["paging_info"]["has_next_page"]749                params['paging']['current_page'] += 1750        else:751            result = self._call_rpc("read", params)752            while result.get("entities"):753                records.extend(result.get("entities"))754                if limit and len(records) >= limit:755                    records = records[:limit]756                    break757                if len(records) == result["paging_info"]["entity_count"]:758                    break759                params['paging']['current_page'] += 1760                result = self._call_rpc("read", params)761        return self._parse_records(records)762    def _construct_read_parameters(self,763                                   entity_type,764                                   fields,765                                   filters,766                                   retired_only,767                                   order,768                                   include_archived_projects,769                                   additional_filter_presets):770        params = {}771        params["type"] = entity_type772        params["return_fields"] = fields or ["id"]773        params["filters"] = filters774        params["return_only"] = (retired_only and 'retired') or "active"775        params["paging"] = { "entities_per_page": self.config.records_per_page,776                             "current_page": 1 }777        if additional_filter_presets:778            params["additional_filter_presets"] = additional_filter_presets;779        if include_archived_projects is False:780            # Defaults to True on the server, so only pass it if it's False781            params["include_archived_projects"] = False782        if order:783            sort_list = []784            for sort in order:785                if sort.has_key('column'):786                    # TODO: warn about deprecation of 'column' param name787                    sort['field_name'] = sort['column']788                sort.setdefault("direction", "asc")789                sort_list.append({790                    'field_name': sort['field_name'],791                    'direction' : sort['direction']792                })793            params['sorts'] = sort_list794        return params795    def _add_project_param(self, params, project_entity):796        if project_entity and self.server_caps.ensure_per_project_customization():797            params["project"] = project_entity798        return params799    def summarize(self,800                  entity_type,801                  filters,802                  summary_fields,803                  filter_operator=None,804                  grouping=None,805                  include_archived_projects=True):806        """807        Summarize field data returned by a query.808        This provides the same functionality as the summaries in the UI. You can specify one or809        more fields to summarize, choose the summary type for each, and optionally group the810        results which will return summary information for each group as well as the total for811        the query.812        **Example: Count all Assets for a Project**813        >>> sg.summarize(entity_type='Asset',814        ...              filters = [['project', 'is', {'type':'Project', 'id':4}]],815        ...              summary_fields=[{'field':'id', 'type':'count'}])816        {'groups': [], 'summaries': {'id': 15}}817        ``summaries`` contains the total summary for the query. Each key is the field summarized818        and the value is the result of the summary operation for the entire result set.819        .. note::820            You cannot perform more than one summary on a field at a time, but you can summarize821            several different fields in the same call.822        **Example: Count all Assets for a Project, grouped by sg_asset_type**823        >>> sg.summarize(entity_type='Asset',824        ...              filters=[['project', 'is', {'type': 'Project', 'id': 4}]],825        ...              summary_fields=[{'field': 'id', 'type': 'count'}],826        ...              grouping=[{'field': 'sg_asset_type', 'type': 'exact', 'direction': 'asc'}])827        {'groups': [{'group_name': 'Character','group_value': 'Character', 'summaries': {'id': 3}},828                    {'group_name': 'Environment','group_value': 'Environment', 'summaries': {'id': 3}},829                    {'group_name': 'Matte Painting', 'group_value': 'Matte Painting', 'summaries': {'id': 1}},830                    {'group_name': 'Prop', 'group_value': 'Prop', 'summaries': {'id': 4}},831                    {'group_name': 'Vehicle', 'group_value': 'Vehicle', 'summaries': {'id': 4}}],832         'summaries': {'id': 15}}833        - ``summaries`` contains the total summary for the query.834        - ``groups`` contains the summary for each group.835            - ``group_name`` is the display name for the group.836            - ``group_value`` is the actual value of the grouping value. This is often the same as837              ``group_name`` but in the case when grouping by entity, the ``group_name`` may be838              ``PuppyA`` and the group_value would be839              ``{'type':'Asset','id':922,'name':'PuppyA'}``.840            - ``summaries`` contains the summary calculation dict for each field requested.841        **Example: Count all Tasks for a Sequence and find the latest due_date**842        >>> sg.summarize(entity_type='Task',843        ...              filters = [844        ...                 ['entity.Shot.sg_sequence', 'is', {'type':'Sequence', 'id':2}],845        ...                 ['sg_status_list', 'is_not', 'na']],846        ...              summary_fields=[{'field':'id', 'type':'count'},847        ...                              {'field':'due_date','type':'latest'}])848        {'groups': [], 'summaries': {'due_date': '2013-07-05', 'id': 30}}849        This shows that the there are 30 Tasks for Shots in the Sequence and the latest ``due_date``850        of any Task is ``2013-07-05``.851        **Example: Count all Tasks for a Sequence, find the latest due_date and group by Shot**852        >>> sg.summarize(entity_type='Task',853        ...              filters = [854        ...                 ['entity.Shot.sg_sequence', 'is', {'type': 'Sequence', 'id': 2}],855        ...                 ['sg_status_list', 'is_not', 'na']],856        ...              summary_fields=[{'field': 'id', 'type': 'count'}, {'field': 'due_date', 'type': 'latest'}],857        ...              grouping=[{'field': 'entity', 'type': 'exact', 'direction': 'asc'}]))858        {'groups': [{'group_name': 'shot_010',859                     'group_value': {'id': 2, 'name': 'shot_010', 'type': 'Shot', 'valid': 'valid'},860                     'summaries': {'due_date': '2013-06-18', 'id': 10}},861                    {'group_name': 'shot_020',862                     'group_value': {'id': 3, 'name': 'shot_020', 'type': 'Shot', 'valid': 'valid'},863                     'summaries': {'due_date': '2013-06-28', 'id': 10}},864                    {'group_name': 'shot_030',865                     'group_value': {'id': 4, 'name': 'shot_030', 'type': 'Shot', 'valid': 'valid'},866                     'summaries': {'due_date': '2013-07-05', 'id': 10}}],867         'summaries': {'due_date': '2013-07-05', 'id': 30}}868        This shows that the there are 30 Tasks for Shots in the Sequence and the latest ``due_date``869        of any Task is ``2013-07-05``. Because the summary is grouped by ``entity``, we can also870        see the summaries for each Shot returned. Each Shot has 10 Tasks and the latest ``due_date``871        for each Shot. The difference between ``group_name`` and ``group_value`` is highlighted in872        this example as the name of the Shot is different from its value.873        **Example: Count all Tasks for a Sequence, find the latest due_date, group by Shot and874        Pipeline Step**875        >>> sg.summarize(entity_type='Task',876        ...                 filters = [877        ...                    ['entity.Shot.sg_sequence', 'is', {'type': 'Sequence', 'id': 2}],878        ...                    ['sg_status_list', 'is_not', 'na']],879        ...                 summary_fields=[{'field': 'id', 'type': 'count'},880        ...                                 {'field': 'due_date', 'type': 'latest'}],881        ...                 grouping=[{'field': 'entity', 'type': 'exact', 'direction': 'asc'},882        ...                           {'field': 'step', 'type': 'exact', 'direction': 'asc'}])883        {'groups': [{'group_name': 'shot_010',884                     'group_value': {'id': 2, 'name': 'shot_010', 'type': 'Shot', 'valid': 'valid'},885                     'groups': [{'group_name': 'Client',886                                 'group_value': {'id': 1, 'name': 'Client', 'type': 'Step', 'valid': 'valid'},887                                 'summaries': {'due_date': '2013-05-04', 'id': 1}},888                                {'group_name': 'Online',889                                 'group_value': {'id': 2, 'name': 'Online', 'type': 'Step', 'valid': 'valid'},890                                 'summaries': {'due_date': '2013-05-05', 'id': 1}},891                                ...892                                ... truncated for brevity893                                ...894                                {'group_name': 'Comp',895                                 'group_value': {'id': 8, 'name': 'Comp', 'type': 'Step', 'valid': 'valid'},896                                 'summaries': {'due_date': '2013-06-18', 'id': 1}}],897                     'summaries': {'due_date': '2013-06-18', 'id': 10}},898                    {'group_name': 'shot_020',899                     'group_value': {'id': 3, 'name': 'shot_020', 'type': 'Shot', 'valid': 'valid'},900                     'groups': [{'group_name': 'Client',901                                 'group_value': {'id': 1, 'name': 'Client', 'type': 'Step', 'valid': 'valid'},902                                 'summaries': {'due_date': '2013-05-15', 'id': 1}},903                                {'group_name': 'Online',904                                 'group_value': {'id': 2, 'name': 'Online', 'type': 'Step', 'valid': 'valid'},905                                 'summaries': {'due_date': '2013-05-16', 'id': 1}},906                                ...907                                ... truncated for brevity908                                ...909                                {'group_name': 'Comp',910                                 'group_value': {'id': 8, 'name': 'Comp', 'type': 'Step', 'valid': 'valid'},911                                 'summaries': {'due_date': '2013-06-28', 'id': 1}}],912                     'summaries': {'due_date': '2013-06-28', 'id': 10}},913                    {'group_name': 'shot_030',914                     'group_value': {'id': 4, 'name': 'shot_030', 'type': 'Shot', 'valid': 'valid'},915                     'groups': [{'group_name': 'Client',916                                 'group_value': {'id': 1, 'name': 'Client', 'type': 'Step', 'valid': 'valid'},917                                 'summaries': {'due_date': '2013-05-20', 'id': 1}},918                                {'group_name': 'Online',919                                 'group_value': {'id': 2, 'name': 'Online', 'type': 'Step', 'valid': 'valid'},920                                 'summaries': {'due_date': '2013-05-21', 'id': 1}},921                                ...922                                ... truncated for brevity923                                ...924                                {'group_name': 'Comp',925                                 'group_value': {'id': 8, 'name': 'Comp', 'type': 'Step', 'valid': 'valid'},926                                 'summaries': {'due_date': '2013-07-05', 'id': 1}}],927                     'summaries': {'due_date': '2013-07-05', 'id': 10}}],928        'summaries': {'due_date': '2013-07-05', 'id': 30}}929        When grouping my more than one field, the grouping structure is repeated for each sub-group930        and summary values are returned for each group on each level.931        :param str entity_type: The entity type to summarize932        :param list filters: A list of conditions used to filter the find query. Uses the same933            syntax as :meth:`~shotgun_api3.Shotgun.find` method.934        :param list summary_fields: A list of dictionaries with the following keys:935            :field: The internal Shotgun field name you are summarizing.936            :type: The type of summary you are performing on the field. Summary types can be any of937                ``record_count``, ``count``, ``sum``, ``maximum``, ``minimum``, ``average``,938                ``earliest``, ``latest``, ``percentage``, ``status_percentage``, ``status_list``,939                ``checked``, ``unchecked`` depending on the type of field you're summarizing.940        :param str filter_operator: Operator to apply to the filters. Supported values are ``"all"``941            and ``"any"``. These are just another way of defining if the query is an AND or OR942            query. Defaults to ``"all"``.943        :param list grouping: Optional list of dicts with the following keys:944                :field: a string indicating the internal Shotgun field name on ``entity_type`` to945                    group results by.946                :type: A string indicating the type of grouping to perform for each group.947                    Valid types depend on the type of field you are grouping on and can be one of948                    ``exact``, ``tens``, ``hundreds``, ``thousands``, ``tensofthousands``,949                    ``hundredsofthousands``, ``millions``, ``day``, ``week``, ``month``,950                    ``quarter``,``year``, ``clustered_date``, ``oneday``, ``fivedays``,951                    ``entitytype``, ``firstletter``.952                :direction: A string that sets the order to display the grouped results. Valid953                    options are ``asc`` and  ``desc``. Defaults to ``asc``.954        :returns: dictionary containing grouping and summaries keys.955        :rtype: dict956        """957        if not isinstance(grouping, list) and grouping is not None:958            msg = "summarize() 'grouping' parameter must be a list or None"959            raise ValueError(msg)960        if isinstance(filters, (list, tuple)):961            filters = _translate_filters(filters, filter_operator)962        if not include_archived_projects:963            # This defaults to True on the server (no argument is sent)964            # So we only need to check the server version if it is False965            self.server_caps.ensure_include_archived_projects()966        params = {"type": entity_type,967                  "summaries": summary_fields,968                  "filters": filters}969        if include_archived_projects is False:970            # Defaults to True on the server, so only pass it if it's False971            params["include_archived_projects"] = False972        if grouping is not None:973            params['grouping'] = grouping974        records = self._call_rpc('summarize', params)975        return records976    def create(self, entity_type, data, return_fields=None):977        """978        Create a new entity of the specified ``entity_type``.979            >>> data = {980            ...     "project": {"type": "Project", "id": 161},981            ...     "sg_sequence": {"type": "Sequence", "id": 109},982            ...     "code": "001_100",983            ...     'sg_status_list': "ip"984            ... }985            >>> sg.create('Shot', data)986            {'code': '001_100',987             'id': 2557,988             'project': {'id': 161, 'name': 'Pied Piper', 'type': 'Project'},989             'sg_sequence': {'id': 109, 'name': 'Sequence 001', 'type': 'Sequence'},990             'sg_status_list': 'ip',991             'type': 'Shot'}992        :param str entity_type: Shotgun entity type to create.993        :param dict data: Dictionary of fields and corresponding values to set on the new entity. If994            ``image`` or ``filmstrip_image`` fields are provided, the file path will be uploaded995            to the server automatically.996        :param list return_fields: Optional list of additional field values to return from the new997            entity. Defaults to ``id`` field.998        :returns: Shotgun entity dictionary containing the field/value pairs of all of the fields999            set from the ``data`` parameter as well as the defaults ``type`` and ``id``. If any1000            additional fields were provided using the ``return_fields`` parameter, these would be1001            included as well.1002        :rtype: dict1003        """1004        data = data.copy()1005        if not return_fields:1006            return_fields = ["id"]1007        upload_image = None1008        if 'image' in data:1009            upload_image = data.pop('image')1010        upload_filmstrip_image = None1011        if 'filmstrip_image' in data:1012            if not self.server_caps.version or self.server_caps.version < (3, 1, 0):1013                raise ShotgunError("Filmstrip thumbnail support requires server version 3.1 or "\1014                    "higher, server is %s" % (self.server_caps.version,))1015            upload_filmstrip_image = data.pop('filmstrip_image')1016        params = {1017            "type" : entity_type,1018            "fields" : self._dict_to_list(data),1019            "return_fields" : return_fields1020        }1021        record = self._call_rpc("create", params, first=True)1022        result = self._parse_records(record)[0]1023        if upload_image:1024            image_id = self.upload_thumbnail(entity_type, result['id'],1025                                             upload_image)1026            image = self.find_one(entity_type, [['id', 'is', result.get('id')]],1027                                  fields=['image'])1028            result['image'] = image.get('image')1029        if upload_filmstrip_image:1030            filmstrip_id = self.upload_filmstrip_thumbnail(entity_type, result['id'], upload_filmstrip_image)1031            filmstrip = self.find_one(entity_type,1032                                     [['id', 'is', result.get('id')]],1033                                     fields=['filmstrip_image'])1034            result['filmstrip_image'] = filmstrip.get('filmstrip_image')1035        return result1036    def update(self, entity_type, entity_id, data, multi_entity_update_modes=None):1037        """1038        Update the specified entity with the supplied data.1039        >>> shots = [1040        ...    {'type':'Shot', 'id':'40435'},1041        ...    {'type':'Shot', 'id':'40438'},1042        ...    {'type':'Shot', 'id':'40441'}]1043        >>> data = {1044        ...    'shots': shots_asset_is_in,1045        ...    'sg_status_list':'rev'}1046        >>> sg.update("Asset", 55, data)1047        {'type': 'Shot',1048         'id': 55,1049         'sg_status_`list`': 'rev',1050         'shots': [{'id': 40435, 'name': '100_010', 'type': 'Shot', 'valid': 'valid'},1051                   {'id': 40438, 'name': '100_040', 'type': 'Shot', 'valid': 'valid'},1052                   {'id': 40441, 'name': '100_070', 'type': 'Shot', 'valid': 'valid'}]1053        }1054        :param str entity_type: Entity type to update.1055        :param id entity_id: id of the entity to update.1056        :param dict data: key/value pairs where key is the field name and value is the value to set1057            for that field. This method does not restrict the updating of fields hidden in the web1058            UI via the Project Tracking Settings panel.1059        :param dict multi_entity_update_modes: Optional dict indicating what update mode to use1060            when updating a multi-entity link field. The keys in the dict are the fields to set1061            the mode for, and the values from the dict are one of ``set``, ``add``, or ``remove``.1062            Defaults to ``set``.1063            ::1064                multi_entity_update_modes={"shots": "add", "assets": "remove"}1065        :returns: Dictionary of the fields updated, with the default keys `type` and `id` added as well.1066        :rtype: dict1067        """1068        data = data.copy()1069        upload_image = None1070        if 'image' in data and data['image'] is not None:1071            upload_image = data.pop('image')1072        upload_filmstrip_image = None1073        if 'filmstrip_image' in data:1074            if not self.server_caps.version or self.server_caps.version < (3, 1, 0):1075                raise ShotgunError("Filmstrip thumbnail support requires server version 3.1 or "\1076                    "higher, server is %s" % (self.server_caps.version,))1077            upload_filmstrip_image = data.pop('filmstrip_image')1078        if data:1079            params = {1080                "type" : entity_type,1081                "id" : entity_id,1082                "fields" : self._dict_to_list(1083                    data,1084                    extra_data=self._dict_to_extra_data(1085                        multi_entity_update_modes, "multi_entity_update_mode"))1086            }1087            record = self._call_rpc("update", params)1088            result = self._parse_records(record)[0]1089        else:1090            result = {'id': entity_id, 'type': entity_type}1091        if upload_image:1092            image_id = self.upload_thumbnail(entity_type, entity_id,1093                                             upload_image)1094            image = self.find_one(entity_type, [['id', 'is', result.get('id')]],1095                                  fields=['image'])1096            result['image'] = image.get('image')1097        if upload_filmstrip_image:1098            filmstrip_id = self.upload_filmstrip_thumbnail(entity_type, result['id'], upload_filmstrip_image)1099            filmstrip = self.find_one(entity_type,1100                                     [['id', 'is', result.get('id')]],1101                                     fields=['filmstrip_image'])1102            result['filmstrip_image'] = filmstrip.get('filmstrip_image')1103        return result1104    def delete(self, entity_type, entity_id):1105        """1106        Retire the specified entity.1107        Entities in Shotgun are not "deleted" destructively, they are instead, "retired". This1108        means they are placed in the trash where they are no longer accessible to users.1109        The entity can be brought back to life using :meth:`~shotgun_api3.Shotgun.revive`.1110            >>> sg.delete("Shot", 2557)1111            True1112        :param str entity_type: Shotgun entity type to delete.1113        :param id entity_id: ``id`` of the entity to delete.1114        :returns: ``True`` if the entity was deleted, ``False`` otherwise (for example, if the1115            entity was already deleted).1116        :rtype: bool1117        :raises: :class:`Fault` if entity does not exist (deleted or not).1118       """1119        params = {1120            "type" : entity_type,1121            "id" : entity_id1122        }1123        return self._call_rpc("delete", params)1124    def revive(self, entity_type, entity_id):1125        """1126        Revive an entity that has previously been deleted.1127        >>> sg.revive("Shot", 860)1128        True1129        :param str entity_type: Shotgun entity type to revive.1130        :param int entity_id: id of the entity to revive.1131        :returns: ``True`` if the entity was revived, ``False`` otherwise (e.g. if the1132            entity is not currently retired).1133        :rtype: bool1134        """1135        params = {1136            "type" : entity_type,1137            "id" : entity_id1138        }1139        return self._call_rpc("revive", params)1140    def batch(self, requests):1141        """1142        Make a batch request of several :meth:`~shotgun_api3.Shotgun.create`,1143        :meth:`~shotgun_api3.Shotgun.update`, and :meth:`~shotgun_api3.Shotgun.delete` calls.1144        All requests are performed within a transaction, so either all will complete or none will.1145        Ex. Make a bunch of shots::1146            batch_data = []1147            for i in range(1,100):1148                data = {1149                    "code": "shot_%04d" % i,1150                    "project": project1151                }1152                batch_data.append({"request_type": "create", "entity_type": "Shot", "data": data})1153            sg.batch(batch_data)1154        Example output::1155             [{'code': 'shot_0001',1156               'type': 'Shot',1157               'id': 3624,1158               'project': {'id': 4, 'name': 'Demo Project', 'type': 'Project'}},1159              ...1160              ... and a bunch more ...1161              ...1162              {'code': 'shot_0099',1163               'type': 'Shot',1164               'id': 3722,1165               'project': {'id': 4, 'name': 'Demo Project', 'type': 'Project'}}]1166        Ex. All three types of requests in one batch::1167            batch_data = [1168              {"request_type": "create", "entity_type": "Shot", "data": {"code": "New Shot 1", "project": project}},1169              {"request_type": "update", "entity_type": "Shot", "entity_id": 3624, "data": {"code": "Changed 1"}},1170              {"request_type": "delete", "entity_type": "Shot", "entity_id": 3624}1171            ]1172            sg.batch(batch_data)1173        Example output::1174             [{'code': 'New Shot 1', 'type': 'Shot', 'id': 3723, 'project': {'id': 4, 'name': 'Demo Project', 'type': 'Project'}},1175              {'code': 'Changed 1', 'type': 'Shot', 'id': 3624},1176              True]1177        :param list requests: A list of dict's of the form which have a request_type key and also1178            specifies:1179            - create: ``entity_type``, data dict of fields to set1180            - update: ``entity_type``, ``entity_id``, data dict of fields to set, and optionally ``multi_entity_update_modes``1181            - delete: ``entity_type`` and entity_id1182        :returns: A list of values for each operation. Create and update requests return a dict of1183            the fields updated. Delete requests return ``True`` if the entity was deleted.1184        :rtype: list1185        """1186        if not isinstance(requests, list):1187            raise ShotgunError("batch() expects a list.  Instead was sent "\1188                "a %s" % type(requests))1189        # If we have no requests, just return an empty list immediately.1190        # Nothing to process means nothing to get results of.1191        if len(requests) == 0:1192            return []1193        calls = []1194        def _required_keys(message, required_keys, data):1195            missing = set(required_keys) - set(data.keys())1196            if missing:1197                raise ShotgunError("%s missing required key: %s. "\1198                    "Value was: %s." % (message, ", ".join(missing), data))1199        for req in requests:1200            _required_keys("Batched request",1201                           ['request_type', 'entity_type'],1202                           req)1203            request_params = {'request_type': req['request_type'],1204                              "type" : req["entity_type"]}1205            if req["request_type"] == "create":1206                _required_keys("Batched create request", ['data'], req)1207                request_params['fields'] = self._dict_to_list(req["data"])1208                request_params["return_fields"] = req.get("return_fields") or["id"]1209            elif req["request_type"] == "update":1210                _required_keys("Batched update request",1211                               ['entity_id', 'data'],1212                               req)1213                request_params['id'] = req['entity_id']1214                request_params['fields'] = self._dict_to_list(req["data"],1215                    extra_data=self._dict_to_extra_data(1216                        req.get("multi_entity_update_modes"),1217                        "multi_entity_update_mode"))1218                if "multi_entity_update_mode" in req:1219                    request_params['multi_entity_update_mode'] = req["multi_entity_update_mode"]1220            elif req["request_type"] == "delete":1221                _required_keys("Batched delete request", ['entity_id'], req)1222                request_params['id'] = req['entity_id']1223            else:1224                raise ShotgunError("Invalid request_type '%s' for batch" % (1225                                   req["request_type"]))1226            calls.append(request_params)1227        records = self._call_rpc("batch", calls)1228        return self._parse_records(records)1229    def work_schedule_read(self, start_date, end_date, project=None, user=None):1230        """1231        Return the work day rules for a given date range.1232        .. versionadded:: 3.0.91233            Requires Shotgun server v3.2.0+1234        This returns the defined WorkDayRules between the ``start_date`` and ``end_date`` inclusive1235        as a dict where the key is the date and the value is another dict describing the rule for1236        that date.1237        Rules are represented by a dict with the following keys:1238        :description: the description entered into the work day rule exception if applicable.1239        :reason: one of six options:1240            - STUDIO_WORK_WEEK: standard studio schedule applies1241            - STUDIO_EXCEPTION: studio-wide exception applies1242            - PROJECT_WORK_WEEK: standard project schedule applies1243            - PROJECT_EXCEPTION: project-specific exception applies1244            - USER_WORK_WEEK: standard user work week applies1245            - USER_EXCEPTION: user-specific exception applies1246        :working: boolean indicating whether it is a "working" day or not.1247        >>> sg.work_schedule_read("2015-12-21", "2015-12-25")1248        {'2015-12-21': {'description': None,1249                        'reason': 'STUDIO_WORK_WEEK',1250                        'working': True},1251         '2015-12-22': {'description': None,1252                        'reason': 'STUDIO_WORK_WEEK',1253                        'working': True},1254         '2015-12-23': {'description': None,1255                        'reason': 'STUDIO_WORK_WEEK',1256                        'working': True},1257         '2015-12-24': {'description': 'Closed for Christmas Eve',1258                        'reason': 'STUDIO_EXCEPTION',1259                        'working': False},1260         '2015-12-25': {'description': 'Closed for Christmas',1261                        'reason': 'STUDIO_EXCEPTION',1262                        'working': False}}1263        :param str start_date: Start date of date range. ``YYYY-MM-DD``1264        :param str end_date: End date of date range. ``YYYY-MM-DD``1265        :param dict project: Optional Project entity to query `WorkDayRules` for.1266        :param dict user: Optional HumanUser entity to query WorkDayRules for.1267        :returns: Complex dict containing each date and the WorkDayRule defined for that date1268            between the ``start_date`` and ``end date`` inclusive. See above for details.1269        :rtype: dict1270        """1271        if not self.server_caps.version or self.server_caps.version < (3, 2, 0):1272            raise ShotgunError("Work schedule support requires server version 3.2 or "\1273                "higher, server is %s" % (self.server_caps.version,))1274        if not isinstance(start_date, str) or not isinstance(end_date, str):1275            raise ShotgunError("The start_date and end_date arguments must be strings in YYYY-MM-DD format")1276        params = dict(1277            start_date=start_date,1278            end_date=end_date,1279            project=project,1280            user=user1281        )1282        return self._call_rpc('work_schedule_read', params)1283    def work_schedule_update(self, date, working, description=None, project=None, user=None,1284                             recalculate_field=None):1285        """1286        Update the work schedule for a given date.1287        .. versionadded:: 3.0.91288            Requires Shotgun server v3.2.0+1289        If neither ``project`` nor ``user`` are passed in, the studio work schedule will be updated.1290        ``project`` and ``user`` can only be used exclusively of each other.1291        >>> sg.work_schedule_update ("2015-12-31", working=False,1292        ...                          description="Studio closed for New Years Eve", project=None,1293        ...                          user=None, recalculate_field=None)1294        {'date': '2015-12-31',1295         'description': "Studio closed for New Years Eve",1296         'project': None,1297         'user': None,1298         'working': False}1299        :param str date: Date of WorkDayRule to update. ``YYY-MM-DD``1300        :param bool working: Indicates whether the day is a working day or not.1301        :param str description: Optional reason for time off.1302        :param dict project: Optional Project entity to assign the rule to. Cannot be used with the1303            ``user`` param.1304        :param dict user: Optional HumanUser entity to assign the rule to. Cannot be used with the1305            ``project`` param.1306        :param str recalculate_field: Optional schedule field that will be recalculated on Tasks1307            when they are affected by a change in working schedule. Options are ``due_date`` or1308            ``duration``. Defaults to the value set in the Shotgun web application's Site1309            Preferences.1310        :returns: dict containing key/value pairs for each value of the work day rule updated.1311        :rtype: dict1312        """1313        if not self.server_caps.version or self.server_caps.version < (3, 2, 0):1314            raise ShotgunError("Work schedule support requires server version 3.2 or "\1315                "higher, server is %s" % (self.server_caps.version,))1316        if not isinstance(date, str):1317            raise ShotgunError("The date argument must be string in YYYY-MM-DD format")1318        params = dict(1319            date=date,1320            working=working,1321            description=description,1322            project=project,1323            user=user,1324            recalculate_field=recalculate_field1325        )1326        return self._call_rpc('work_schedule_update', params)1327    def follow(self, user, entity):1328        """1329        Add the entity to the user's followed entities.1330        If the user is already following the entity, the method will succeed but nothing will be1331        changed on the server-side.1332            >>> sg.follow({"type": "HumanUser", "id": 42}, {"type": "Shot", "id": 2050})1333            {'followed': True, 'user': {'type': 'HumanUser', 'id': 42},1334             'entity': {'type': 'Shot', 'id': 2050}}1335        :param dict user: User entity that will follow the entity.1336        :param dict entity: Shotgun entity to be followed.1337        :returns: dict with ``"followed": True`` as well as key/values for the params that were1338            passed in.1339        :rtype: dict1340        """1341        if not self.server_caps.version or self.server_caps.version < (5, 1, 22):1342            raise ShotgunError("Follow support requires server version 5.2 or "\1343                "higher, server is %s" % (self.server_caps.version,))1344        params = dict(1345            user=user,1346            entity=entity1347        )1348        return self._call_rpc('follow', params)1349    def unfollow(self, user, entity):1350        """1351        Remove entity from the user's followed entities.1352        This does nothing if the user is not following the entity.1353        >>> sg.unfollow({"type": "HumanUser", "id": 42}, {"type": "Shot", "id": 2050})1354        {'entity': {'type': 'Shot', 'id': 2050}, 'user': {'type': 'HumanUser', 'id': 42},1355         'unfollowed': True}1356        :param dict user: User entity that will unfollow the entity.1357        :param dict entity: Entity to be unfollowed1358        :returns: dict with ``"unfollowed": True`` as well as key/values for the params that were1359            passed in.1360        :rtype: dict1361        """1362        if not self.server_caps.version or self.server_caps.version < (5, 1, 22):1363            raise ShotgunError("Follow support requires server version 5.2 or "\1364                "higher, server is %s" % (self.server_caps.version,))1365        params = dict(1366            user=user,1367            entity=entity1368        )1369        return self._call_rpc('unfollow', params)1370    def followers(self, entity):1371        """1372        Return all followers for an entity.1373            >>> sg.followers({"type": "Shot", "id": 2050})1374            [{'status': 'act', 'valid': 'valid', 'type': 'HumanUser', 'name': 'Richard Hendriks',1375              'id': 42},1376             {'status': 'act', 'valid': 'valid', 'type': 'HumanUser', 'name': 'Bertram Gilfoyle',1377              'id': 33},1378             {'status': 'act', 'valid': 'valid', 'type': 'HumanUser', 'name': 'Dinesh Chugtai',1379              'id': 57}]1380        :param dict entity: Entity to find followers of.1381        :returns: list of dicts representing each user following the entity1382        :rtype: list1383        :versionadded:1384        """1385        if not self.server_caps.version or self.server_caps.version < (5, 1, 22):1386            raise ShotgunError("Follow support requires server version 5.2 or "\1387                "higher, server is %s" % (self.server_caps.version,))1388        params = dict(1389            entity=entity1390        )1391        return self._call_rpc('followers', params)1392    def following(self, user, project=None, entity_type=None):1393        """1394        Return all entity instances a user is following.1395        Optionally, a project and/or entity_type can be supplied to restrict returned results.1396            >>> user = {"type": "HumanUser", "id": 1234}1397            >>> project = {"type": "Project", "id": 1234}1398            >>> entity_type = "Task"1399            >>> sg.following(user, project=project, entity_type=entity_type)1400            [{"type":"Task", "id":1},1401             {"type":"Task", "id":2},1402             {"type":"Task", "id":3}]1403        :param dict user: Find what this person is following.1404        :param dict project: Optional filter to only return results from a specific project.1405        :param str entity_type: Optional filter to only return results from one entity type.1406        :returns: list of dictionaries, each containing entity type & id's being followed.1407        :rtype: list1408        """1409        self.server_caps.ensure_user_following_support()1410        params = {1411            "user":user1412        }1413        if project:1414            params["project"] = project1415        if entity_type:1416            params["entity_type"] = entity_type1417        return self._call_rpc('following', params)1418    def schema_entity_read(self, project_entity=None):1419        """1420        Return all active entity types, their display names, and their visibility.1421        If the project parameter is specified, the schema visibility for the given project is1422        being returned. If the project parameter is omitted or set to ``None``, a full listing is1423        returned where per-project entity type visibility settings are not considered.1424        >>> sg.schema_entity_read()1425        {'ActionMenuItem': {'name': {'editable': False, 'value': 'Action Menu Item'},1426                            'visible': {'editable': False, 'value': True}},1427         'ApiUser': {'name': {'editable': False, 'value': 'Script'},1428                     'visible': {'editable': False, 'value': True}},1429         'AppWelcomeUserConnection': {'name': {'editable': False,1430                                               'value': 'App Welcome User Connection'},1431                                      'visible': {'editable': False, 'value': True}},1432         'Asset': {'name': {'editable': False, 'value': 'Asset'},1433                   'visible': {'editable': False, 'value': True}},1434         'AssetAssetConnection': {'name': {'editable': False,1435                                           'value': 'Asset Asset Connection'},1436                                  'visible': {'editable': False, 'value': True}},1437         '...'1438        }1439        :param dict project_entity: Optional Project entity specifying which project to return1440            the listing for. If omitted or set to ``None``, per-project visibility settings are1441            not taken into consideration and the global list is returned. Example:1442            ``{'type': 'Project', 'id': 3}``1443        :returns: dict of Entity Type to dict containing the display name.1444        :rtype: dict1445        """1446        params = {}1447        params = self._add_project_param(params, project_entity)1448        if params:1449            return self._call_rpc("schema_entity_read", params)1450        else:1451            return self._call_rpc("schema_entity_read", None)1452    def schema_read(self, project_entity=None):1453        """1454        Get the schema for all fields on all entities.1455        .. note::1456            If ``project_entity`` is not specified, everything is reported as visible.1457        >>> sg.schema_read()1458        {'ActionMenuItem': {'created_at': {'data_type': {'editable': False, 'value': 'date_time'},1459                                           'description': {'editable': True,  'value': ''},1460                                           'editable': {'editable': False, 'value': False},1461                                           'entity_type': {'editable': False, 'value': 'ActionMenuItem'},1462                                           'mandatory': {'editable': False, 'value': False},1463                                           'name': {'editable': True, 'value': 'Date Created'},1464                                           'properties': {'default_value': {'editable': False, 'value': None},1465                                                          'summary_default': {'editable': True, 'value': 'none'}},1466                                           'unique': {'editable': False, 'value': False},1467                                           'visible': {'editable': False, 'value': True}},1468                            'created_by': {'data_type': {'editable': False,'value': 'entity'},1469                                           'description': {'editable': True,'value': ''},1470                                           'editable': {'editable': False,'value': False},1471                                           'entity_type': {'editable': False,'value': 'ActionMenuItem'},1472                                           'mandatory': {'editable': False,'value': False},1473                                           'name': {'editable': True,'value': 'Created by'},1474                                           'properties': {'default_value': {'editable': False,'value': None},1475                                                          'summary_default': {'editable': True,'value': 'none'},1476                                                          'valid_types': {'editable': True,'value': ['HumanUser','ApiUser']}},1477                                           'unique': {'editable': False,'value': False},1478                                           'visible': {'editable': False,'value': True}},1479                            ...1480                            ...1481         ...1482         ...1483         'Version': {'client_approved': {'data_type': {'editable': False,'value': 'checkbox'},1484                                         'description': {'editable': True,'value': ''},1485                                         'editable': {'editable': False,'value': True},1486                                         'entity_type': {'editable': False,'value': 'Version'},1487                                         'mandatory': {'editable': False,'value': False},1488                                         'name': {'editable': True,'value': 'Client Approved'},1489                                         'properties': {'default_value': {'editable': False,'value': False},1490                                                        'summary_default': {'editable': False,'value': 'none'}},1491                                         'unique': {'editable': False,'value': False},1492                                         'visible': {'editable': False,'value': True}},1493                     ...1494                     ...1495         ...1496         ...1497        }1498        :param dict project_entity: Optional, Project entity specifying which project to return1499            the listing for. If omitted or set to ``None``, per-project visibility settings are1500            not taken into consideration and the global list is returned. Example:1501            ``{'type': 'Project', 'id': 3}``. Defaults to ``None``.1502        :returns: A nested dict object containing a key/value pair for all fields of all entity1503            types. Properties that are ``'editable': True``, can be updated using the1504            :meth:`~shotgun_api3.Shotgun.schema_field_update` method.1505        :rtype: dict1506        """1507        params = {}1508        params = self._add_project_param(params, project_entity)1509        if params:1510            return self._call_rpc("schema_read", params)1511        else:1512            return self._call_rpc("schema_read", None)1513    def schema_field_read(self, entity_type, field_name=None, project_entity=None):1514        """1515        Get schema for all fields on the specified entity type or just the field name specified1516        if provided.1517        .. note::1518            Unlike how the results of a :meth:`~shotgun_api3.Shotgun.find` can be pumped into a1519            :meth:`~shotgun_api3.Shotgun.create` or :meth:`~shotgun_api3.Shotgun.update`, the1520            results of :meth:`~shotgun_api3.Shotgun.schema_field_read` are not compatible with1521            the format used for :meth:`~shotgun_api3.Shotgun.schema_field_create` or1522            :meth:`~shotgun_api3.Shotgun.schema_field_update`. If you need to pipe the results1523            from :meth:`~shotgun_api3.Shotgun.schema_field_read` into a1524            :meth:`~shotgun_api3.Shotgun.schema_field_create` or1525            :meth:`~shotgun_api3.Shotgun.schema_field_update`, you will need to reformat the1526            data in your script.1527        .. note::1528            If you don't specify a ``project_entity``, everything is reported as visible.1529        >>> sg.schema_field_read('Asset', 'shots')1530        {'shots': {'data_type': {'editable': False, 'value': 'multi_entity'},1531                   'description': {'editable': True, 'value': ''},1532                   'editable': {'editable': False, 'value': True},1533                   'entity_type': {'editable': False, 'value': 'Asset'},1534                   'mandatory': {'editable': False, 'value': False},1535                   'name': {'editable': True, 'value': 'Shots'},1536                   'properties': {'default_value': {'editable': False,1537                                                    'value': None},1538                                  'summary_default': {'editable': True,1539                                                      'value': 'none'},1540                                  'valid_types': {'editable': True,1541                                                  'value': ['Shot']}},1542                   'unique': {'editable': False, 'value': False},1543                   'visible': {'editable': False, 'value': True}}}1544        :param str entity_type: Entity type to get the schema for.1545        :param str field_name: Optional internal Shotgun name of the field to get the schema1546            definition for. If this parameter is excluded or set to ``None``, data structures of1547            all fields will be returned. Defaults to ``None``. Example: ``sg_temp_field``.1548        :param dict project_entity: Optional Project entity specifying which project to return1549            the listing for. If omitted or set to ``None``, per-project visibility settings are1550            not taken into consideration and the global list is returned. Example:1551            ``{'type': 'Project', 'id': 3}``1552        :returns: a nested dict object containing a key/value pair for the ``field_name`` specified1553            and its properties, or if no field_name is specified, for all the fields of the1554            ``entity_type``. Properties that are ``'editable': True``, can be updated using the1555            :meth:`~shotgun_api3.Shotgun.schema_field_update` method.1556        :rtype: dict1557        """1558        params = {1559            "type": entity_type,1560        }1561        if field_name:1562            params["field_name"] = field_name1563        params = self._add_project_param(params, project_entity)1564        return self._call_rpc("schema_field_read", params)1565    def schema_field_create(self, entity_type, data_type, display_name, properties=None):1566        """1567        Create a field for the specified entity type.1568        .. note::1569            If the internal Shotgun field name computed from the provided ``display_name`` already1570            exists, the internal Shotgun field name will automatically be appended with ``_1`` in1571            order to create a unique name. The integer suffix will be incremented by 1 until a1572            unique name is found.1573        >>> properties = {"summary_default": "count", "description": "Complexity breakdown of Asset"}1574        >>> sg.schema_field_create("Asset", "text", "Complexity", properties)1575        'sg_complexity'1576        :param str entity_type: Entity type to add the field to.1577        :param str data_type: Shotgun data type for the new field.1578        :param str display_name: Specifies the display name of the field you are creating. The1579            system name will be created from this display name and returned upon successful1580            creation.1581        :param dict properties: Dict of valid properties for the new field. Use this to specify1582            other field properties such as the 'description' or 'summary_default'.1583        :returns: The internal Shotgun name for the new field, this is different to the1584            ``display_name`` parameter passed in.1585        :rtype: str1586        """1587        params = {1588            "type" : entity_type,1589            "data_type" : data_type,1590            "properties" : [1591                {'property_name': 'name', 'value': display_name}1592            ]1593        }1594        params["properties"].extend(self._dict_to_list(properties,1595            key_name="property_name", value_name="value"))1596        return self._call_rpc("schema_field_create", params)1597    def schema_field_update(self, entity_type, field_name, properties):1598        """1599        Update the properties for the specified field on an entity.1600        .. note::1601            Although the property name may be the key in a nested dictionary, like1602            'summary_default', it is treated no differently than keys that are up1603            one level, like 'description'.1604        >>> properties = {"name": "Test Number Field Renamed", "summary_default": "sum",1605        ...               "description": "this is only a test"}1606        >>> sg.schema_field_update("Asset", "sg_test_number", properties)1607        True1608        :param entity_type: Entity type of field to update.1609        :param field_name: Internal Shotgun name of the field to update.1610        :param properties: Dictionary with key/value pairs where the key is the property to be1611            updated and the value is the new value.1612        :returns: ``True`` if the field was updated.1613        :rtype: bool1614        """1615        params = {1616            "type" : entity_type,1617            "field_name" : field_name,1618            "properties": [1619                {"property_name" : k, "value" : v}1620                for k, v in (properties or {}).iteritems()1621            ]1622        }1623        return self._call_rpc("schema_field_update", params)1624    def schema_field_delete(self, entity_type, field_name):1625        """1626        Delete the specified field from the entity type.1627        >>> sg.schema_field_delete("Asset", "sg_temp_field")1628        True1629        :param str entity_type: Entity type to delete the field from.1630        :param str field_name: Internal Shotgun name of the field to delete.1631        :returns: ``True`` if the field was deleted.1632        :rtype: bool1633        """1634        params = {1635            "type" : entity_type,1636            "field_name" : field_name1637        }1638        return self._call_rpc("schema_field_delete", params)1639    def add_user_agent(self, agent):1640        """1641        Add agent to the user-agent header.1642        Appends agent to the user-agent string sent with every API request.1643        >>> sg.add_user_agent("my_tool 1.0")1644        :param str agent: string to append to user-agent.1645        """1646        self._user_agents.append(agent)1647    def reset_user_agent(self):1648        """1649        Reset user agent to the default value.1650        Example default user-agent::1651            shotgun-json (3.0.17); Python 2.6 (Mac); ssl OpenSSL 1.0.2d 9 Jul 2015 (validate)1652        """1653        ua_platform = "Unknown"1654        if self.client_caps.platform is not None:1655            ua_platform = self.client_caps.platform.capitalize()1656        # create ssl validation string based on settings1657        validation_str = "validate"1658        if self.config.no_ssl_validation:1659            validation_str = "no-validate"1660        self._user_agents = ["shotgun-json (%s)" % __version__,1661                             "Python %s (%s)" % (self.client_caps.py_version, ua_platform),1662                             "ssl %s (%s)" % (self.client_caps.ssl_version, validation_str)]1663    def set_session_uuid(self, session_uuid):1664        """1665        Set the browser session_uuid in the current Shotgun API instance.1666        When this is set, any events generated by the API will include the ``session_uuid`` value1667        on the corresponding EventLogEntries. If there is a current browser session open with1668        this ``session_uuid``, the browser will display updates for these events.1669        >>> sg.set_session_uuid("5a1d49b0-0c69-11e0-a24c-003048d17544")1670        :param str session_uuid: The uuid of the browser session to be updated.1671        """1672        self.config.session_uuid = session_uuid1673        return1674    def share_thumbnail(self, entities, thumbnail_path=None, source_entity=None,1675        filmstrip_thumbnail=False, **kwargs):1676        """1677        Associate a thumbnail with more than one Shotgun entity.1678        .. versionadded:: 3.0.91679            Requires Shotgun server v4.0.0+1680        Share the thumbnail from between entities without requiring uploading the thumbnail file1681        multiple times. You can use this in two ways:1682        1) Upload an image to set as the thumbnail on multiple entities.1683        2) Update multiple entities to point to an existing entity's thumbnail.1684        .. note::1685            When sharing a filmstrip thumbnail, it is required to have a static thumbnail in1686            place before the filmstrip will be displayed in the Shotgun web UI.1687        >>> thumb = '/data/show/ne2/100_110/anim/01.mlk-02b.jpg'1688        >>> e = [{'type': 'Version', 'id': 123}, {'type': 'Version', 'id': 456}]1689        >>> sg.share_thumbnail(entities=e, thumbnail_path=thumb)1690        42711691        >>> e = [{'type': 'Version', 'id': 123}, {'type': 'Version', 'id': 456}]1692        >>> sg.share_thumbnail(entities=e, source_entity={'type':'Version', 'id': 789})1693        42711694        :param list entities: The entities to update to point to the shared  thumbnail provided in1695            standard entity dict format::1696                [{'type': 'Version', 'id': 123},1697                 {'type': 'Version', 'id': 456}]1698        :param str thumbnail_path: The full path to the local thumbnail file to upload and share.1699            Required if ``source_entity`` is not provided.1700        :param dict source_entity: The entity whos thumbnail will be the source for sharing.1701            Required if ``source_entity`` is not provided.1702        :param bool filmstrip_thumbnail: ``True`` to share the filmstrip thumbnail. ``False`` to1703            share the static thumbnail. Defaults to ``False``.1704        :returns: ``id`` of the Attachment entity representing the source thumbnail that is shared.1705        :rtype: int1706        """1707        if not self.server_caps.version or self.server_caps.version < (4, 0, 0):1708            raise ShotgunError("Thumbnail sharing support requires server "\1709                "version 4.0 or higher, server is %s" % (self.server_caps.version,))1710        if not isinstance(entities, list) or len(entities) == 0:1711            raise ShotgunError("'entities' parameter must be a list of entity "\1712                "hashes and may not be empty")1713        for e in entities:1714            if not isinstance(e, dict) or 'id' not in e or 'type' not in e:1715                raise ShotgunError("'entities' parameter must be a list of "\1716                    "entity hashes with at least 'type' and 'id' keys.\nInvalid "\1717                    "entity: %s" % e)1718        if (not thumbnail_path and not source_entity) or \1719            (thumbnail_path and source_entity):1720            raise ShotgunError("You must supply either thumbnail_path OR "\1721                "source_entity.")1722        # upload thumbnail1723        if thumbnail_path:1724            source_entity = entities.pop(0)1725            if filmstrip_thumbnail:1726                thumb_id = self.upload_filmstrip_thumbnail(source_entity['type'],1727                    source_entity['id'], thumbnail_path, **kwargs)1728            else:1729                thumb_id = self.upload_thumbnail(source_entity['type'],1730                    source_entity['id'], thumbnail_path, **kwargs)1731        else:1732            if not isinstance(source_entity, dict) or 'id' not in source_entity \1733                or 'type' not in source_entity:1734                raise ShotgunError("'source_entity' parameter must be a dict "\1735                    "with at least 'type' and 'id' keys.\nGot: %s (%s)" \1736                    % (source_entity, type(source_entity)))1737        # only 1 entity in list and we already uploaded the thumbnail to it1738        if len(entities) == 0:1739            return thumb_id1740        # update entities with source_entity thumbnail1741        entities_str = []1742        for e in entities:1743            entities_str.append("%s_%s" % (e['type'], e['id']))1744        # format for post request1745        if filmstrip_thumbnail:1746            filmstrip_thumbnail = 11747        params = {1748            "entities" : ','.join(entities_str),1749            "source_entity": "%s_%s" % (source_entity['type'], source_entity['id']),1750            "filmstrip_thumbnail" : filmstrip_thumbnail,1751        }1752        url = urlparse.urlunparse((self.config.scheme, self.config.server,1753            "/upload/share_thumbnail", None, None, None))1754        result = self._send_form(url, params)1755        if not str(result).startswith("1"):1756            raise ShotgunError("Unable to share thumbnail: %s" % result)1757        else:1758            # clearing thumbnail returns no attachment_id1759            try:1760                attachment_id = int(str(result).split(":")[1].split("\n")[0])1761            except ValueError:1762                attachment_id = None1763        return attachment_id1764    def upload_thumbnail(self, entity_type, entity_id, path, **kwargs):1765        """1766        Upload a file from a local path and assign it as the thumbnail for the specified entity.1767        .. note::1768            Images will automatically be re-sized on the server to generate a size-appropriate image1769            file. However, the original file is retained as well and is accessible when you click1770            on the thumbnail image in the web UI. If you are using a local install of Shotgun and1771            have not enabled S3, this can eat up disk space if you're uploading really large source1772            images for your thumbnails.1773        You can un-set (aka clear) a thumbnail on an entity using the1774        :meth:`~shotgun_api3.Shotgun.update` method and setting the **image** field to ``None``.1775        This will also unset the ``filmstrip_thumbnail`` field if it is set.1776        Supported image file types include ``.jpg` and ``.png`` (preferred) but will also accept.1777        ``.gif```, ``.tif``, ``.tiff``, ``.bmp``, ``.exr``, ``.dpx``, and ``.tga``.1778        This method wraps over :meth:`~shotgun_api3.Shotgun.upload`. Additional keyword arguments1779        passed to this method will be forwarded to the :meth:`~shotgun_api3.Shotgun.upload` method.1780        :param str entity_type: Entity type to set the thumbnail for.1781        :param int entity_id: Id of the entity to set the thumbnail for.1782        :param str path: Full path to the thumbnail file on disk.1783        :returns: Id of the new attachment1784        """1785        return self.upload(entity_type, entity_id, path,1786            field_name="thumb_image", **kwargs)1787    def upload_filmstrip_thumbnail(self, entity_type, entity_id, path, **kwargs):1788        """1789        Upload filmstrip thumbnail to specified entity.1790        .. versionadded:: 3.0.91791            Requires Shotgun server v3.1.0+1792        Uploads a file from a local directory and assigns it as the filmstrip thumbnail for the1793        specified entity. The image must be a horizontal strip of any number of frames that are1794        exactly 240 pixels wide. Therefore the whole strip must be an exact multiple of 240 pixels1795        in width. The height can be anything (and will depend on the aspect ratio of the frames).1796        Any image file type that works for thumbnails will work for filmstrip thumbnails.1797        Filmstrip thumbnails will only be visible in the Thumbnail field on an entity if a1798        regular thumbnail image is also uploaded to the entity. The standard thumbnail is1799        displayed by default as the poster frame. Then, on hover, the filmstrip thumbnail is1800        displayed and updated based on your horizontal cursor position for scrubbing. On mouseout,1801        the default thumbnail is displayed again as the poster frame.1802        The url for a filmstrip thumbnail on an entity is available by querying for the1803        ``filmstrip_image field``.1804        You can un-set (aka clear) a thumbnail on an entity using the1805        :meth:`~shotgun_api3.Shotgun.update` method and setting the **image** field to ``None``.1806        This will also unset the ``filmstrip_thumbnail`` field if it is set.1807        This method wraps over :meth:`~shotgun_api3.Shotgun.upload`. Additional keyword arguments1808        passed to this method will be forwarded to the :meth:`~shotgun_api3.Shotgun.upload` method.1809        >>> filmstrip_thumbnail = '/data/show/ne2/100_110/anim/01.mlk-02b_filmstrip.jpg'1810        >>> sg.upload_filmstrip_thumbnail("Version", 27, filmstrip_thumbnail)1811        871812        :param str entity_type: Entity type to set the filmstrip thumbnail for.1813        :param int entity_id: Id of the entity to set the filmstrip thumbnail for.1814        :param str path: Full path to the filmstrip thumbnail file on disk.1815        :returns: Id of the new Attachment entity created for the filmstrip thumbnail1816        :rtype: int1817        """1818        if not self.server_caps.version or self.server_caps.version < (3, 1, 0):1819            raise ShotgunError("Filmstrip thumbnail support requires server version 3.1 or "\1820                "higher, server is %s" % (self.server_caps.version,))1821        return self.upload(entity_type, entity_id, path,1822            field_name="filmstrip_thumb_image", **kwargs)1823    def upload(self, entity_type, entity_id, path, field_name=None, display_name=None,1824               tag_list=None):1825        """1826        Upload a file to the specified entity.1827        Creates an Attachment entity for the file in Shotgun and links it to the specified entity.1828        You can optionally store the file in a field on the entity, change the display name, and1829        assign tags to the Attachment.1830        >>> mov_file = '/data/show/ne2/100_110/anim/01.mlk-02b.mov'1831        >>> sg.upload("Shot", 423, mov_file, field_name="sg_latest_quicktime",1832        ...           display_name="Latest QT")1833        721834        :param str entity_type: Entity type to link the upload to.1835        :param int entity_id: Id of the entity to link the upload to.1836        :param str path: Full path to an existing non-empty file on disk to upload.1837        :param str field_name: The internal Shotgun field name on the entity to store the file in.1838            This field must be a File/Link field type.1839        :param str display_name: The display name to use for the file. Defaults to the file name.1840        :param str tag_list: comma-separated string of tags to assign to the file.1841        :returns: Id of the Attachment entity that was created for the image.1842        :rtype: int1843        """1844        # Basic validations of the file to upload.1845        path = os.path.abspath(os.path.expanduser(path or ""))1846        if not os.path.isfile(path):1847            raise ShotgunError("Path must be a valid file, got '%s'" % path)1848        if os.path.getsize(path) == 0:1849            raise ShotgunError("Path cannot be an empty file: '%s'" % path)1850        is_thumbnail = (field_name in ["thumb_image", "filmstrip_thumb_image", "image",1851                                       "filmstrip_image"])1852        # Version.sg_uploaded_movie is handled as a special case and uploaded1853        # directly to Cloud storage1854        if self.server_info.get("s3_direct_uploads_enabled", False) \1855                and entity_type == "Version" and field_name == "sg_uploaded_movie":1856            return self._upload_to_storage(entity_type, entity_id, path, field_name, display_name,1857                                           tag_list, is_thumbnail)1858        else:1859            return self._upload_to_sg(entity_type, entity_id, path, field_name, display_name,1860                                      tag_list, is_thumbnail)1861    def _upload_to_storage(self, entity_type, entity_id, path, field_name, display_name,1862                           tag_list, is_thumbnail):1863        """1864        Internal function to upload a file to the Cloud storage and link it to the specified entity.1865        :param str entity_type: Entity type to link the upload to.1866        :param int entity_id: Id of the entity to link the upload to.1867        :param str path: Full path to an existing non-empty file on disk to upload.1868        :param str field_name: The internal Shotgun field name on the entity to store the file in.1869            This field must be a File/Link field type.1870        :param str display_name: The display name to use for the file. Defaults to the file name.1871        :param str tag_list: comma-separated string of tags to assign to the file.1872        :param bool is_thumbnail: indicates if the attachment is a thumbnail.1873        :returns: Id of the Attachment entity that was created for the image.1874        :rtype: int1875        """1876        filename = os.path.basename(path)1877        # Step 1: get the upload url1878        is_multipart_upload = (os.path.getsize(path)  > self._MULTIPART_UPLOAD_CHUNK_SIZE)1879        upload_info = self._get_attachment_upload_info(is_thumbnail, filename, is_multipart_upload)1880        # Step 2: upload the file1881        # We upload large files in multiple parts because it is more robust1882        # (and required when using S3 storage)1883        if is_multipart_upload:1884            self._multipart_upload_file_to_storage(path, upload_info)1885        else:1886            self._upload_file_to_storage(path, upload_info["upload_url"])1887        # Step 3: create the attachment1888        url = urlparse.urlunparse((self.config.scheme, self.config.server,1889                                   "/upload/api_link_file", None, None, None))1890        params = {1891            "entity_type" : entity_type,1892            "entity_id" : entity_id,1893            "upload_link_info": upload_info['upload_info']1894        }1895        params.update(self._auth_params())1896        if is_thumbnail:1897            if field_name == "filmstrip_thumb_image" or field_name == "filmstrip_image":1898                params["filmstrip"] = True1899        else:1900            if display_name is None:1901                display_name = filename1902            # we allow linking to nothing for generic reference use cases1903            if field_name is not None:1904                params["field_name"] = field_name1905            params["display_name"] = display_name1906            # None gets converted to a string and added as a tag...1907            if tag_list:1908                params["tag_list"] = tag_list1909        result = self._send_form(url, params)1910        if not str(result).startswith("1"):1911            raise ShotgunError("Could not upload file successfully, but " \1912                               "not sure why.\nPath: %s\nUrl: %s\nError: %s" % (1913                                   path, url, str(result)))1914        LOG.debug("Attachment linked to content on Cloud storage")1915        attachment_id = int(str(result).split(":")[1].split("\n")[0])1916        return attachment_id1917    def _upload_to_sg(self, entity_type, entity_id, path, field_name, display_name,1918                      tag_list, is_thumbnail):1919        """1920        Internal function to upload a file to Shotgun and link it to the specified entity.1921        :param str entity_type: Entity type to link the upload to.1922        :param int entity_id: Id of the entity to link the upload to.1923        :param str path: Full path to an existing non-empty file on disk to upload.1924        :param str field_name: The internal Shotgun field name on the entity to store the file in.1925            This field must be a File/Link field type.1926        :param str display_name: The display name to use for the file. Defaults to the file name.1927        :param str tag_list: comma-separated string of tags to assign to the file.1928        :param bool is_thumbnail: indicates if the attachment is a thumbnail.1929        :returns: Id of the Attachment entity that was created for the image.1930        :rtype: int1931        """1932        params = {1933            "entity_type" : entity_type,1934            "entity_id" : entity_id,1935        }1936        params.update(self._auth_params())1937        if is_thumbnail:1938            url = urlparse.urlunparse((self.config.scheme, self.config.server,1939                "/upload/publish_thumbnail", None, None, None))1940            params["thumb_image"] = open(path, "rb")1941            if field_name == "filmstrip_thumb_image" or field_name == "filmstrip_image":1942                params["filmstrip"] = True1943        else:1944            url = urlparse.urlunparse((self.config.scheme, self.config.server,1945                "/upload/upload_file", None, None, None))1946            if display_name is None:1947                display_name = os.path.basename(path)1948            # we allow linking to nothing for generic reference use cases1949            if field_name is not None:1950                params["field_name"] = field_name1951            params["display_name"] = display_name1952            # None gets converted to a string and added as a tag...1953            if tag_list:1954                params["tag_list"] = tag_list1955            params["file"] = open(path, "rb")1956        result = self._send_form(url, params)1957        if not str(result).startswith("1"):1958            raise ShotgunError("Could not upload file successfully, but "\1959                "not sure why.\nPath: %s\nUrl: %s\nError: %s" % (1960                path, url, str(result)))1961        attachment_id = int(str(result).split(":")[1].split("\n")[0])1962        return attachment_id1963    def _get_attachment_upload_info(self, is_thumbnail, filename, is_multipart_upload):1964        """1965        Internal function to get the information needed to upload a file to Cloud storage.1966        :param bool is_thumbnail: indicates if the attachment is a thumbnail.1967        :param str filename: name of the file that will be uploaded.1968        :param bool is_multipart_upload: Indicates if we want multi-part upload information back.1969        :returns: dictionary containing upload details from the server.1970            These details are used throughout the upload process.1971        :rtype: dict1972        """1973        if is_thumbnail:1974            upload_type = "Thumbnail"1975        else:1976            upload_type = "Attachment"1977        params = {1978            "upload_type" : upload_type,1979            "filename" : filename1980        }1981        params["multipart_upload"] = is_multipart_upload1982        upload_url = "/upload/api_get_upload_link_info"1983        url = urlparse.urlunparse((self.config.scheme, self.config.server,1984                                   upload_url, None, None, None))1985        upload_info = self._send_form(url, params)1986        if not str(upload_info).startswith("1"):1987            raise ShotgunError("Could not get upload_url but " \1988                               "not sure why.\nPath: %s\nUrl: %s\nError: %s" % (1989                                   filename, url, str(upload_info)))1990        LOG.debug("Completed rpc call to %s" % (upload_url))1991        upload_info_parts = str(upload_info).split("\n")1992        return {1993            "upload_url" : upload_info_parts[1],1994            "timestamp" : upload_info_parts[2],1995            "upload_type" : upload_info_parts[3],1996            "upload_id": upload_info_parts[4],1997            "upload_info" : upload_info1998        }1999    def download_attachment(self, attachment=False, file_path=None, attachment_id=None):2000        """2001        Download the file associated with a Shotgun Attachment.2002            >>> version = sg.find_one("Version", [["id", "is", 7115]], ["sg_uploaded_movie"])2003            >>> local_file_path = "/var/tmp/%s" % version["sg_uploaded_movie"]["name"]2004            >>> sg.download_attachment(version["sg_uploaded_movie"], file_path=local_file_path)2005            /var/tmp/100b_scene_output_v032.mov2006        .. warning::2007            On older (< v5.1.0) Shotgun versions, non-downloadable files2008            on Shotgun don't raise exceptions, they cause a server error which2009            returns a 200 with the page content.2010        :param dict attachment: Usually a dictionary representing an Attachment entity.2011            The dictionary should have a ``url`` key that specifies the download url.2012            Optionally, the dictionary can be a standard entity hash format with ``id`` and2013            ``type`` keys as long as ``"type"=="Attachment"``. This is only supported for2014            backwards compatibility (#22150).2015            If an int value is passed in, the Attachment entity with the matching id will2016            be downloaded from the Shotgun server.2017        :param str file_path: Optional file path to write the data directly to local disk. This2018            avoids loading all of the data in memory and saves the file locally at the given path.2019        :param id attachment_id: (deprecated) Optional ``id`` of the Attachment entity in Shotgun to2020            download.2021            .. note:2022                This parameter exists only for backwards compatibility for scripts specifying2023                the parameter with keywords.2024        :returns: If ``file_path`` is provided, returns the path to the file on disk.  If2025            ``file_path`` is ``None``, returns the actual data of the file as a string.2026        :rtype: str2027        """2028        # backwards compatibility when passed via keyword argument2029        if attachment is False:2030            if type(attachment_id) == int:2031                attachment = attachment_id2032            else:2033                raise TypeError("Missing parameter 'attachment'. Expected a "\2034                                "dict, int, NoneType value or"\2035                                "an int for parameter attachment_id")2036        # write to disk2037        if file_path:2038            try:2039                fp = open(file_path, 'wb')2040            except IOError, e:2041                raise IOError("Unable to write Attachment to disk using "\2042                              "file_path. %s" % e)2043        url = self.get_attachment_download_url(attachment)2044        if url is None:2045            return None2046        # We only need to set the auth cookie for downloads from Shotgun server2047        if self.config.server in url:2048            self.set_up_auth_cookie()2049        try:2050            request = urllib2.Request(url)2051            request.add_header('user-agent', "; ".join(self._user_agents))2052            req = urllib2.urlopen(request)2053            if file_path:2054                shutil.copyfileobj(req, fp)2055            else:2056                attachment = req.read()2057        # 400 [sg] Attachment id doesn't exist or is a local file2058        # 403 [s3] link is invalid2059        except urllib2.URLError, e:2060            if file_path:2061                fp.close()2062            err = "Failed to open %s\n%s" % (url, e)2063            if hasattr(e, 'code'):2064                if e.code == 400:2065                    err += "\nAttachment may not exist or is a local file?"2066                elif e.code == 403:2067                    # Only parse the body if it is an Amazon S3 url.2068                    if url.find('s3.amazonaws.com') != -1 \2069                        and e.headers['content-type'] == 'application/xml':2070                        body = e.readlines()2071                        if body:2072                            xml = ''.join(body)2073                            # Once python 2.4 support is not needed we can think about using2074                            # elementtree. The doc is pretty small so this shouldn't be an issue.2075                            match = re.search('<Message>(.*)</Message>', xml)2076                            if match:2077                                err += ' - %s' % (match.group(1))2078            raise ShotgunFileDownloadError(err)2079        else:2080            if file_path:2081                if not fp.closed:2082                    fp.close()2083                return file_path2084            else:2085                return attachment2086    def set_up_auth_cookie(self):2087        """2088        Set up urllib2 with a cookie for authentication on the Shotgun instance.2089        Looks up session token and sets that in a cookie in the :mod:`urllib2` handler. This is2090        used internally for downloading attachments from the Shotgun server.2091        """2092        sid = self.get_session_token()2093        cj = cookielib.LWPCookieJar()2094        c = cookielib.Cookie('0', '_session_id', sid, None, False,2095            self.config.server, False, False, "/", True, False, None, True,2096            None, None, {})2097        cj.set_cookie(c)2098        cookie_handler = urllib2.HTTPCookieProcessor(cj)2099        opener = self._build_opener(cookie_handler)2100        urllib2.install_opener(opener)2101    def get_attachment_download_url(self, attachment):2102        """2103        Return the URL for downloading provided Attachment.2104        :param mixed attachment: Usually a dict representing An Attachment entity in Shotgun to2105            return the download url for. If the ``url`` key is present, it will be used as-is for2106            the download url. If the ``url`` key is not present, a url will be constructed pointing2107            at the current Shotgun server for downloading the Attachment entity using the ``id``.2108            If ``None`` is passed in, it is silently ignored in order to avoid raising an error when2109            results from a :meth:`~shotgun_api3.Shotgun.find` are passed off to2110            :meth:`~shotgun_api3.Shotgun.download_attachment`2111        .. note::2112            Support for passing in an int representing the Attachment ``id`` is deprecated2113        .. todo::2114            Support for a standard entity hash should be removed: #221502115        :returns: the download URL for the Attachment or ``None`` if ``None`` was passed to2116            ``attachment`` parameter.2117        :rtype: str2118        """2119        attachment_id = None2120        if isinstance(attachment, int):2121            attachment_id = attachment2122        elif isinstance(attachment, dict):2123            try:2124                url = attachment['url']2125            except KeyError:2126                if ('id' in attachment and 'type' in attachment and2127                    attachment['type'] == 'Attachment'):2128                    attachment_id = attachment['id']2129                else:2130                    raise ValueError("Missing 'url' key in Attachment dict")2131        elif attachment is None:2132            url = None2133        else:2134            raise TypeError("Unable to determine download url. Expected "\2135                "dict, int, or NoneType. Instead got %s" % type(attachment))2136        if attachment_id:2137            url = urlparse.urlunparse((self.config.scheme, self.config.server,2138                "/file_serve/attachment/%s" % urllib.quote(str(attachment_id)),2139                None, None, None))2140        return url2141    def authenticate_human_user(self, user_login, user_password, auth_token=None):2142        """2143        Authenticate Shotgun HumanUser.2144        Authenticates a user given the login, password, and optionally, one-time auth token (when2145        two-factor authentication is required). The user must be a ``HumanUser`` entity and the2146        account must be active.2147        >>> sg.authenticate_human_user("rhendriks", "c0mPre$Hi0n", None)2148        {"type": "HumanUser", "id": 123, "name": "Richard Hendriks"}2149        :param str user_login: Login name of Shotgun HumanUser2150        :param str user_password: Password for Shotgun HumanUser2151        :param str auth_token: One-time token required to authenticate Shotgun HumanUser2152            when two-factor authentication is turned on.2153        :returns: Standard Shotgun dictionary representing the HumanUser if authentication2154            succeeded. ``None`` if authentication failed for any reason.2155        :rtype: dict2156        """2157        if not user_login:2158            raise ValueError('Please supply a username to authenticate.')2159        if not user_password:2160            raise ValueError('Please supply a password for the user.')2161        # Override permissions on Config obj2162        original_login = self.config.user_login2163        original_password = self.config.user_password2164        original_auth_token = self.config.auth_token2165        self.config.user_login = user_login2166        self.config.user_password = user_password2167        self.config.auth_token = auth_token2168        try:2169            data = self.find_one('HumanUser', [['sg_status_list', 'is', 'act'],2170                                               ['login', 'is', user_login]],2171                                 ['id', 'login'], '', 'all')2172            # Set back to default - There finally and except cannot be used together in python2.42173            self.config.user_login = original_login2174            self.config.user_password = original_password2175            self.config.auth_token = original_auth_token2176            return data2177        except Fault:2178            # Set back to default - There finally and except cannot be used together in python2.42179            self.config.user_login = original_login2180            self.config.user_password = original_password2181            self.config.auth_token = original_auth_token2182        except:2183            # Set back to default - There finally and except cannot be used together in python2.42184            self.config.user_login = original_login2185            self.config.user_password = original_password2186            self.config.auth_token = original_auth_token2187            raise2188    def update_project_last_accessed(self, project, user=None):2189        """2190        Update a Project's ``last_accessed_by_current_user`` field to the current timestamp.2191        This helps keep track of the recent Projects each user has worked on and enables scripts2192        and apps to use this information to display "Recent Projects" for users as a convenience.2193        .. versionadded::2194            Requires Shotgun v5.3.20+2195        >>> sg.update_project_last_accessed({"type": "Project", "id": 66},2196        ...                                 {"type": "HumanUser", "id": 43})2197        :param dict project: Standard Project entity dictionary2198        :param dict user: Standard user entity dictionary. This is optional if the current API2199            instance is using user-based authenitcation, or has specified ``sudo_as_login``. In2200            these cases, if ``user`` is not provided, the ``sudo_as_login`` value or ``login``2201            value from the current instance will be used instead.2202        """2203        if self.server_caps.version and self.server_caps.version < (5, 3, 20):2204                raise ShotgunError("update_project_last_accessed requires server version 5.3.20 or "2205                    "higher, server is %s" % (self.server_caps.version,))2206        if not user:2207            # Try to use sudo as user if present2208            if self.config.sudo_as_login:2209                user = self.find_one('HumanUser', [['login', 'is', self.config.sudo_as_login]])2210            # Try to use login if present2211            if self.config.user_login:2212                user = self.find_one('HumanUser', [['login', 'is', self.config.user_login]])2213        params = { "project_id": project['id'], }2214        if user:2215            params['user_id'] = user['id']2216        record = self._call_rpc("update_project_last_accessed_by_current_user", params)2217        self._parse_records(record)[0]2218    def note_thread_read(self, note_id, entity_fields=None):2219        """2220        Return the full conversation for a given note, including Replies and Attachments.2221        Returns a complex data structure on the following form::2222            [{'content': 'Please add more awesomeness to the color grading.',2223              'created_at': '2015-07-14 21:33:28 UTC',2224              'created_by': {'id': 38,2225                             'name': 'John Pink',2226                             'status': 'act',2227                             'type': 'HumanUser',2228                             'valid': 'valid'},2229              'id': 6013,2230              'type': 'Note'},2231             {'created_at': '2015-07-14 21:33:32 UTC',2232              'created_by': {'id': 38,2233                             'name': 'John Pink',2234                             'status': 'act',2235                             'type': 'HumanUser',2236                             'valid': 'valid'},2237              'id': 159,2238              'type': 'Attachment'},2239             {'content': 'More awesomeness added',2240              'created_at': '2015-07-14 21:54:51 UTC',2241              'id': 5,2242              'type': 'Reply',2243              'user': {'id': 38,2244                       'name': 'David Blue',2245                       'status': 'act',2246                       'type': 'HumanUser',2247                       'valid': 'valid'}}]2248        The list is returned in descending chronological order.2249        If you wish to include additional fields beyond the ones that are2250        returned by default, you can specify these in an entity_fields2251        dictionary. This dictionary should be keyed by entity type and each2252        key should contain a list of fields to retrieve, for example::2253            { "Note":       ["created_by.HumanUser.image",2254                             "addressings_to",2255                             "playlist",2256                             "user" ],2257              "Reply":      ["content"],2258              "Attachment": ["filmstrip_image",2259                            "local_storage",2260                            "this_file",2261                            "image"]2262            }2263        :param int note_id: The id for the note to be retrieved2264        :param dict entity_fields: Additional fields to retrieve as part of the request.2265            See above for details.2266        :returns: A list of dictionaries. See above for example.2267        :rtype: list2268        """2269        if self.server_caps.version and self.server_caps.version < (6, 2, 0):2270                raise ShotgunError("note_thread requires server version 6.2.0 or "\2271                    "higher, server is %s" % (self.server_caps.version,))2272        entity_fields = entity_fields or {}2273        if not isinstance(entity_fields, dict):2274            raise ValueError("entity_fields parameter must be a dictionary")2275        params = { "note_id": note_id, "entity_fields": entity_fields }2276        record = self._call_rpc("note_thread_contents", params)2277        result = self._parse_records(record)2278        return result2279    def text_search(self, text, entity_types, project_ids=None, limit=None):2280        """2281        Search across the specified entity types for the given text.2282        This method can be used to implement auto completion or a Shotgun global search. The method2283        requires a text input phrase that is at least three characters long, or an exception will2284        be raised.2285        Several ways to limit the results of the query are available:2286        - Using the ``project_ids`` parameter, you can provide a list of Project ids to search2287          across. Leaving this at its default value of ``None`` will search across all Shotgun data.2288        - You need to define which subset of entity types to search using the ``entity_types``2289          parameter. Each of these entity types can be associated with a filter query to further2290          reduce the list of matches. The filter list is using the standard filter syntax used by2291          for example the :meth:`~shotgun_api3.Shotgun.find` method.2292        **Example: Constrain the search to all Tasks but Character Assets only**2293        >>> entity_types = {2294        ...     "Asset": [["sg_asset_type", "is", "Character"]],2295        ...     "Task": []2296        ... }2297        >>> sg.text_search("bunny", entity_types)2298        {'matches': [{'id': 734,2299                      'type': 'Asset',2300                      'name': 'Bunny',2301                      'project_id': 65,2302                      'image': 'https://...',2303                      'links': ['', ''],2304                      'status': 'fin'},2305                      ...2306                      {'id': 558,2307                       'type': 'Task'2308                       'name': 'FX',2309                       'project_id': 65,2310                       'image': 'https://...',2311                       'links': ['Shot', 'bunny_010_0010'],2312                       'status': 'fin'}],2313            'terms': ['bunny']}2314        The links field will contain information about any linked entity. This is useful when, for2315        example, presenting Tasks and you want to display what Shot or Asset the Task is associated2316        with.2317        :param str text: Text to search for. This must be at least three characters long, or an2318            exception will be raised.2319        :param dict entity_types: Dictionary to specify which entity types to search across. See2320            above for usage examples.2321        :param list project_ids: List of Projects to search. By default, all projects will be2322            searched.2323        :param int limit: Specify the maximum number of matches to return.2324        :returns: A complex dictionary structure, see above for example.2325        :rtype: dict2326        """2327        if self.server_caps.version and self.server_caps.version < (6, 2, 0):2328                raise ShotgunError("auto_complete requires server version 6.2.0 or "\2329                    "higher, server is %s" % (self.server_caps.version,))2330        # convert entity_types structure into the form2331        # that the API endpoint expects2332        if not isinstance(entity_types, dict):2333            raise ValueError("entity_types parameter must be a dictionary")2334        api_entity_types = {}2335        for (entity_type, filter_list) in entity_types.iteritems():2336            if isinstance(filter_list, (list, tuple)):2337                resolved_filters = _translate_filters(filter_list, filter_operator=None)2338                api_entity_types[entity_type] = resolved_filters2339            else:2340                raise ValueError("value of entity_types['%s'] must "2341                                 "be a list or tuple." % entity_type)2342        project_ids = project_ids or []2343        params = { "text": text,2344                   "entity_types": api_entity_types,2345                   "project_ids": project_ids,2346                   "max_results": limit }2347        record = self._call_rpc("query_display_name_cache", params)2348        result = self._parse_records(record)[0]2349        return result2350    def activity_stream_read(self, entity_type, entity_id, entity_fields=None, min_id=None,2351                             max_id=None, limit=None):2352        """2353        Retrieve activity stream data from Shotgun.2354        This data corresponds to the data that is displayed in the2355        Activity tab for an entity in the Shotgun Web UI.2356        A complex data structure on the following form will be2357        returned from Shotgun::2358            {'earliest_update_id': 50,2359             'entity_id': 65,2360             'entity_type': 'Project',2361             'latest_update_id': 79,2362             'updates': [{'created_at': '2015-07-15 11:06:55 UTC',2363                          'created_by': {'id': 38,2364                                         'image': '6641',2365                                         'name': 'John Smith',2366                                         'status': 'act',2367                                         'type': 'HumanUser'},2368                          'id': 79,2369                          'meta': {'entity_id': 6004,2370                                   'entity_type': 'Version',2371                                   'type': 'new_entity'},2372                          'primary_entity': {'id': 6004,2373                                             'name': 'Review_turntable_v2',2374                                             'status': 'rev',2375                                             'type': 'Version'},2376                          'read': False,2377                          'update_type': 'create'},2378                         {...},2379                        ]2380            }2381        The main payload of the return data can be found inside the 'updates'2382        key, containing a list of dictionaries. This list is always returned2383        in descending date order. Each item may contain different fields2384        depending on their update type. The primary_entity key represents the2385        main Shotgun entity that is associated with the update. By default,2386        this entity is returned with a set of standard fields. By using the2387        entity_fields parameter, you can extend the returned data to include2388        additional fields. If for example you wanted to return the asset type2389        for all assets and the linked sequence for all Shots, pass the2390        following entity_fields::2391            {"Shot": ["sg_sequence"], "Asset": ["sg_asset_type"]}2392        Deep queries can be used in this syntax if you want to2393        traverse into connected data.2394        :param str entity_type: Entity type to retrieve activity stream for2395        :param int entity_id: Entity id to retrieve activity stream for2396        :param list entity_fields: List of additional fields to include.2397                              See above for details2398        :param int max_id: Do not retrieve ids greater than this id.2399                       This is useful when implementing paging.2400        :param int min_id: Do not retrieve ids lesser than this id.2401                       This is useful when implementing caching of2402                       the event stream data and you want to2403                       "top up" an existing cache.2404        :param int limit: Limit the number of returned records. If not specified,2405                      the system default will be used.2406        :returns: A complex activity stream data structure. See above for details.2407        :rtype: dict2408        """2409        if self.server_caps.version and self.server_caps.version < (6, 2, 0):2410                raise ShotgunError("activity_stream requires server version 6.2.0 or "\2411                    "higher, server is %s" % (self.server_caps.version,))2412        # set up parameters to send to server.2413        entity_fields = entity_fields or {}2414        if not isinstance(entity_fields, dict):2415            raise ValueError("entity_fields parameter must be a dictionary")2416        params = { "type": entity_type,2417                   "id": entity_id,2418                   "max_id": max_id,2419                   "min_id": min_id,2420                   "limit": limit,2421                   "entity_fields": entity_fields }2422        record = self._call_rpc("activity_stream", params)2423        result = self._parse_records(record)[0]2424        return result2425    def nav_expand(self, path, seed_entity_field=None, entity_fields=None):2426        """2427        Expand the navigation hierarchy for the supplied path.2428        .. warning::2429            This is an experimental method that is not officially part of the2430            python-api. Usage of this method is discouraged. This method's name,2431            arguments, and argument types may change at any point.2432        """2433        return self._call_rpc(2434            "nav_expand",2435            {2436                "path":path,2437                "seed_entity_field": seed_entity_field,2438                "entity_fields": entity_fields2439            }2440        )2441    def nav_search_string(self, root_path, search_string, seed_entity_field=None):2442        """2443        Search function adapted to work with the navigation hierarchy.2444        .. warning::2445            This is an experimental method that is not officially part of the2446            python-api. Usage of this method is discouraged. This method's name,2447            arguments, and argument types may change at any point.2448        """2449        return self._call_rpc(2450                "nav_search",2451                {2452                    "root_path":root_path,2453                    "seed_entity_field": seed_entity_field,2454                    "search_criteria": { "search_string": search_string }2455                }2456        )2457    def nav_search_entity(self, root_path, entity, seed_entity_field=None):2458        """2459        Search function adapted to work with the navigation hierarchy.2460        .. warning::2461            This is an experimental method that is not officially part of the2462            python-api. Usage of this method is discouraged. This method's name,2463            arguments, and argument types may change at any point.2464        """2465        return self._call_rpc(2466                "nav_search",2467                {2468                    "root_path": root_path,2469                    "seed_entity_field": seed_entity_field,2470                    "search_criteria": {"entity": entity }2471                }2472        )2473    def get_session_token(self):2474        """2475        Get the session token associated with the current session.2476        If a session token has already been established, this is returned, otherwise a new one is2477        generated on the server and returned.2478        >>> sg.get_session_token()2479        dd638be7d07c39fa73d935a775558a502480        :returns: String containing a session token.2481        :rtype: str2482        """2483        if self.config.session_token:2484            return self.config.session_token2485        rv = self._call_rpc("get_session_token", None)2486        session_token = (rv or {}).get("session_id")2487        if not session_token:2488            raise RuntimeError("Could not extract session_id from %s", rv)2489        self.config.session_token = session_token2490        return session_token2491    def _build_opener(self, handler):2492        """2493        Build urllib2 opener with appropriate proxy handler.2494        """2495        if self.config.proxy_handler:2496            opener = urllib2.build_opener(self.config.proxy_handler, handler)2497        else:2498            opener = urllib2.build_opener(handler)2499        return opener2500    def _turn_off_ssl_validation(self):2501        """2502        Turn off SSL certificate validation.2503        """2504        global NO_SSL_VALIDATION2505        self.config.no_ssl_validation = True2506        NO_SSL_VALIDATION = True2507        # reset ssl-validation in user-agents2508        self._user_agents = ["ssl %s (no-validate)" % self.client_caps.ssl_version2509                             if ua.startswith("ssl ") else ua2510                             for ua in self._user_agents]2511    # Deprecated methods from old wrapper2512    def schema(self, entity_type):2513        """2514        .. deprecated:: 3.0.02515           Use :meth:`~shotgun_api3.Shotgun.schema_field_read` instead.2516        """2517        raise ShotgunError("Deprecated: use schema_field_read('type':'%s') "2518            "instead" % entity_type)2519    def entity_types(self):2520        """2521        .. deprecated:: 3.0.02522           Use :meth:`~shotgun_api3.Shotgun.schema_entity_read` instead.2523        """2524        raise ShotgunError("Deprecated: use schema_entity_read() instead")2525    # ========================================================================2526    # RPC Functions2527    def _call_rpc(self, method, params, include_auth_params=True, first=False):2528        """2529        Call the specified method on the Shotgun Server sending the supplied payload.2530        """2531        LOG.debug("Starting rpc call to %s with params %s" % (2532            method, params))2533        params = self._transform_outbound(params)2534        payload = self._build_payload(method, params,2535            include_auth_params=include_auth_params)2536        encoded_payload = self._encode_payload(payload)2537        req_headers = {2538            "content-type" : "application/json; charset=utf-8",2539            "connection" : "keep-alive"2540        }2541        http_status, resp_headers, body = self._make_call("POST",2542            self.config.api_path, encoded_payload, req_headers)2543        LOG.debug("Completed rpc call to %s" % (method))2544        try:2545            self._parse_http_status(http_status)2546        except ProtocolError, e:2547            e.headers = resp_headers2548            # 403 is returned with custom error page when api access is blocked2549            if e.errcode == 403:2550                e.errmsg += ": %s" % body2551            raise2552        response = self._decode_response(resp_headers, body)2553        self._response_errors(response)2554        response = self._transform_inbound(response)2555        if not isinstance(response, dict) or "results" not in response:2556            return response2557        results = response.get("results")2558        if first and isinstance(results, list):2559            return results[0]2560        return results2561    def _auth_params(self):2562        """2563        Return a dictionary of the authentication parameters being used.2564        """2565        # Used to authenticate HumanUser credentials2566        if self.config.user_login and self.config.user_password:2567            auth_params = {2568                "user_login" : str(self.config.user_login),2569                "user_password" : str(self.config.user_password),2570            }2571            if self.config.auth_token:2572                auth_params["auth_token"] = str(self.config.auth_token)2573        # Use script name instead2574        elif self.config.script_name and self.config.api_key:2575            auth_params = {2576                "script_name" : str(self.config.script_name),2577                "script_key" : str(self.config.api_key),2578            }2579        # Authenticate using session_id2580        elif self.config.session_token:2581            if self.server_caps.version and self.server_caps.version < (5, 3, 0):2582                raise ShotgunError("Session token based authentication requires server version "2583                    "5.3.0 or higher, server is %s" % (self.server_caps.version,))2584            auth_params = {"session_token" : str(self.config.session_token)}2585            # Request server side to raise exception for expired sessions.2586            # This was added in as part of Shotgun 5.4.42587            if self.server_caps.version and self.server_caps.version > (5, 4, 3):2588                auth_params["reject_if_expired"] = True2589        else:2590            raise ValueError("invalid auth params")2591        if self.config.session_uuid:2592            auth_params["session_uuid"] = self.config.session_uuid2593        # Make sure sudo_as_login is supported by server version2594        if self.config.sudo_as_login:2595            if self.server_caps.version and self.server_caps.version < (5, 3, 12):2596                raise ShotgunError("Option 'sudo_as_login' requires server version 5.3.12 or "\2597                    "higher, server is %s" % (self.server_caps.version,))2598            auth_params["sudo_as_login"] = self.config.sudo_as_login2599        if self.config.extra_auth_params:2600            auth_params.update(self.config.extra_auth_params)2601        return auth_params2602    def _sanitize_auth_params(self, params):2603        """2604        Given an authentication parameter dictionary, sanitize any sensitive2605        information and return the sanitized dict copy.2606        """2607        sanitized_params = copy.copy(params)2608        for k in ['user_password', 'script_key', 'session_token']:2609            if k in sanitized_params:2610                sanitized_params[k] = '********'2611        return sanitized_params2612    def _build_payload(self, method, params, include_auth_params=True):2613        """2614        Build the payload to be send to the rpc endpoint.2615        """2616        if not method:2617            raise ValueError("method is empty")2618        call_params = []2619        if include_auth_params:2620            auth_params = self._auth_params()2621            call_params.append(auth_params)2622        if params:2623            call_params.append(params)2624        return {2625            "method_name" : method,2626            "params" : call_params2627        }2628    def _encode_payload(self, payload):2629        """2630        Encode the payload to a string to be passed to the rpc endpoint.2631        The payload is json encoded as a unicode string if the content2632        requires it. The unicode string is then encoded as 'utf-8' as it must2633        be in a single byte encoding to go over the wire.2634        """2635        wire = json.dumps(payload, ensure_ascii=False)2636        if isinstance(wire, unicode):2637            return wire.encode("utf-8")2638        return wire2639    def _make_call(self, verb, path, body, headers):2640        """2641        Make an HTTP call to the server.2642        Handles retry and failure.2643        """2644        attempt = 02645        req_headers = {}2646        req_headers["user-agent"] = "; ".join(self._user_agents)2647        if self.config.authorization:2648            req_headers["Authorization"] = self.config.authorization2649        req_headers.update(headers or {})2650        body = body or None2651        max_rpc_attempts = self.config.max_rpc_attempts2652        while (attempt < max_rpc_attempts):2653            attempt += 12654            try:2655                return self._http_request(verb, path, body, req_headers)2656            except SSLHandshakeError, e:2657                # Test whether the exception is due to the fact that this is an older version of2658                # Python that cannot validate certificates encrypted with SHA-2. If it is, then2659                # fall back on disabling the certificate validation and try again - unless the2660                # SHOTGUN_FORCE_CERTIFICATE_VALIDATION environment variable has been set by the2661                # user. In that case we simply raise the exception. Any other exceptions simply2662                # get raised as well.2663                #2664                # For more info see:2665                # http://blog.shotgunsoftware.com/2016/01/important-ssl-certificate-renewal-and.html2666                #2667                # SHA-2 errors look like this:2668                #   [Errno 1] _ssl.c:480: error:0D0C50A1:asn1 encoding routines:ASN1_item_verify:2669                #   unknown message digest algorithm2670                #2671                # Any other exceptions simply get raised.2672                if not str(e).endswith("unknown message digest algorithm") or \2673                   "SHOTGUN_FORCE_CERTIFICATE_VALIDATION" in os.environ:2674                    raise2675                if self.config.no_ssl_validation is False:2676                    LOG.warning("SSLHandshakeError: this Python installation is incompatible with "2677                                "certificates signed with SHA-2. Disabling certificate validation. "2678                                "For more information, see http://blog.shotgunsoftware.com/2016/01/"2679                                "important-ssl-certificate-renewal-and.html")2680                    self._turn_off_ssl_validation()2681                    # reload user agent to reflect that we have turned off ssl validation2682                    req_headers["user-agent"] = "; ".join(self._user_agents)2683                self._close_connection()2684                if attempt == max_rpc_attempts:2685                    raise2686            except Exception:2687                #TODO: LOG ?2688                self._close_connection()2689                if attempt == max_rpc_attempts:2690                    raise2691    def _http_request(self, verb, path, body, headers):2692        """2693        Make the actual HTTP request.2694        """2695        url = urlparse.urlunparse((self.config.scheme, self.config.server,2696            path, None, None, None))2697        LOG.debug("Request is %s:%s" % (verb, url))2698        LOG.debug("Request headers are %s" % headers)2699        LOG.debug("Request body is %s" % body)2700        conn = self._get_connection()2701        resp, content = conn.request(url, method=verb, body=body,2702            headers=headers)2703        #http response code is handled else where2704        http_status = (resp.status, resp.reason)2705        resp_headers = dict(2706            (k.lower(), v)2707            for k, v in resp.iteritems()2708        )2709        resp_body = content2710        LOG.debug("Response status is %s %s" % http_status)2711        LOG.debug("Response headers are %s" % resp_headers)2712        LOG.debug("Response body is %s" % resp_body)2713        return (http_status, resp_headers, resp_body)2714    def _parse_http_status(self, status):2715        """2716        Parse the status returned from the http request.2717        :param tuple status: Tuple of (code, reason).2718        :raises: RuntimeError if the http status is non success.2719        """2720        error_code = status[0]2721        errmsg = status[1]2722        if status[0] >= 300:2723            headers = "HTTP error from server"2724            if status[0] == 503:2725                errmsg = "Shotgun is currently down for maintenance or too busy to reply. Please " \2726                         "try again later."2727            raise ProtocolError(self.config.server,2728                                error_code,2729                                errmsg,2730                                headers)2731        return2732    def _decode_response(self, headers, body):2733        """2734        Decode the response from the server from the wire format to2735        a python data structure.2736        :param dict headers: Headers from the server.2737        :param str body: Raw response body from the server.2738        :returns: If the content-type starts with application/json or2739            text/javascript the body is json decoded. Otherwise the raw body is2740            returned.2741        :rtype: str2742        """2743        if not body:2744            return body2745        ct = (headers.get("content-type") or "application/json").lower()2746        if ct.startswith("application/json") or ct.startswith("text/javascript"):2747            return self._json_loads(body)2748        return body2749    def _json_loads(self, body):2750        return json.loads(body)2751    def _json_loads_ascii(self, body):2752        """2753        See http://stackoverflow.com/questions/9568672754        """2755        def _decode_list(lst):2756            newlist = []2757            for i in lst:2758                if isinstance(i, unicode):2759                    i = i.encode('utf-8')2760                elif isinstance(i, list):2761                    i = _decode_list(i)2762                newlist.append(i)2763            return newlist2764        def _decode_dict(dct):2765            newdict = {}2766            for k, v in dct.iteritems():2767                if isinstance(k, unicode):2768                    k = k.encode('utf-8')2769                if isinstance(v, unicode):2770                    v = v.encode('utf-8')2771                elif isinstance(v, list):2772                    v = _decode_list(v)2773                newdict[k] = v2774            return newdict2775        return json.loads(body, object_hook=_decode_dict)2776    def _response_errors(self, sg_response):2777        """2778        Raise any API errors specified in the response.2779        :raises ShotgunError: If the server response contains an exception.2780        """2781        ERR_AUTH = 102 # error code for authentication related problems2782        ERR_2FA  = 106 # error code when 2FA authentication is required but no 2FA token provided.2783        ERR_SSO  = 108 # error code when SSO is activated on the site, preventing the use of username/password for authentication.2784        if isinstance(sg_response, dict) and sg_response.get("exception"):2785            if sg_response.get("error_code") == ERR_AUTH:2786                raise AuthenticationFault(sg_response.get("message", "Unknown Authentication Error"))2787            elif sg_response.get("error_code") == ERR_2FA:2788                raise MissingTwoFactorAuthenticationFault(sg_response.get("message", "Unknown 2FA Authentication Error"))2789            elif sg_response.get("error_code") == ERR_SSO:2790                raise UserCredentialsNotAllowedForSSOAuthenticationFault(2791                    sg_response.get("message", "Authentication using username/password is not allowed for an SSO-enabled Shotgun site")2792                )2793            else:2794                # raise general Fault2795                raise Fault(sg_response.get("message", "Unknown Error"))2796        return2797    def _visit_data(self, data, visitor):2798        """2799        Walk the data (simple python types) and call the visitor.2800        """2801        if not data:2802            return data2803        recursive = self._visit_data2804        if isinstance(data, list):2805            return [recursive(i, visitor) for i in data]2806        if isinstance(data, tuple):2807            return tuple(recursive(i, visitor) for i in data)2808        if isinstance(data, dict):2809            return dict(2810                (k, recursive(v, visitor))2811                for k, v in data.iteritems()2812            )2813        return visitor(data)2814    def _transform_outbound(self, data):2815        """2816        Transform data types or values before they are sent by the client.2817        - changes timezones2818        - converts dates and times to strings2819        """2820        if self.config.convert_datetimes_to_utc:2821            def _change_tz(value):2822                if value.tzinfo is None:2823                    value = value.replace(tzinfo=SG_TIMEZONE.local)2824                return value.astimezone(SG_TIMEZONE.utc)2825        else:2826            _change_tz = None2827        local_now = datetime.datetime.now()2828        def _outbound_visitor(value):2829            if isinstance(value, datetime.datetime):2830                if _change_tz:2831                    value = _change_tz(value)2832                return value.strftime("%Y-%m-%dT%H:%M:%SZ")2833            if isinstance(value, datetime.date):2834                #existing code did not tz transform dates.2835                return value.strftime("%Y-%m-%d")2836            if isinstance(value, datetime.time):2837                value = local_now.replace(hour=value.hour,2838                    minute=value.minute, second=value.second,2839                    microsecond=value.microsecond)2840                if _change_tz:2841                    value = _change_tz(value)2842                return value.strftime("%Y-%m-%dT%H:%M:%SZ")2843            if isinstance(value, str):2844                # Convert strings to unicode2845                return value.decode("utf-8")2846            return value2847        return self._visit_data(data, _outbound_visitor)2848    def _transform_inbound(self, data):2849        """2850        Transforms data types or values after they are received from the server.2851        """2852        # NOTE: The time zone is removed from the time after it is transformed2853        # to the local time, otherwise it will fail to compare to datetimes2854        # that do not have a time zone.2855        if self.config.convert_datetimes_to_utc:2856            _change_tz = lambda x: x.replace(tzinfo=SG_TIMEZONE.utc)\2857                .astimezone(SG_TIMEZONE.local)2858        else:2859            _change_tz = None2860        def _inbound_visitor(value):2861            if isinstance(value, basestring):2862                if len(value) == 20 and self._DATE_TIME_PATTERN.match(value):2863                    try:2864                        # strptime was not on datetime in python2.42865                        value = datetime.datetime(2866                            *time.strptime(value, "%Y-%m-%dT%H:%M:%SZ")[:6])2867                    except ValueError:2868                        return value2869                    if _change_tz:2870                        return _change_tz(value)2871                    return value2872            return value2873        return self._visit_data(data, _inbound_visitor)2874    # ========================================================================2875    # Connection Functions2876    def _get_connection(self):2877        """2878        Return the current connection or creates a new connection to the current server.2879        """2880        if self._connection is not None:2881            return self._connection2882        if self.config.proxy_server:2883            pi = ProxyInfo(socks.PROXY_TYPE_HTTP, self.config.proxy_server,2884                 self.config.proxy_port, proxy_user=self.config.proxy_user,2885                 proxy_pass=self.config.proxy_pass)2886            self._connection = Http(timeout=self.config.timeout_secs, ca_certs=self.__ca_certs,2887                proxy_info=pi, disable_ssl_certificate_validation=self.config.no_ssl_validation)2888        else:2889            self._connection = Http(timeout=self.config.timeout_secs, ca_certs=self.__ca_certs,2890                proxy_info=None, disable_ssl_certificate_validation=self.config.no_ssl_validation)2891        return self._connection2892    def _close_connection(self):2893        """2894        Close the current connection.2895        """2896        if self._connection is None:2897            return2898        for conn in self._connection.connections.values():2899            try:2900                conn.close()2901            except Exception:2902                pass2903        self._connection.connections.clear()2904        self._connection = None2905        return2906    # ========================================================================2907    # Utility2908    def _parse_records(self, records):2909        """2910        Parse 'records' returned from the api to do local modifications:2911        - Insert thumbnail urls2912        - Insert local file paths.2913        - Revert &lt; html entities that may be the result of input sanitization2914          mechanisms back to a litteral < character.2915        :param records: List of records (dicts) to process or a single record.2916        :returns: A list of the records processed.2917        """2918        if not records:2919            return []2920        if not isinstance(records, (list, tuple)):2921            records = [records, ]2922        for rec in records:2923            # skip results that aren't entity dictionaries2924            if not isinstance(rec, dict):2925                continue2926            # iterate over each item and check each field for possible injection2927            for k, v in rec.iteritems():2928                if not v:2929                    continue2930                # Check for html entities in strings2931                if isinstance(v, types.StringTypes):2932                    rec[k] = rec[k].replace('&lt;', '<')2933                # check for thumbnail for older version (<3.3.0) of shotgun2934                if k == 'image' and \2935                   self.server_caps.version and \2936                   self.server_caps.version < (3, 3, 0):2937                    rec['image'] = self._build_thumb_url(rec['type'],2938                        rec['id'])2939                    continue2940                if isinstance(v, dict) and v.get('link_type') == 'local' \2941                    and self.client_caps.local_path_field in v:2942                    local_path = v[self.client_caps.local_path_field]2943                    v['local_path'] = local_path2944                    v['url'] = "file://%s" % (local_path or "",)2945        return records2946    def _build_thumb_url(self, entity_type, entity_id):2947        """2948        Return the URL for the thumbnail of an entity given the entity type and the entity id.2949        Note: This makes a call to the server for every thumbnail.2950        :param entity_type: Entity type the id is for.2951        :param entity_id: id of the entity to get the thumbnail for.2952        :returns: Fully qualified url to the thumbnail.2953        """2954        # Example response from the end point2955        # curl "https://foo.com/upload/get_thumbnail_url?entity_type=Version&entity_id=1"2956        # 12957        # /files/0000/0000/0012/232/shot_thumb.jpg.jpg2958        entity_info = {'e_type':urllib.quote(entity_type),2959                       'e_id':urllib.quote(str(entity_id))}2960        url = ("/upload/get_thumbnail_url?" +2961                "entity_type=%(e_type)s&entity_id=%(e_id)s" % entity_info)2962        body = self._make_call("GET", url, None, None)[2]2963        code, thumb_url = body.splitlines()2964        code = int(code)2965        # code of 0 means error, second line is the error code2966        if code == 0:2967            raise ShotgunError(thumb_url)2968        if code == 1:2969            return urlparse.urlunparse((self.config.scheme,2970                self.config.server, thumb_url.strip(), None, None, None))2971        # Comments in prev version said we can get this sometimes.2972        raise RuntimeError("Unknown code %s %s" % (code, thumb_url))2973    def _dict_to_list(self, d, key_name="field_name", value_name="value", extra_data=None):2974        """2975        Utility function to convert a dict into a list dicts using the key_name and value_name keys.2976        e.g. d {'foo' : 'bar'} changed to [{'field_name':'foo', 'value':'bar'}]2977        Any dictionary passed in via extra_data will be merged into the resulting dictionary.2978        e.g. d as above and extra_data of {'foo': {'thing1': 'value1'}} changes into2979        [{'field_name': 'foo', 'value': 'bar', 'thing1': 'value1'}]2980        """2981        ret = []2982        for k, v in (d or {}).iteritems():2983            d = {key_name: k, value_name: v}2984            d.update((extra_data or {}).get(k, {}))2985            ret.append(d)2986        return ret2987    def _dict_to_extra_data(self, d, key_name="value"):2988        """2989        Utility function to convert a dict into a dict compatible with the extra_data arg2990        of _dict_to_list.2991        e.g. d {'foo' : 'bar'} changed to {'foo': {"value": 'bar'}]2992        """2993        return dict([(k, {key_name: v}) for (k,v) in (d or {}).iteritems()])2994    def _upload_file_to_storage(self, path, storage_url):2995        """2996        Internal function to upload an entire file to the Cloud storage.2997        :param str path: Full path to an existing non-empty file on disk to upload.2998        :param str storage_url: Target URL for the uploaded file.2999        """3000        filename = os.path.basename(path)3001        fd = open(path, "rb")3002        try:3003            content_type = mimetypes.guess_type(filename)[0]3004            content_type = content_type or "application/octet-stream"3005            file_size = os.fstat(fd.fileno())[stat.ST_SIZE]3006            self._upload_data_to_storage(fd, content_type, file_size, storage_url )3007        finally:3008            fd.close()3009        LOG.debug("File uploaded to Cloud storage: %s", filename)3010    def _multipart_upload_file_to_storage(self, path, upload_info):3011        """3012        Internal function to upload a file to the Cloud storage in multiple parts.3013        :param str path: Full path to an existing non-empty file on disk to upload.3014        :param dict upload_info: Contains details received from the server, about the upload.3015        """3016        fd = open(path, "rb")3017        try:3018            content_type = mimetypes.guess_type(path)[0]3019            content_type = content_type or "application/octet-stream"3020            file_size = os.fstat(fd.fileno())[stat.ST_SIZE]3021            filename = os.path.basename(path)3022            etags = []3023            part_number = 13024            bytes_read = 03025            chunk_size = self._MULTIPART_UPLOAD_CHUNK_SIZE3026            while bytes_read < file_size:3027                data = fd.read(chunk_size)3028                bytes_read += len(data)3029                part_url = self._get_upload_part_link(upload_info, filename, part_number)3030                etags.append(self._upload_data_to_storage(data, content_type, len(data), part_url ))3031                part_number += 13032            self._complete_multipart_upload(upload_info, filename, etags)3033        finally:3034            fd.close()3035        LOG.debug("File uploaded in multiple parts to Cloud storage: %s", path)3036    def _get_upload_part_link(self, upload_info, filename, part_number):3037        """3038        Internal function to get the url to upload the next part of a file to the3039        Cloud storage, in a multi-part upload process.3040        :param dict upload_info: Contains details received from the server, about the upload.3041        :param str filename: Name of the file for which we want the link.3042        :param int part_number: Part number for the link.3043        :returns: upload url.3044        :rtype: str3045        """3046        params = {3047            "upload_type": upload_info["upload_type"],3048            "filename": filename,3049            "timestamp": upload_info["timestamp"],3050            "upload_id": upload_info["upload_id"],3051            "part_number": part_number3052        }3053        url = urlparse.urlunparse((self.config.scheme, self.config.server,3054                                   "/upload/api_get_upload_link_for_part", None, None, None))3055        result = self._send_form(url, params)3056        # Response is of the form: 1\n<url> (for success) or 0\n (for failure).3057        # In case of success, we know we the second line of the response contains the3058        # requested URL.3059        if not str(result).startswith("1"):3060            raise ShotgunError("Unable get upload part link: %s" % result)3061        LOG.debug("Got next upload link from server for multipart upload.")3062        return str(result).split("\n")[1]3063    def _upload_data_to_storage(self, data, content_type, size, storage_url):3064        """3065        Internal function to upload data to Cloud storage.3066        :param stream data: Contains details received from the server, about the upload.3067        :param str content_type: Content type of the data stream.3068        :param int size: Number of bytes in the data stream.3069        :param str storage_url: Target URL for the uploaded file.3070        :returns: upload url.3071        :rtype: str3072        """3073        try:3074            opener = urllib2.build_opener(urllib2.HTTPHandler)3075            request = urllib2.Request(storage_url, data=data)3076            request.add_header("Content-Type", content_type)3077            request.add_header("Content-Length", size)3078            request.get_method = lambda: "PUT"3079            result = opener.open(request)3080            etag = result.info().getheader("ETag")3081        except urllib2.HTTPError, e:3082            if e.code == 500:3083                raise ShotgunError("Server encountered an internal error.\n%s\n%s\n\n" % (storage_url, e))3084            else:3085                raise ShotgunError("Unanticipated error occurred uploading to %s: %s" % (storage_url, e))3086        LOG.debug("Part upload completed successfully.")3087        return etag3088    def _complete_multipart_upload(self, upload_info, filename, etags):3089        """3090        Internal function to complete a multi-part upload to the Cloud storage.3091        :param dict upload_info: Contains details received from the server, about the upload.3092        :param str filename: Name of the file for which we want to complete the upload.3093        :param tupple etags: Contains the etag of each uploaded file part.3094        """3095        params = {3096            "upload_type": upload_info["upload_type"],3097            "filename": filename,3098            "timestamp": upload_info["timestamp"],3099            "upload_id": upload_info["upload_id"],3100            "etags": ",".join(etags)3101        }3102        url = urlparse.urlunparse((self.config.scheme, self.config.server,3103                                   "/upload/api_complete_multipart_upload", None, None, None))3104        result = self._send_form(url, params)3105        # Response is of the form: 1\n or 0\n to indicate success or failure of the call.3106        if not str(result).startswith("1"):3107            raise ShotgunError("Unable get upload part link: %s" % result)3108    def _send_form(self, url, params):3109        """3110        Utility function to send a Form to Shotgun and process any HTTP errors that3111        could occur.3112        :param url: endpoint where the form is sent.3113        :param params: form data3114        :returns: result from the server.3115        """3116        params.update(self._auth_params())3117        opener = self._build_opener(FormPostHandler)3118        # Perform the request3119        try:3120            resp = opener.open(url, params)3121            result = resp.read()3122            # response headers are in str(resp.info()).splitlines()3123        except urllib2.HTTPError, e:3124            if e.code == 500:3125                raise ShotgunError("Server encountered an internal error. "3126                                   "\n%s\n(%s)\n%s\n\n" % (url, self._sanitize_auth_params(params), e))3127            else:3128                raise ShotgunError("Unanticipated error occurred %s" % (e))3129        return result3130# Helpers from the previous API, left as is.3131# Based on http://code.activestate.com/recipes/146306/3132class FormPostHandler(urllib2.BaseHandler):3133    """3134    Handler for multipart form data3135    """3136    handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first3137    def http_request(self, request):3138        data = request.get_data()3139        if data is not None and not isinstance(data, basestring):3140            files = []3141            params = []3142            for key, value in data.items():3143                if isinstance(value, file):3144                    files.append((key, value))3145                else:3146                    params.append((key, value))3147            if not files:3148                data = urllib.urlencode(params, True) # sequencing on3149            else:3150                boundary, data = self.encode(params, files)3151                content_type = 'multipart/form-data; boundary=%s' % boundary3152                request.add_unredirected_header('Content-Type', content_type)3153            request.add_data(data)3154        return request3155    def encode(self, params, files, boundary=None, buffer=None):3156        if boundary is None:3157            boundary = mimetools.choose_boundary()3158        if buffer is None:3159            buffer = cStringIO.StringIO()3160        for (key, value) in params:3161            buffer.write('--%s\r\n' % boundary)3162            buffer.write('Content-Disposition: form-data; name="%s"' % key)3163            buffer.write('\r\n\r\n%s\r\n' % value)3164        for (key, fd) in files:3165            filename = fd.name.split('/')[-1]3166            content_type = mimetypes.guess_type(filename)[0]3167            content_type = content_type or 'application/octet-stream'3168            file_size = os.fstat(fd.fileno())[stat.ST_SIZE]3169            buffer.write('--%s\r\n' % boundary)3170            c_dis = 'Content-Disposition: form-data; name="%s"; filename="%s"%s'3171            content_disposition = c_dis % (key, filename, '\r\n')3172            buffer.write(content_disposition)3173            buffer.write('Content-Type: %s\r\n' % content_type)3174            buffer.write('Content-Length: %s\r\n' % file_size)3175            fd.seek(0)3176            buffer.write('\r\n%s\r\n' % fd.read())3177        buffer.write('--%s--\r\n\r\n' % boundary)3178        buffer = buffer.getvalue()3179        return boundary, buffer3180    def https_request(self, request):3181        return self.http_request(request)3182def _translate_filters(filters, filter_operator):3183    """3184    Translate filters params into data structure expected by rpc call.3185    """3186    wrapped_filters = {3187        "filter_operator": filter_operator or "all",3188        "filters": filters3189    }3190    return _translate_filters_dict(wrapped_filters)3191def _translate_filters_dict(sg_filter):3192    new_filters = {}3193    filter_operator = sg_filter.get("filter_operator")3194    if filter_operator == "all" or filter_operator == "and":3195        new_filters["logical_operator"] = "and"3196    elif filter_operator == "any" or filter_operator == "or":3197        new_filters["logical_operator"] = "or"3198    else:3199        raise ShotgunError("Invalid filter_operator %s" % filter_operator)3200    if not isinstance(sg_filter["filters"], (list,tuple)):3201        raise ShotgunError("Invalid filters, expected a list or a tuple, got %s"3202                           % sg_filter["filters"])3203    new_filters["conditions"] = _translate_filters_list(sg_filter["filters"])3204    return new_filters3205def _translate_filters_list(filters):3206    conditions = []3207    for sg_filter in filters:3208        if isinstance(sg_filter, (list,tuple)):3209            conditions.append(_translate_filters_simple(sg_filter))3210        elif isinstance(sg_filter, dict):3211            conditions.append(_translate_filters_dict(sg_filter))3212        else:3213            raise ShotgunError("Invalid filters, expected a list, tuple or dict, got %s"3214                               % sg_filter)3215    return conditions3216def _translate_filters_simple(sg_filter):3217    condition = {3218        "path": sg_filter[0],3219        "relation": sg_filter[1]3220    }3221    values = sg_filter[2:]3222    if len(values) == 1 and isinstance(values[0], (list, tuple)):3223        values = values[0]3224    condition["values"] = values3225    return condition3226def _version_str(version):3227    """3228    Convert a tuple of int's to a '.' separated str.3229    """...

icap_encryption.py

Source:icap_encryption.py

1#!/bin/env python2# -*- coding: utf8 -*-3import datetime4import re5import threading6import os7import subprocess8import getpass9from hashlib import sha25610import hashlib11import base6412import xml.etree.ElementTree as et13try:14    import socketserver15except ImportError:16    import SocketServer17    socketserver = SocketServer18import sys19sys.path.append('.')20from .pyicap import *21from .object_crypto import AESCipher22from .aws_signature_icap import AWSSignature23from .auth_manager import AuthManager24BASE_DIR = os.path.dirname(os.path.dirname(__file__))25class ThreadingSimpleServer(socketserver.ThreadingMixIn, ICAPServer):26    pass27class ICAPHandler(BaseICAPRequestHandler):28    pending_urls = {}29    def request_OPTIONS(self):30        self.set_icap_response(200)31        self.set_icap_header(b'Methods', b'REQMOD')32        self.set_icap_header(b'Service', b'PyICAP Server 1.0')33        self.send_headers(False)34    def request_REQMOD(self):35        """36        fname = re.sub('\/', '_', self.enc_req[1].decode())37        f = open(fname, 'w')38        f.write('Request URL:\n')39        f.write(self.enc_req[1].decode())40        f.write('\n------------\n')41        f.write('Headers:\n')42        for i in self.enc_req_headers:43            f.write(i.decode() + " : ")44            for v in self.enc_req_headers[i]:45                f.write(v.decode() + ',')46            f.write('\n')47        f.close()48        """49        if not self.server.auth_manager.is_setup(): # Authentication manager is not set up50            self.send_error(403, message='error')51            return52        awss = AWSSignature()53        self.set_icap_response(200)54        self.set_enc_request(b' '.join(self.enc_req))55        for h in self.enc_req_headers:56            for v in self.enc_req_headers[h]:57                self.set_enc_header(h, v)58        content = b''59        if not self.has_body:60            if self.enc_req[0] == b'DELETE': # DeleteObject operation61                self.pending_urls[('DELETE', self.enc_req[1].decode())] = ''62            if self.enc_req[0] == b'PUT' and b'x-amz-copy-source' in self.enc_req_headers:63                # Object duplication, copy object key from existing record64                bucket_name = re.search("(?<=^\/)[^\/]+", self.enc_req_headers[b'x-amz-copy-source'][0].decode()).group()65                key = re.search("^.*\.s3.*\.amazonaws.com", self.enc_req[1].decode()).group() + \66                    re.search("(?<=^\/{})\/.*".format(bucket_name), self.enc_req_headers[b'x-amz-copy-source'][0].decode()).group()67                key = re.sub("(?<=^https:\/\/)[a-z0-9]+(?=.s3)", bucket_name, key)68                pending_entry = {69                    'source': key,70                    'new_path': self.enc_req[1].decode()71                }72                self.pending_urls[('PUT', self.enc_req[1].decode())] = pending_entry73            if self.enc_req[0] == b'POST' and b'?uploads' in self.enc_req[1]:74                # New multipart upload (CreateMultipartUpload)75                object_key = re.search("^.*(?=\?)", self.enc_req[1].decode()).group()76                self.pending_urls[('POST', self.enc_req[1].decode())] = {'multipart': 1,77                                                                'object_key': object_key}78            self.send_headers(False)79            return80        if self.preview:81            prevbuf = b''82            while True:83                chunk = self.read_chunk()84                if chunk == b'':85                    break86                prevbuf += chunk87            if self.ieof:88                self.send_headers(True)89                if len(prevbuf) > 0:90                    self.write_chunk(prevbuf)91                self.write_chunk(b'')92                return93            self.cont()94            self.send_headers(True)95            if len(prevbuf) > 0:96                self.write_chunk(prevbuf)97            while True:98                chunk = self.read_chunk()99                self.write_chunk(chunk)100                if chunk == b'':101                    break102        else:103            while True:  # Read request body104                chunk = self.read_chunk()105                content += chunk106                if chunk == b'':107                    break108            not_object_upload = re.search("\.s3\.(.+)\.amazonaws\.com/\?", self.enc_req[1].decode())109            is_multipart_upload = re.search("^.*\.s3\..*\.amazonaws.com\/.*\?partNumber=", self.enc_req[1].decode()) is not None110            complete_multipart_request = re.search("s3\..*\.amazonaws\.com\/.*\?uploadId=(?!.*partNumber=)",111                                                   self.enc_req[1].decode()) is not None112            if self.enc_req[0] == b'PUT' and not_object_upload is None and not is_multipart_upload:113                # Single object upload114                cp = AESCipher()115                enc_tup = cp.encrypt(content)116                encrypted_content_length = str(len(enc_tup[1])).encode()117                # Update these headers if they exist118                if b'content-md5' in self.enc_headers:119                    self.enc_headers.pop(b'content-md5')120                    encrypted_content_md5 = hashlib.md5(enc_tup[1]).hexdigest()121                    self.set_enc_header(b'content-md5', encrypted_content_md5.encode())122                elif b'x-amz-content-sha256' in self.enc_headers:123                    self.enc_headers.pop(b'x-amz-content-sha256')124                    encrypted_content_sha256 = sha256(enc_tup[1]).hexdigest()125                    self.set_enc_header(b'x-amz-content-sha256', encrypted_content_sha256.encode())126                self.enc_headers.pop(b'content-length')127                self.set_enc_header(b'content-length', encrypted_content_length)128                if b'content-type' in self.enc_headers:  # Binary files have no content-type header129                    self.enc_headers.pop(b'content-type')130                    self.set_enc_header(b'content-type', b'')131                content = enc_tup[1]132                key = enc_tup[0]133                #payload_hash = sha256(enc_tup[1]).hexdigest()134                sig = awss.gen_signature(request=self.enc_req, headers=self.enc_headers)  # Generate AWS signature135                self.set_enc_request(b'PUT ' + sig['url'].encode() + b' HTTP/1.1')  # Update URL of request136                if b'authorization' in self.enc_headers:137                    # Header should always be present if authenticating with header option138                    self.enc_headers.pop(b'authorization')139                    self.set_enc_header(b'authorization', sig['authorization-header'].encode())140                self.pending_urls[('PUT', sig['url'])] = {'key': key['key'], 'nonce': key['nonce'], 'tag': key['tag']}141            elif self.enc_req[0] == b'PUT' and is_multipart_upload:142                # Multipart upload143                cp = AESCipher()144                object_key = re.search("^.*\.s3\..*\.amazonaws.com\/.*(?=\?)", self.enc_req[1].decode()).group()145                part_num = re.search("(?<=partNumber=)[0-9]+", self.enc_req[1].decode()).group()146                upload_id = re.search("(?<=uploadId=)[^&]+", self.enc_req[1].decode()).group()147                file_params = self.server.auth_manager.get_object(object_key)  # Get key from existing record148                params_key = base64.b64decode(file_params.key)149                params_nonce = base64.b64decode(file_params.nonce)150                enc_tup = cp.encrypt(content, key=params_key, nonce=params_nonce)151                content = enc_tup[1]152                part_length = len(enc_tup[1])153                if b'content-md5' in self.enc_headers:154                    self.enc_headers.pop(b'content-md5')155                    encrypted_content_md5 = base64.b64encode(hashlib.md5(enc_tup[1]).digest())156                    self.set_enc_header(b'content-md5', encrypted_content_md5)157                if b'x-amz-content-sha256' in self.enc_headers:158                    self.enc_headers.pop(b'x-amz-content-sha256')159                    encrypted_content_sha256 = sha256(enc_tup[1]).hexdigest()160                    self.set_enc_header(b'x-amz-content-sha256', encrypted_content_sha256.encode())161                self.enc_headers.pop(b'content-length')162                self.set_enc_header(b'content-length', str(part_length).encode())163                if b'content-type' in self.enc_headers:  # binary files have no content-type header164                    self.enc_headers.pop(b'content-type')165                    self.set_enc_header(b'content-type', b'')166                sig = awss.gen_signature(request=self.enc_req, headers=self.enc_headers)167                self.set_enc_request(b'PUT ' + sig['url'].encode() + b' HTTP/1.1')168                if b'authorization' in self.enc_headers:169                    self.enc_headers.pop(b'authorization')170                    self.set_enc_header(b'authorization', sig['authorization-header'].encode())171                self.pending_urls[('PUT', sig['url'])] = { 'part_num': part_num,172                                                           'part_length': part_length,173                                                           'part_tag': enc_tup[0]['tag'],174                                                           'upload_id': upload_id175                                                           }176            elif self.enc_req[0] == b'POST' and complete_multipart_request is True:177                # Complete multipart upload request178                xmldata = et.fromstring(content.decode())179                valid_parts = xmldata.findall("./{*}Part/{*}PartNumber")180                valid_parts = [x.text for x in valid_parts]181                upload_id = re.search("(?<=\?uploadId=)([^\&]*)", self.enc_req[1].decode()).group()182                all_parts = self.server.auth_manager.get_object_parts(upload_id=upload_id)183                for part in all_parts:184                    # Remove invalid parts not referenced by CompleteMultiPartUpload185                    if str(part['part_num']) not in valid_parts:186                        self.server.auth_manager.delete_part(upload_id=upload_id, part_num=part['part_num'])187            self.send_headers(True)188            self.write_chunk(content)189    def response_OPTIONS(self):190        self.set_icap_response(200)191        self.set_icap_header(b'Methods', b'RESPMOD')192        self.set_icap_header(b'Service', b'PyICAP Server 1.0')193        self.set_icap_header(b'Preview', b'0')194        self.set_icap_header(b'Transfer-Preview', b'*')195        self.set_icap_header(b'Transfer-Ignore', b'jpg,jpeg,gif,png,swf,flv')196        self.set_icap_header(b'Transfer-Complete', b'')197        self.set_icap_header(b'Max-Connections', b'100')198        self.set_icap_header(b'Options-TTL', b'3600')199        self.send_headers(False)200    def read_chunks(self):201        content = b''202        while True:203            chunk = self.read_chunk()204            content += chunk205            if chunk == b'':206                return content207    def response_RESPMOD(self):208        print("resp")209        obj = None210        self.set_icap_response(200)211        self.set_enc_status(b' '.join(self.enc_res_status))212        for h in self.enc_res_headers:213            for v in self.enc_res_headers[h]:214                self.set_enc_header(h, v)215        request_url = self.enc_req[1].decode()216        op = self.enc_req[0].decode()217        url = re.search("^.*\.(s3)\..*(\.amazonaws.com).*(?=\?)", self.enc_req[1].decode()) # Get object key218        if url is None:219            url = re.search("^.*\.(s3)\..*(\.amazonaws.com).*", self.enc_req[1].decode())220        print(self.enc_req)221        print("op: {} url: {}".format(op,url))222        if url is not None:223            url = url.group()224        else:225            url = request_url  # No object key226        if (op, request_url) in self.pending_urls.keys():227            obj = self.pending_urls.pop((op, request_url))228            if op == 'PUT' and 'part_num' not in obj: # Response for object upload (regular, not multipart)229                if re.search("\.s3\.(.+)\.amazonaws\.com/\?", request_url) is None:230                    if self.enc_res_status[1] == b'200':231                        if 'source' in obj:  # Upload/copy existing object232                            self.server.auth_manager.duplicate_object(object_path=obj['source'],233                                                                      new_object_path=obj['new_path'])234                        else:235                            self.server.auth_manager.add_object(236                                object_path=url,237                                object_key=obj['key'],238                                object_nonce=obj['nonce'],239                                object_tag=obj['tag']240                            )241                    else:242                        print("Received response not HTTP 200, will not store key for {}".format(obj['key']))243            elif op == 'PUT' and 'part_num' in obj:244                if self.enc_res_status[1] == b'200': # Part upload succeeded, add it to database245                    self.server.auth_manager.add_object_part(upload_id=obj['upload_id'], part_num=obj['part_num'], part_size=obj['part_length'], part_tag=obj['part_tag'])246            elif op == 'DELETE':247                self.server.auth_manager.delete_object(url)248        if not self.has_body:249            self.send_headers(False)250            return251        content = self.read_chunks()252        if self.preview and not self.ieof:253            self.cont()254            content = self.read_chunks()255        if op == 'GET':256            multipart_data = self.server.auth_manager.get_object_parts(url)257            if multipart_data is None:258                # Single object GET, not uploaded in parts259                obj_entry = self.server.auth_manager.get_object(object_path=url) # TODO: merge ops in single&multipart uploaded260                if obj_entry is not None: # GET response for known object261                    cp = AESCipher()262                    json_key = {'key': obj_entry.key, 'nonce': obj_entry.nonce, 'tag': obj_entry.tag}263                    content = cp.decrypt(json_key, content)264                else: # No object found in database265                    pass266            elif multipart_data is not None:267                # GET object uploaded in parts268                decrypted_content = b''269                obj_entry = self.server.auth_manager.get_object(object_path=url)270                index = 0271                for item in multipart_data:  # Decrypt each offset from database values272                    cp = AESCipher()273                    json_key = {'key': obj_entry.key, 'nonce': obj_entry.nonce, 'tag': item['part_tag']}274                    piece = content[index:index + item['part_size']]275                    decrypted_piece = cp.decrypt(json_key, piece)276                    decrypted_content += decrypted_piece277                    index += item['part_size']278                content = decrypted_content279        if op == 'POST':280            if re.search("\.s3\.(.+)\.amazonaws\.com/\?delete$", request_url) is not None:281                # Bucket paths starting with /? (no file key) are always non-upload config requests282                xmldata = et.fromstring(content.decode())283                for item in xmldata.findall("./{*}Deleted/{*}Key"):284                    key = request_url[:-7] + item.text285                    self.server.auth_manager.delete_object(key)286            elif obj is not None and 'multipart' in obj:  # Confirmed multipart upload initiated287                xmldata = et.fromstring(content.decode())288                upload_id = xmldata.find("./{*}UploadId").text289                self.server.auth_manager.new_multipart_upload(obj['object_key'], upload_id)290        self.send_headers(True)291        self.write_chunk(content)292        self.write_chunk(b'')293class S3_encryption_icap(object):294    def __init__(self, port, auth_manager):295        if port < 1024 or port > 65535:296            self.port = 1344297        else:298            self.port = port299        self.server = ThreadingSimpleServer((b'', self.port), ICAPHandler)300        self.server.auth_manager = auth_manager301        self.run = False302    def start_server(self):303        t = threading.current_thread()304        self.reload_squid()305        # Start handle request loop306        while getattr(t, "run", False):307            self.server.handle_request()308        print("stopped " + str(getattr(t, "run", "def")))309    def stop_server(self):310        self.server.shutdown()311        self.run = False312    def reload_squid(self): #Reload Squid process using system script313        script_path = os.path.join(BASE_DIR, "app/squid_reload.sh")314        try:315            subprocess.run(["sudo", script_path], check=True, capture_output=True)316        except subprocess.CalledProcessError as e:317            print(e)318            print("Could not reload squid. Make sure user {} has non-interactive execute permission for {}".format(getpass.getuser(), script_path))319            return e.stderr...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.