Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use compute method in pytest-benchmark

Best Python code snippet using pytest-benchmark

views.py

Source:views.py

1import hashlib2import json3import subprocess4from django.core.exceptions import PermissionDenied5from django.shortcuts import get_object_or_4046from rest_framework import status7from rest_framework.decorators import detail_route8from rest_framework.generics import RetrieveUpdateAPIView, ListAPIView9from rest_framework.parsers import MultiPartParser10from rest_framework.permissions import IsAuthenticated11from rest_framework.response import Response12from rest_framework.views import APIView13from rest_framework.viewsets import ModelViewSet, ViewSet14from tasks import poweron_nodes, poweroff_nodes15from tasks import submit_computeset, cancel_computeset, attach_iso16import hostlist17from models import Cluster, Compute, ComputeSet, Frontend18from serializers import ComputeSerializer, ComputeSetSerializer, FullComputeSetSerializer19from serializers import ClusterSerializer, FrontendSerializer, ProjectSerializer20from serializers import UserDetailsSerializer21import re, os, random, string, datetime22from django.db.models import Q23from rest_framework.authentication import BasicAuthentication24from rest_framework.permissions import IsAuthenticated25# #################################################26#  CLUSTER27# #################################################28class ClusterViewSet(ModelViewSet):29    lookup_field = 'cluster_name'30    serializer_class = ClusterSerializer31    def get_queryset(self):32        """Obtain details about all clusters."""33        clusters = Cluster.objects.filter(34            project__in=self.request.user.groups.all())35        return clusters36    def retrieve(self, request, cluster_name, format=None):37        """Obtain details about the named cluster."""38        clust = get_object_or_404(Cluster, name=cluster_name)39        if not clust.project in request.user.groups.all():40            raise PermissionDenied()41        serializer = ClusterSerializer(clust)42        return Response(serializer.data)43    def destroy(self, request, cluster_name, format=None):44        """Destroy the named cluster."""45        return Response("destroy todo")46# #################################################47#  COMPUTE48# #################################################49class ComputeViewSet(ViewSet):50    lookup_field = 'compute_name'51    serializer_class = ClusterSerializer52    def retrieve(self, request, compute_name_cluster_name, compute_name, format=None):53        """Obtain the details of a named compute resource in a named cluster."""54        compute = get_object_or_404(55            Compute, name=compute_name, cluster__name=compute_name_cluster_name)56        if not compute.cluster.project in request.user.groups.all():57            raise PermissionDenied()58        serializer = ComputeSerializer(compute)59        return Response(serializer.data)60    def destroy(self, request, compute_name_cluster_name, compute_name, format=None):61        """Destroy the named compute resource in a named cluster."""62        return Response("todo")63    @detail_route(methods=['put'])64    def shutdown(self, request, compute_name_cluster_name, compute_name, format=None):65        """Shutdown the named compute resource in a named cluster.66        """67        compute = get_object_or_404(68            Compute, name=compute_name, cluster__name=compute_name_cluster_name)69        if not compute.cluster.project in request.user.groups.all():70            raise PermissionDenied()71        if(not compute.computeset.filter(state=ComputeSet.CSET_STATE_RUNNING).exists()):72            return Response("Compute is not a member of an active computeset",73                            status=status.HTTP_400_BAD_REQUEST)74        poweroff_nodes.delay([compute.rocks_name], "shutdown")75        return Response(status=204)76    @detail_route(methods=['put'])77    def reboot(self, request, compute_name_cluster_name, compute_name, format=None):78        """Reboot the named compute resource in a named cluster.79        """80        compute = get_object_or_404(81            Compute, name=compute_name, cluster__name=compute_name_cluster_name)82        if not compute.cluster.project in request.user.groups.all():83            raise PermissionDenied()84        if(not compute.computeset.filter(state=ComputeSet.CSET_STATE_RUNNING).exists()):85            return Response("Compute is not a member of an active computeset",86                            status=status.HTTP_400_BAD_REQUEST)87        poweroff_nodes.delay([compute.rocks_name], "reboot")88        return Response(status=204)89    @detail_route(methods=['put'])90    def reset(self, request, compute_name_cluster_name, compute_name, format=None):91        """Reset the named compute resource in a named cluster.92        """93        compute = get_object_or_404(94            Compute, name=compute_name, cluster__name=compute_name_cluster_name)95        if not compute.cluster.project in request.user.groups.all():96            raise PermissionDenied()97        if(not compute.computeset.filter(state=ComputeSet.CSET_STATE_RUNNING).exists()):98            return Response("Compute is not a member of an active computeset",99                            status=status.HTTP_400_BAD_REQUEST)100        poweroff_nodes.delay([compute.rocks_name], "reset")101        return Response(status=204)102    @detail_route(methods=['put'])103    def poweroff(self, request, compute_name_cluster_name, compute_name, format=None):104        """Power off the named compute resource in a named cluster.105        """106        compute = get_object_or_404(107            Compute, name=compute_name, cluster__name=compute_name_cluster_name)108        if not compute.cluster.project in request.user.groups.all():109            raise PermissionDenied()110        if(not compute.computeset.filter(state=ComputeSet.CSET_STATE_RUNNING).exists()):111            return Response("Compute is not a member of an active computeset",112                            status=status.HTTP_400_BAD_REQUEST)113        poweroff_nodes.delay([compute.rocks_name], "poweroff")114        return Response(status=204)115    @detail_route(methods=['put'])116    def poweron(self, request, compute_name_cluster_name, compute_name, format=None):117        """Power on the named compute resource in a named cluster.118        """119        compute = get_object_or_404(120            Compute, name=compute_name, cluster__name=compute_name_cluster_name)121        if not compute.cluster.project in request.user.groups.all():122            raise PermissionDenied()123        if(not compute.computeset.filter(state=ComputeSet.CSET_STATE_RUNNING).exists()):124            return Response("Compute is not a member of an active computeset",125                            status=status.HTTP_400_BAD_REQUEST)126        poweron_nodes.delay([compute.rocks_name])127        return Response(status=204)128    @detail_route(methods=['put'])129    def attach_iso(self, request, compute_name_cluster_name, compute_name, format=None):130        """Attach an ISO to the named compute resource in a named cluster.131        """132        compute = get_object_or_404(133            Compute, name=compute_name, cluster__name=compute_name_cluster_name)134        if not compute.cluster.project in request.user.groups.all():135            raise PermissionDenied()136        if not "iso_name" in request.GET:137            return Response("Please provide the iso_name", status=400)138        attach_iso.delay([compute.rocks_name], request.GET["iso_name"])139        return Response(status=204)140    @detail_route(methods=['post'])141    def rename(self, request, compute_name_cluster_name, compute_name, format=None):142        """Rename the named compute resource in a named cluster.143        """144        compute = get_object_or_404(145            Compute, name=compute_name, cluster__name=compute_name_cluster_name)146        if not compute.cluster.project in request.user.groups.all():147            raise PermissionDenied()148        new_name = request.data.get("name")149        if(not re.match('^[a-zA-Z0-9_-]+$',new_name)):150            return Response("New name can opnly contain alphanumeric symbols, digits and '-_'.",151                            status=status.HTTP_400_BAD_REQUEST)152        compute.name = new_name153        compute.save()154        return Response(status=204)155# #################################################156#  CONSOLE157##################################################158def get_console(request, console_compute_name, nucleus_name=None, is_frontend=False):159    """Open VNC console to named resource."""160    resp = "Success"161    sleep_time = 15162    from xml.dom.minidom import parse, parseString163    import libvirt164    if(is_frontend):165        compute = Frontend.objects.get(rocks_name=console_compute_name)166    else:167        compute = Compute.objects.get(rocks_name=console_compute_name)168    physical_host = compute.physical_host169    if(not physical_host):170        return Response("The VM is not running",171                        status=status.HTTP_400_BAD_REQUEST)172    hypervisor = libvirt.open("qemu+tls://%s.comet/system?pkipath=/var/secrets/cometvc" %173                              physical_host)174    domU = hypervisor.lookupByName(compute.name)175    # Grab the current XML definition of the domain...176    flags = libvirt.VIR_DOMAIN_XML_SECURE177    domU_xml = parseString(domU.XMLDesc(flags))178    # Parse out the <graphics>...</graphics> device node...179    for gd in domU_xml.getElementsByTagName('graphics'):180        xml = gd.toxml()181    duration = 3600182    password = ''.join(183        random.SystemRandom().choice(184            string.ascii_uppercase +185            string.ascii_lowercase +186            string.digits) for _ in range(16))187    # Generate a new passwdValidTo string...188    dt1 = datetime.datetime.utcnow()189    dt2 = dt1 + datetime.timedelta(0, int(duration))190    timestr = dt2.strftime("%Y-%m-%dT%H:%M:%S")191    # Modify the passwd and passwdValidUntil fields...192    gd.setAttribute('passwd', password)193    gd.setAttribute('passwdValidTo', timestr)194    port = gd.getAttribute("port")195    # Apply the change to the domain...196    flags = libvirt.VIR_DOMAIN_DEVICE_MODIFY_FORCE | \197        libvirt.VIR_DOMAIN_DEVICE_MODIFY_LIVE198    retval = domU.updateDeviceFlags(gd.toxml(), flags)199    cmd = ['/usr/bin/sudo',200           '-u',201           'nucleus_comet',202           '/opt/nucleus-scripts/bin/open_tunnel.py',203           '-H',204           '{hostname}'.format(hostname=physical_host),205           '-p',206           '{hostport}'.format(hostport=port),207           '-s',208           '{duration}'.format(duration=sleep_time)]209    try:210        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)211    except OSError as e:212        resp = "Execution failed: %s" % (e)213        return Response(resp)214    tun_port = ''215    tun_port = proc.stdout.readline().strip()216    url_base = "/nucleus-guacamole-0.9.8/index.html?hostname=localhost"217    url = request.build_absolute_uri("%s&port=%s&token=%s&host=%s" % (url_base, tun_port, password, nucleus_name))218    response = Response(219        url,220        status=303,221        headers={'Location': url})222    return response223class ConsoleViewSet(ViewSet):224    """Open VNC console to named compute resource."""225    authentication_classes = (BasicAuthentication,)226    permission_classes = (IsAuthenticated,)227    def retrieve(self, request, compute_name_cluster_name, console_compute_name, format=None):228        compute = get_object_or_404(229            Compute, name=console_compute_name, cluster__name=compute_name_cluster_name)230        if not compute.cluster.project in request.user.groups.all():231            raise PermissionDenied()232        return get_console(request, compute.rocks_name, console_compute_name)233class FrontendConsoleViewSet(ViewSet):234    """Open VNC console to name frontend resource."""235    authentication_classes = (BasicAuthentication,)236    permission_classes = (IsAuthenticated,)237    def retrieve(self, request, console_cluster_name, format=None):238        clust = get_object_or_404(Cluster, name=console_cluster_name)239        if not clust.project in request.user.groups.all():240            raise PermissionDenied()241        return get_console(request, clust.frontend.rocks_name, console_cluster_name, True)242##################################################243#  COMPUTESET244# #################################################245class ComputeSetViewSet(ModelViewSet):246    lookup_field = 'computeset_id'247    serializer_class = ComputeSetSerializer248    def get_queryset(self):249        """Obtain the details of all ComputeSets."""250        cset = ComputeSet.objects.filter(251            cluster__project__in=self.request.user.groups.all())252        states = self.request.GET.getlist('state')253        #state = self.request.query_params.get('state', None)254        if states:255            cset = cset.filter(state__in=states)256        return cset257    def retrieve(self, request, computeset_id, format=None):258        """Obtain the details of the identified ComputeSet."""259        cset = get_object_or_404(ComputeSet, pk=computeset_id)260        if not cset.cluster.project in request.user.groups.all():261            raise PermissionDenied()262        serializer = ComputeSetSerializer(cset)263        return Response(serializer.data)264    @detail_route(methods=['put'])265    def poweroff(self, request, computeset_id, format=None):266        """Power off the identified ComputeSet."""267        cset = ComputeSet.objects.get(pk=computeset_id)268        if not cset.cluster.project in request.user.groups.all():269            raise PermissionDenied()270        computes = []271        for compute in cset.computes.all():272            computes.append(compute.rocks_name)273        poweroff_nodes.delay(computes, "poweroff")274        cancel_computeset.delay(FullComputeSetSerializer(cset).data)275        return Response(status=204)276    def poweron(self, request, format=None):277        """ Power on a set of computes creating a ComputeSet."""278        clust = get_object_or_404(Cluster, name=request.data["cluster"])279        if not clust.project in request.user.groups.all():280            raise PermissionDenied()281        walltime_mins = request.data.get("walltime_mins")282        if not walltime_mins:283            walltime_mins = 2880284        nodes = []285        hosts = []286        if request.data.get("computes"):287            if request.data["computes"] is list:288                for obj in request.data["computes"]:289                    nodes.append(obj["name"])290                    hosts.append(obj["host"])291            else:292                nodes = hostlist.expand_hostlist("%s" % request.data["computes"])293                if request.data.get("hosts"):294                    hosts = hostlist.expand_hostlist("%s" % request.data["hosts"])295        elif request.data.get("count"):296            computes_selected = Compute.objects.filter(cluster=clust).exclude(297                    computeset__state__in=[298                        ComputeSet.CSET_STATE_CREATED,299                        ComputeSet.CSET_STATE_SUBMITTED,300                        ComputeSet.CSET_STATE_RUNNING,301                        ComputeSet.CSET_STATE_ENDING]302                ).exclude(state="active").filter(Q(image_state="unmapped") | Q(image_state__isnull=True)).exclude(image_locked=True)[:int(request.data["count"])]303            nodes.extend([comp.name for comp in computes_selected])304            if(len(nodes) < int(request.data["count"]) or int(request.data["count"]) == 0):305                return Response("There are %i nodes available for starting. Requested number should be greater than zero."%len(nodes),306                                status=status.HTTP_400_BAD_REQUEST)307        if hosts and len(nodes) != len(hosts):308            return Response("The length of hosts should be equal to length of nodes",309                            status=status.HTTP_400_BAD_REQUEST)310        cset = ComputeSet()311        cset.cluster = clust312        cset.user = clust.username313        if(request.data.get("allocation")):314            cset.account = request.data["allocation"]315        elif(clust.allocations.count() == 1):316            cset.account = clust.allocations.get().allocation317        else:318            return Response("Please specify the allocation",319                    status=status.HTTP_400_BAD_REQUEST)320        if(not clust.allocations.filter(allocation=cset.account).exists()):321            return Response("Allocation %s does not belong to the cluster."%cset.account,322                    status=status.HTTP_400_BAD_REQUEST)323        cset.walltime_mins = walltime_mins324        cset.jobid = None325        cset.name = None326        cset.nodelist = ""327        cset.reservation = request.data.get("reservation")328        cset.state = ComputeSet.CSET_STATE_CREATED329        cset.node_count = len(nodes)330        cset.save()331        for node in nodes:332            compute = Compute.objects.get(name=node, cluster=clust)333            other_cs_query = ComputeSet.objects.filter(computes__id__exact=compute.id).exclude(334                state__in=[ComputeSet.CSET_STATE_COMPLETED, ComputeSet.CSET_STATE_FAILED, ComputeSet.CSET_STATE_CANCELLED])335            if other_cs_query.exists():336                cset.delete()337                err_cs = other_cs_query.get()338                return Response("The compute %s belongs to computeset %s which is in %s state" % (node, err_cs.id, err_cs.state), status=status.HTTP_400_BAD_REQUEST)339            if (compute.image_state not in ["unmapped", None]) or compute.image_locked:340                cset.delete()341                return Response("The node %s's image is in %s state and image locked status is %s. Please contact the user support if the VM is not running." %(node, compute.image_state, compute.image_locked), status=status.HTTP_400_BAD_REQUEST)342            if compute.cluster.name != request.data["cluster"]:343                cset.delete()344                return Response("The node %s does not belong to the cluster %s, belongs to %s" % (node, request.data["cluster"], compute.cluster.name), status=status.HTTP_400_BAD_REQUEST)345            cset.computes.add(compute)346        submit_computeset.delay(FullComputeSetSerializer(cset).data)347        # We should only poweron computes after entering jobscript and348        # finishing the PROLOG on all allocated nodes. At that point the349        # nodelist will be returned and we can call poweron_nodeset()350        #poweron_nodeset.delay(nodes, hosts)351        location = "/nucleus/v1/computeset/%s" % (cset.id)352        serializer = ComputeSetSerializer(cset)353        response = Response(354            serializer.data,355            status=201,356            headers={'Location': request.build_absolute_uri(location)})357        return response358    @detail_route(methods=['put'])359    def shutdown(self, request, computeset_id, format=None):360        """Shutdown the nodes in the identified ComputeSet."""361        cset = ComputeSet.objects.get(pk=computeset_id)362        if not cset.cluster.project in request.user.groups.all():363            raise PermissionDenied()364        computes = []365        for compute in cset.computes.all():366            computes.append(compute.rocks_name)367            if compute.cluster.name != request.data["cluster"]:368                cset.delete()369                return Response("The node %s does not belong to the cluster %s, belongs to %s" % (node, request.data["cluster"], compute.cluster.name), status=status.HTTP_400_BAD_REQUEST)370            cset.computes.add(compute)371        submit_computeset.delay(FullComputeSetSerializer(cset).data)372        # We should only poweron computes after entering jobscript and373        # finishing the PROLOG on all allocated nodes. At that point the374        # nodelist will be returned and we can call poweron_nodeset()375        #poweron_nodeset.delay(nodes, hosts)376        location = "/nucleus/v1/computeset/%s" % (cset.id)377        serializer = ComputeSetSerializer(cset)378        response = Response(379            serializer.data,380            status=201,381            headers={'Location': request.build_absolute_uri(location)})382        return response383    @detail_route(methods=['put'])384    def shutdown(self, request, computeset_id, format=None):385        """Shutdown the nodes in the identified ComputeSet."""386        cset = ComputeSet.objects.get(pk=computeset_id)387        if not cset.cluster.project in request.user.groups.all():388            raise PermissionDenied()389        computes = []390        for compute in cset.computes.all():391            computes.append(compute.rocks_name)392            if compute.cluster.name != request.data["cluster"]:393                cset.delete()394                return Response("The node %s does not belong to the cluster %s, belongs to %s" % (node, request.data["cluster"], compute.cluster.name), status=status.HTTP_400_BAD_REQUEST)395            cset.computes.add(compute)396        submit_computeset.delay(FullComputeSetSerializer(cset).data)397        # We should only poweron computes after entering jobscript and398        # finishing the PROLOG on all allocated nodes. At that point the399        # nodelist will be returned and we can call poweron_nodeset()400        #poweron_nodeset.delay(nodes, hosts)401        location = "/nucleus/v1/computeset/%s" % (cset.id)402        serializer = ComputeSetSerializer(cset)403        response = Response(404            serializer.data,405            status=201,406            headers={'Location': request.build_absolute_uri(location)})407        return response408    @detail_route(methods=['put'])409    def shutdown(self, request, computeset_id, format=None):410        """Shutdown the nodes in the identified ComputeSet."""411        cset = ComputeSet.objects.get(pk=computeset_id)412        if not cset.cluster.project in request.user.groups.all():413            raise PermissionDenied()414        computes = []415        for compute in cset.computes.all():416            computes.append(compute.rocks_name)417        poweroff_nodes.delay(computes, "shutdown")418        cancel_computeset.delay(FullComputeSetSerializer(cset).data)419        return Response(status=204)420    @detail_route(methods=['put'])421    def reboot(self, request, computeset_id, format=None):422        """Reboot the nodes in the identified ComputeSet."""423        cset = ComputeSet.objects.get(pk=computeset_id)424        if not cset.cluster.project in request.user.groups.all():425            raise PermissionDenied()426        computes = []427        for compute in cset.computes.all():428            computes.append(compute.rocks_name)429        poweroff_nodes.delay(computes, "reboot")430        return Response(status=204)431    @detail_route(methods=['put'])432    def reset(self, request, computeset_id, format=None):433        """Reset the nodes in the identified ComputeSet."""434        cset = ComputeSet.objects.get(pk=computeset_id)435        if not cset.cluster.project in request.user.groups.all():436            raise PermissionDenied()437        computes = []438        for compute in cset.computes.all():439            computes.append(compute.rocks_name)440        poweroff_nodes.delay(computes, "reset")441        return Response(status=204)442# #################################################443#  FRONTEND444# #################################################445class FrontendViewSet(ViewSet):446    def retrieve(self, request, frontend_cluster_name, format=None):447        """Obtain the details of a frontend resource in a named cluster."""448        clust = get_object_or_404(Cluster, name=frontend_cluster_name)449        if not clust.project in request.user.groups.all():450            raise PermissionDenied()451        serializer = FrontendSerializer(clust.frontend)452        return Response(serializer.data)453    @detail_route(methods=['put'])454    def shutdown(self, request, frontend_cluster_name, format=None):455        """Shutdown the frontend of a named cluster."""456        clust = get_object_or_404(Cluster, name=frontend_cluster_name)457        if not clust.project in request.user.groups.all():458            raise PermissionDenied()459        poweroff_nodes.delay([clust.frontend.rocks_name], "shutdown")460        return Response(status=204)461    @detail_route(methods=['put'])462    def reboot(self, request, frontend_cluster_name, format=None):463        """Reboot the frontend of a named cluster."""464        clust = get_object_or_404(Cluster, name=frontend_cluster_name)465        if not clust.project in request.user.groups.all():466            raise PermissionDenied()467        poweroff_nodes.delay([clust.frontend.rocks_name], "reboot")468        return Response(status=204)469    @detail_route(methods=['put'])470    def reset(self, request, frontend_cluster_name, format=None):471        """Reset the frontend of a named cluster."""472        clust = get_object_or_404(Cluster, name=frontend_cluster_name)473        if not clust.project in request.user.groups.all():474            raise PermissionDenied()475        poweroff_nodes.delay([clust.frontend.rocks_name], "reset")476        return Response(status=204)477    @detail_route(methods=['put'])478    def poweron(self, request, frontend_cluster_name, format=None):479        """Power on the frontend of a named cluster."""480        clust = get_object_or_404(Cluster, name=frontend_cluster_name)481        if not clust.project in request.user.groups.all():482            raise PermissionDenied()483        poweron_nodes.delay([clust.frontend.rocks_name])484        return Response(status=204)485    @detail_route(methods=['put'])486    def poweroff(self, request, frontend_cluster_name, format=None):487        """Power off the frontend of a named cluster."""488        clust = get_object_or_404(Cluster, name=frontend_cluster_name)489        if not clust.project in request.user.groups.all():490            raise PermissionDenied()491        poweroff_nodes.delay([clust.frontend.rocks_name], "poweroff")492        return Response(status=204)493    @detail_route(methods=['put'])494    def attach_iso(self, request, frontend_cluster_name, format=None):495        """Attach an ISO to the frontendresource in a named cluster.496        """497        clust = get_object_or_404(Cluster, name=frontend_cluster_name)498        if not clust.project in request.user.groups.all():499            raise PermissionDenied()500        if not "iso_name" in request.GET:501            return Response("Please provide the iso_name", status=400)502        attach_iso.delay([clust.frontend.rocks_name], request.GET["iso_name"])503        return Response(status=204)504# #################################################505#  USER506# #################################################507class UserDetailsView(RetrieveUpdateAPIView):508    """509    Returns User's details in JSON format.510    Accepts the following GET parameters: token511    Accepts the following POST parameters:512        Required: token513        Optional: email, first_name, last_name and UserProfile fields514    Returns the updated UserProfile and/or User object.515    """516    serializer_class = UserDetailsSerializer517    permission_classes = (IsAuthenticated,)518    def get_object(self):519        return self.request.user520# #################################################521#  PROJECT522# #################################################523class ProjectListView(ListAPIView):524    """Returns project details."""525    serializer_class = ProjectSerializer526    def get_queryset(self):527        return self.request.user.groups.all()528class ImageUploadView(APIView):529    parser_classes = (MultiPartParser,)530    def get(self, request, format=None):531        filepath = '/mnt/images/public'532        return Response(["public/%s"%dir for dir in os.listdir(filepath)])533    def post(self, request, format=None):534        file_obj = request.FILES['file']535        try:536            filepath = '/mnt/images/public/%s' % (file_obj.name)537            if not request.META.get('HTTP_MD5'):538                return Response("md5 was not provided", status=400)539            if request.META['HTTP_MD5'] != md5_for_file(file_obj.chunks()):540                return Response("md5 does not match the file", status=400)541            with open(filepath, 'wb+') as destination:542                for chunk in file_obj.chunks():543                    destination.write(chunk)544            return Response(status=204)545        finally:546            if(file_obj):547                file_obj.close()548def md5_for_file(chunks):549    md5 = hashlib.md5()550    for data in chunks:551        md5.update(data)...

fake_policy.py

Source:fake_policy.py

1# Copyright (c) 2012 OpenStack Foundation2#3# Licensed under the Apache License, Version 2.0 (the "License"); you may4# not use this file except in compliance with the License. You may obtain5# a copy of the License at6#7#      http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the12# License for the specific language governing permissions and limitations13# under the License.14policy_data = """15{16    "admin_api": "is_admin:True",17    "cells_scheduler_filter:TargetCellFilter": "is_admin:True",18    "context_is_admin": "role:admin or role:administrator",19    "compute:create": "",20    "compute:create:attach_network": "",21    "compute:create:attach_volume": "",22    "compute:get": "",23    "compute:get_all": "",24    "compute:get_all_tenants": "",25    "compute:update": "",26    "compute:get_instance_metadata": "",27    "compute:get_all_instance_metadata": "",28    "compute:get_all_instance_system_metadata": "",29    "compute:update_instance_metadata": "",30    "compute:delete_instance_metadata": "",31    "compute:get_instance_faults": "",32    "compute:get_diagnostics": "",33    "compute:get_instance_diagnostics": "",34    "compute:get_lock": "",35    "compute:lock": "",36    "compute:unlock": "",37    "compute:unlock_override": "is_admin:True",38    "compute:get_vnc_console": "",39    "compute:get_spice_console": "",40    "compute:get_rdp_console": "",41    "compute:get_serial_console": "",42    "compute:get_console_output": "",43    "compute:associate_floating_ip": "",44    "compute:reset_network": "",45    "compute:inject_network_info": "",46    "compute:add_fixed_ip": "",47    "compute:remove_fixed_ip": "",48    "compute:attach_volume": "",49    "compute:detach_volume": "",50    "compute:attach_interface": "",51    "compute:detach_interface": "",52    "compute:set_admin_password": "",53    "compute:rescue": "",54    "compute:unrescue": "",55    "compute:suspend": "",56    "compute:resume": "",57    "compute:pause": "",58    "compute:unpause": "",59    "compute:start": "",60    "compute:stop": "",61    "compute:resize": "",62    "compute:confirm_resize": "",63    "compute:revert_resize": "",64    "compute:rebuild": "",65    "compute:reboot": "",66    "compute:snapshot": "",67    "compute:backup": "",68    "compute:shelve": "",69    "compute:shelve_offload": "",70    "compute:unshelve": "",71    "compute:security_groups:add_to_instance": "",72    "compute:security_groups:remove_from_instance": "",73    "compute:delete": "",74    "compute:soft_delete": "",75    "compute:force_delete": "",76    "compute:restore": "",77    "compute:swap_volume": "",78    "compute:volume_snapshot_create": "",79    "compute:volume_snapshot_delete": "",80    "compute:v3:servers:start": "",81    "compute:v3:servers:stop": "",82    "compute_extension:v3:os-access-ips": "",83    "compute_extension:accounts": "",84    "compute_extension:admin_actions:pause": "",85    "compute_extension:admin_actions:unpause": "",86    "compute_extension:admin_actions:suspend": "",87    "compute_extension:admin_actions:resume": "",88    "compute_extension:admin_actions:lock": "",89    "compute_extension:admin_actions:unlock": "",90    "compute_extension:admin_actions:resetNetwork": "",91    "compute_extension:admin_actions:injectNetworkInfo": "",92    "compute_extension:admin_actions:createBackup": "",93    "compute_extension:admin_actions:migrateLive": "",94    "compute_extension:admin_actions:resetState": "",95    "compute_extension:admin_actions:migrate": "",96    "compute_extension:v3:os-admin-actions:reset_network": "",97    "compute_extension:v3:os-admin-actions:inject_network_info": "",98    "compute_extension:v3:os-admin-actions:reset_state": "",99    "compute_extension:v3:os-admin-password": "",100    "compute_extension:aggregates": "rule:admin_api",101    "compute_extension:v3:os-aggregates:index": "rule:admin_api",102    "compute_extension:v3:os-aggregates:create": "rule:admin_api",103    "compute_extension:v3:os-aggregates:show": "rule:admin_api",104    "compute_extension:v3:os-aggregates:update": "rule:admin_api",105    "compute_extension:v3:os-aggregates:delete": "rule:admin_api",106    "compute_extension:v3:os-aggregates:add_host": "rule:admin_api",107    "compute_extension:v3:os-aggregates:remove_host": "rule:admin_api",108    "compute_extension:v3:os-aggregates:set_metadata": "rule:admin_api",109    "compute_extension:agents": "",110    "compute_extension:v3:os-agents": "",111    "compute_extension:attach_interfaces": "",112    "compute_extension:v3:os-attach-interfaces": "",113    "compute_extension:baremetal_nodes": "",114    "compute_extension:cells": "",115    "compute_extension:cells:create": "rule:admin_api",116    "compute_extension:cells:delete": "rule:admin_api",117    "compute_extension:cells:update": "rule:admin_api",118    "compute_extension:cells:sync_instances": "rule:admin_api",119    "compute_extension:v3:os-cells": "",120    "compute_extension:v3:os-cells:create": "rule:admin_api",121    "compute_extension:v3:os-cells:delete": "rule:admin_api",122    "compute_extension:v3:os-cells:update": "rule:admin_api",123    "compute_extension:v3:os-cells:sync_instances": "rule:admin_api",124    "compute_extension:certificates": "",125    "compute_extension:v3:os-certificates:create": "",126    "compute_extension:v3:os-certificates:show": "",127    "compute_extension:cloudpipe": "",128    "compute_extension:cloudpipe_update": "",129    "compute_extension:config_drive": "",130    "compute_extension:v3:os-config-drive": "",131    "compute_extension:console_output": "",132    "compute_extension:v3:os-console-output": "",133    "compute_extension:consoles": "",134    "compute_extension:v3:os-remote-consoles": "",135    "compute_extension:createserverext": "",136    "compute_extension:v3:os-create-backup": "",137    "compute_extension:deferred_delete": "",138    "compute_extension:v3:os-deferred-delete": "",139    "compute_extension:disk_config": "",140    "compute_extension:evacuate": "is_admin:True",141    "compute_extension:v3:os-evacuate": "is_admin:True",142    "compute_extension:extended_server_attributes": "",143    "compute_extension:v3:os-extended-server-attributes": "",144    "compute_extension:extended_status": "",145    "compute_extension:v3:os-extended-status": "",146    "compute_extension:extended_availability_zone": "",147    "compute_extension:v3:os-extended-availability-zone": "",148    "compute_extension:extended_ips": "",149    "compute_extension:extended_ips_mac": "",150    "compute_extension:extended_vif_net": "",151    "compute_extension:extended_volumes": "",152    "compute_extension:v3:os-extended-volumes": "",153    "compute_extension:v3:os-extended-volumes:swap": "",154    "compute_extension:v3:os-extended-volumes:attach": "",155    "compute_extension:v3:os-extended-volumes:detach": "",156    "compute_extension:v3:extensions:discoverable": "",157    "compute_extension:fixed_ips": "",158    "compute_extension:flavor_access": "",159    "compute_extension:flavor_access:addTenantAccess": "rule:admin_api",160    "compute_extension:flavor_access:removeTenantAccess": "rule:admin_api",161    "compute_extension:v3:os-flavor-access": "",162    "compute_extension:v3:os-flavor-access:remove_tenant_access":163    "rule:admin_api",164    "compute_extension:v3:os-flavor-access:add_tenant_access":165    "rule:admin_api",166    "compute_extension:flavor_disabled": "",167    "compute_extension:v3:os-flavor-disabled": "",168    "compute_extension:flavor_rxtx": "",169    "compute_extension:v3:os-flavor-rxtx": "",170    "compute_extension:flavor_swap": "",171    "compute_extension:flavorextradata": "",172    "compute_extension:flavorextraspecs:index": "",173    "compute_extension:flavorextraspecs:show": "",174    "compute_extension:flavorextraspecs:create": "is_admin:True",175    "compute_extension:flavorextraspecs:update": "is_admin:True",176    "compute_extension:flavorextraspecs:delete": "is_admin:True",177    "compute_extension:v3:flavor-extra-specs:index": "",178    "compute_extension:v3:flavor-extra-specs:show": "",179    "compute_extension:v3:flavor-extra-specs:create": "is_admin:True",180    "compute_extension:v3:flavor-extra-specs:update": "is_admin:True",181    "compute_extension:v3:flavor-extra-specs:delete": "is_admin:True",182    "compute_extension:flavormanage": "",183    "compute_extension:v3:flavor-manage": "",184    "compute_extension:v3:flavors:discoverable": "",185    "compute_extension:floating_ip_dns": "",186    "compute_extension:floating_ip_pools": "",187    "compute_extension:floating_ips": "",188    "compute_extension:floating_ips_bulk": "",189    "compute_extension:fping": "",190    "compute_extension:fping:all_tenants": "is_admin:True",191    "compute_extension:hide_server_addresses": "",192    "compute_extension:v3:os-hide-server-addresses": "",193    "compute_extension:hosts": "rule:admin_api",194    "compute_extension:v3:os-hosts": "rule:admin_api",195    "compute_extension:hypervisors": "rule:admin_api",196    "compute_extension:v3:os-hypervisors": "rule:admin_api",197    "compute_extension:image_size": "",198    "compute_extension:v3:image-size": "",199    "compute_extension:instance_actions": "",200    "compute_extension:v3:os-instance-actions": "",201    "compute_extension:instance_actions:events": "is_admin:True",202    "compute_extension:v3:os-instance-actions:events": "is_admin:True",203    "compute_extension:instance_usage_audit_log": "",204    "compute_extension:keypairs": "",205    "compute_extension:keypairs:index": "",206    "compute_extension:keypairs:show": "",207    "compute_extension:keypairs:create": "",208    "compute_extension:keypairs:delete": "",209    "compute_extension:v3:os-keypairs": "",210    "compute_extension:v3:os-keypairs:index": "",211    "compute_extension:v3:os-keypairs:show": "",212    "compute_extension:v3:os-keypairs:create": "",213    "compute_extension:v3:os-keypairs:delete": "",214    "compute_extension:v3:os-lock-server:lock": "",215    "compute_extension:v3:os-lock-server:unlock": "",216    "compute_extension:v3:os-migrate-server:migrate": "",217    "compute_extension:v3:os-migrate-server:migrate_live": "",218    "compute_extension:multinic": "",219    "compute_extension:v3:os-multinic": "",220    "compute_extension:networks": "",221    "compute_extension:networks:view": "",222    "compute_extension:networks_associate": "",223    "compute_extension:os-tenant-networks": "",224    "compute_extension:v3:os-pause-server:pause": "",225    "compute_extension:v3:os-pause-server:unpause": "",226    "compute_extension:v3:os-pci:pci_servers": "",227    "compute_extension:v3:os-pci:index": "",228    "compute_extension:v3:os-pci:detail": "",229    "compute_extension:v3:os-pci:show": "",230    "compute_extension:quotas:show": "",231    "compute_extension:quotas:update": "",232    "compute_extension:quotas:delete": "",233    "compute_extension:v3:os-quota-sets:show": "",234    "compute_extension:v3:os-quota-sets:update": "",235    "compute_extension:v3:os-quota-sets:delete": "",236    "compute_extension:v3:os-quota-sets:detail": "",237    "compute_extension:quota_classes": "",238    "compute_extension:rescue": "",239    "compute_extension:v3:os-rescue": "",240    "compute_extension:security_group_default_rules": "",241    "compute_extension:security_groups": "",242    "compute_extension:v3:os-security-groups": "",243    "compute_extension:server_diagnostics": "",244    "compute_extension:v3:os-server-diagnostics": "",245    "compute_extension:server_groups": "",246    "compute_extension:server_password": "",247    "compute_extension:v3:os-server-password": "",248    "compute_extension:server_usage": "",249    "compute_extension:v3:os-server-usage": "",250    "compute_extension:v3:os-server-groups": "",251    "compute_extension:services": "",252    "compute_extension:v3:os-services": "",253    "compute_extension:shelve": "",254    "compute_extension:shelveOffload": "",255    "compute_extension:v3:os-shelve:shelve": "",256    "compute_extension:v3:os-shelve:shelve_offload": "",257    "compute_extension:simple_tenant_usage:show": "",258    "compute_extension:simple_tenant_usage:list": "",259    "compute_extension:v3:os-simple-tenant-usage:show": "",260    "compute_extension:v3:os-simple-tenant-usage:list": "",261    "compute_extension:unshelve": "",262    "compute_extension:v3:os-shelve:unshelve": "",263    "compute_extension:v3:os-suspend-server:suspend": "",264    "compute_extension:v3:os-suspend-server:resume": "",265    "compute_extension:users": "",266    "compute_extension:virtual_interfaces": "",267    "compute_extension:virtual_storage_arrays": "",268    "compute_extension:volumes": "",269    "compute_extension:volume_attachments:index": "",270    "compute_extension:volume_attachments:show": "",271    "compute_extension:volume_attachments:create": "",272    "compute_extension:volume_attachments:update": "",273    "compute_extension:volume_attachments:delete": "",274    "compute_extension:v3:os-volumes": "",275    "compute_extension:volumetypes": "",276    "compute_extension:zones": "",277    "compute_extension:availability_zone:list": "",278    "compute_extension:v3:os-availability-zone:list": "",279    "compute_extension:availability_zone:detail": "is_admin:True",280    "compute_extension:v3:os-availability-zone:detail": "is_admin:True",281    "compute_extension:used_limits_for_admin": "is_admin:True",282    "compute_extension:v3:os-used-limits": "is_admin:True",283    "compute_extension:migrations:index": "is_admin:True",284    "compute_extension:v3:os-migrations:index": "is_admin:True",285    "compute_extension:os-assisted-volume-snapshots:create": "",286    "compute_extension:os-assisted-volume-snapshots:delete": "",287    "compute_extension:console_auth_tokens": "is_admin:True",288    "compute_extension:v3:os-console-auth-tokens": "is_admin:True",289    "compute_extension:os-server-external-events:create": "rule:admin_api",290    "compute_extension:v3:os-server-external-events:create": "rule:admin_api",291    "volume:create": "",292    "volume:get": "",293    "volume:get_all": "",294    "volume:get_volume_metadata": "",295    "volume:delete": "",296    "volume:update": "",297    "volume:delete_volume_metadata": "",298    "volume:update_volume_metadata": "",299    "volume:attach": "",300    "volume:detach": "",301    "volume:reserve_volume": "",302    "volume:unreserve_volume": "",303    "volume:begin_detaching": "",304    "volume:roll_detaching": "",305    "volume:check_attach": "",306    "volume:check_detach": "",307    "volume:initialize_connection": "",308    "volume:terminate_connection": "",309    "volume:create_snapshot": "",310    "volume:delete_snapshot": "",311    "volume:get_snapshot": "",312    "volume:get_all_snapshots": "",313    "volume_extension:volume_admin_actions:reset_status": "rule:admin_api",314    "volume_extension:snapshot_admin_actions:reset_status": "rule:admin_api",315    "volume_extension:volume_admin_actions:force_delete": "rule:admin_api",316    "volume_extension:volume_actions:upload_image": "",317    "volume_extension:types_manage": "",318    "volume_extension:types_extra_specs": "",319    "network:get_all": "",320    "network:get": "",321    "network:create": "",322    "network:delete": "",323    "network:associate": "",324    "network:disassociate": "",325    "network:get_vifs_by_instance": "",326    "network:get_vif_by_mac_address": "",327    "network:allocate_for_instance": "",328    "network:deallocate_for_instance": "",329    "network:validate_networks": "",330    "network:get_instance_uuids_by_ip_filter": "",331    "network:get_instance_id_by_floating_address": "",332    "network:setup_networks_on_host": "",333    "network:get_floating_ip": "",334    "network:get_floating_ip_pools": "",335    "network:get_floating_ip_by_address": "",336    "network:get_floating_ips_by_project": "",337    "network:get_floating_ips_by_fixed_address": "",338    "network:allocate_floating_ip": "",339    "network:deallocate_floating_ip": "",340    "network:associate_floating_ip": "",341    "network:disassociate_floating_ip": "",342    "network:release_floating_ip": "",343    "network:migrate_instance_start": "",344    "network:migrate_instance_finish": "",345    "network:get_fixed_ip": "",346    "network:get_fixed_ip_by_address": "",347    "network:add_fixed_ip_to_instance": "",348    "network:remove_fixed_ip_from_instance": "",349    "network:add_network_to_project": "",350    "network:get_instance_nw_info": "",351    "network:get_dns_domains": "",352    "network:add_dns_entry": "",353    "network:modify_dns_entry": "",354    "network:delete_dns_entry": "",355    "network:get_dns_entries_by_address": "",356    "network:get_dns_entries_by_name": "",357    "network:create_private_dns_domain": "",358    "network:create_public_dns_domain": "",359    "network:delete_dns_domain": "",360    "network:attach_external_network": "rule:admin_api"361}...

implementations.py

Source:implementations.py

1""" Module containing all implementations of ML techniques required for the project """2import numpy as np3from helpers import (batch_iter, compute_gradient_hinge,4                     compute_gradient_mse, compute_gradient_nll,5                     compute_hessian_nll, compute_loss, compute_loss_hinge,6                     compute_loss_nll, compute_subgradient_mae,7                     map_target_classes_to_boolean)8def least_squares_GD(y, x, initial_w, max_iters, gamma, mae=False, threshold=1e-5):9    """10    Implementation of the Gradient Descent optimization algorithm for linear regression11    Can be run with both MSE and MAE loss12    :param x: data matrix, numpy ndarray with shape with shape (N, D),13              where N is the number of samples and D is the number of features14    :param y: vector of target values, numpy array with dimensions (N, 1)15    :param initial_w: vector of initial weights, numpy array with dimensions (D, 1)16    :param max_iters: how many iterations to run the algorithm, integer17    :param gamma: learning rate, positive float value18    :param mae: whether to use MAE loss, boolean, optional, the default value is False19    :param threshold: convergence threshold, positive float value20    :returns: (final weights, final loss value), tuple21    """22    # Set the initial values for the weights23    w = initial_w24    # Compute the initial loss value25    prev_loss = compute_loss(y, x, initial_w, mae)26    for n_iter in range(max_iters):27        # Compute the total gradient (or subgradient if MAE loss is used)28        grd = compute_subgradient_mae(y, x, w) if mae else compute_gradient_mse(y, x, w)29        # Update the weights using the gradient and learning rate30        w -= gamma * grd31        # Compute the current loss and test convergence32        loss = compute_loss(y, x, w, mae)33        if abs(loss - prev_loss) < threshold:34            print(f'converged at iter : {n_iter}')35            break36        prev_loss = loss.copy()37    # Compute the final loss value38    loss = compute_loss(y, x, w, mae)39    return w, loss40def least_squares_SGD(y, x, initial_w, max_iters, gamma, mae=False, threshold=1e-5):41    """42    Implementation of the Stochastic Gradient Descent optimization algorithm for linear regression43    Can be run with both MSE and MAE loss44    :param x: data matrix, numpy ndarray with shape with shape (N, D),45              where N is the number of samples and D is the number of features46    :param y: vector of target values, numpy array with dimensions (N, 1)47    :param initial_w: vector of initial weights, numpy array with dimensions (D, 1)48    :param max_iters: how many iterations to run the algorithm, integer49    :param gamma: learning rate, positive float value50    :param mae: whether to use MAE loss, boolean, optional, the default value is False51    :param threshold: convergence threshold, positive float value52    :returns: (final weights, final loss value), tuple53    """54    # Set the initial values for the weights55    w = initial_w56    # Compute the initial loss value57    prev_loss = compute_loss(y, x, initial_w, mae)58    # Use the helper function batch_iter from Exercise 2,59    # to get a random sample from the data in the form (y_n, x_n) for each iteration60    for n_iter in range(max_iters):61        for y_n, x_n in batch_iter(y, x, batch_size=1, num_batches=1):62            # Compute the gradient for only one sample (or subgradient if MAE loss is used)63            grd = compute_subgradient_mae(y_n, x_n, w) if mae else compute_gradient_mse(y_n, x_n, w)64            # Update the weights using the gradient and learning rate65            w = w - gamma * grd66        # Compute the current loss and test convergence67        loss = compute_loss(y, x, w, mae)     68        if abs(loss - prev_loss) < threshold:69            print(f'converged at iter : {n_iter}')70            break   71        prev_loss = loss.copy()72    # Compute the final loss value73    loss = compute_loss(y, x, w, mae)74    return w, loss75def least_squares(y, x):76    """77    Calculate the least squares solution explicitly using the normal equations78    :param x: data matrix, numpy ndarray with shape with shape (N, D),79              where N is the number of samples and D is the number of features80    :param y: vector of target values, numpy array with dimensions (N, 1)81    :raises LinAlgError: if the Gram matrix has no inverse82    :returns: (weights, loss value), tuple83    """84    # Compute the Gram matrix85    gram = x.T @ x86    # Use the normal equations to find the best weights87    w = np.linalg.solve(gram, x.T @ y)88    # Compute the loss89    loss = compute_loss(y, x, w)90    return w, loss91def ridge_regression(y, x, lambda_):92    """93    Calculate the least squares solution with L2 regularization explicitly using the normal equations94    :param x: data matrix, numpy ndarray with shape with shape (N, D),95              where N is the number of samples and D is the number of features96    :param y: vector of target values, numpy array with dimensions (N, 1)97    :param lambda_: regularization coefficient, positive float value98    :returns: (weights, loss value), tuple99    """100    # Compute the Gram matrix and update it with the regularization term101    gram = x.T @ x102    gram += 2 * x.shape[0] * lambda_ * np.identity(gram.shape[0])103    # Use the normal equations to find the best weights104    w = np.linalg.solve(gram, x.T @ y)105    # Compute the loss106    loss = compute_loss(y, x, w)107    return w, loss108def logistic_regression(y, x, initial_w, max_iters, gamma, threshold=1e-2):109    """110    Implementation of the Newton optimization algorithm for logistic regression111    :param x: data matrix, numpy ndarray with shape with shape (N, D),112              where N is the number of samples and D is the number of features113    :param y: vector of target values, numpy array with dimensions (N, 1)114    :param initial_w: vector of initial weights, numpy array with dimensions (D, 1)115    :param max_iters: how many iterations to run the algorithm, integer116    :param gamma: learning rate, positive float value117    :param threshold: convergence threshold, positive float value118    :returns: (final weights, final loss value), tuple119    120    :raises: LinAlgError if the Hessian matrix becomes singular121    """122    # Map the {-1, 1} classes to {0, 1}123    y = map_target_classes_to_boolean(y)124    # Set the initial values for the weights125    w = initial_w126    # Compute the initial loss value127    prev_loss = compute_loss_nll(y, x, initial_w)128    for n_iter in range(max_iters):129        # Compute the gradient and Hessian of the loss function130        grd = compute_gradient_nll(y, x, w)131        hess = compute_hessian_nll(y, x, w)132        # Update the weights using the gradient, Hessian and learning rate133        w -= gamma * np.linalg.solve(hess, grd)134        # Compute the current loss and test convergence135        loss = compute_loss_nll(y, x, w)136        if abs(loss - prev_loss) < threshold:137            print(f'converged at iter : {n_iter}')138            break139        prev_loss = loss.copy()140    # Compute the final loss value141    loss = compute_loss_nll(y, x, w)142    return w, loss143def reg_logistic_regression(y, x, lambda_, initial_w, max_iters, gamma, threshold=1e-2):144    """145    Implementation of the Newton optimization algorithm for logistic regression with L2 regularization146    :param x: data matrix, numpy ndarray with shape with shape (N, D),147              where N is the number of samples and D is the number of features148    :param y: vector of target values, numpy array with dimensions (N, 1)149    :param lambda_: regularization coefficient, positive float value150    :param initial_w: vector of initial weights, numpy array with dimensions (D, 1)151    :param max_iters: how many iterations to run the algorithm, integer152    :param gamma: learning rate, positive float value153    :param threshold: convergence threshold, positive float value154    :returns: (final weights, final loss value), tuple155    """156    # Map the {-1, 1} classes to {0, 1}157    y = map_target_classes_to_boolean(y)158    # Set the initial values for the weights159    w = initial_w160    # Compute the initial loss value161    prev_loss = compute_loss_nll(y, x, initial_w, lambda_)162    for n_iter in range(max_iters):163        # Compute the gradient and Hessian of the loss function164        grd = compute_gradient_nll(y, x, w, lambda_)165        hess = compute_hessian_nll(y, x, w, lambda_)166        # Update the weights using the gradient, Hessian and learning rate167        w -= gamma / np.sqrt(n_iter+1) * np.linalg.solve(hess, grd)168        # Compute the current loss and test convergence169        loss = compute_loss_nll(y, x, w, lambda_)170        if loss == np.inf:171            raise ArithmeticError('Training diverges')172        if abs(loss - prev_loss) < threshold:173            print(f'converged at iter : {n_iter}')174            break175        prev_loss = loss.copy()176    # Compute the final loss value177    loss = compute_loss_nll(y, x, w, lambda_)178    return w, loss179def svm(y, x, lambda_, initial_w, max_iters, gamma, threshold=1e-5):180    """181    Implementation of the linear SVM classification algorithm with L2 regularization182    The SVM is simulated through optimization of the Hinge loss function with gradient descent183    :param x: data matrix, numpy ndarray with shape with shape (N, D),184              where N is the number of samples and D is the number of features185    :param y: vector of target values, numpy array with length N186    :param lambda_: regularization coefficient, positive float value187    :param initial_w: vector of initial weights, numpy array with length D188    :param max_iters: how many iterations to run the algorithm, integer189    :param gamma: learning rate, positive float value190    :param threshold: convergence threshold, positive float value191    :returns: (final weights, final loss value), tuple192    """193    # Set the initial values for the weights194    w = initial_w195    196    # Compute the initial loss value197    prev_loss = compute_loss_hinge(y, x, w, lambda_)198    199    for n_iter in range(max_iters):200        # Compute the gradient of the loss function201        grd = compute_gradient_hinge(y, x, w, lambda_)202        # Update the weights using the gradient, Hessian and learning rate203        w -= gamma / (1 + 1e-2*n_iter) * grd204        # Compute the current loss and test convergence205        loss = compute_loss_hinge(y, x, w, lambda_)206        if abs(loss - prev_loss) < threshold:207            print(f'converged at iter : {n_iter}')208            break209        prev_loss = loss210    # Compute the final loss value211    loss = compute_loss_hinge(y, x, w, lambda_)...

gcp_compute_operator.py

Source:gcp_compute_operator.py

1#2# Licensed to the Apache Software Foundation (ASF) under one3# or more contributor license agreements.  See the NOTICE file4# distributed with this work for additional information5# regarding copyright ownership.  The ASF licenses this file6# to you under the Apache License, Version 2.0 (the7# "License"); you may not use this file except in compliance8# with the License.  You may obtain a copy of the License at9#10#   http://www.apache.org/licenses/LICENSE-2.011#12# Unless required by applicable law or agreed to in writing,13# software distributed under the License is distributed on an14# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY15# KIND, either express or implied.  See the License for the16# specific language governing permissions and limitations17# under the License.18"""This module is deprecated. Please use `airflow.providers.google.cloud.operators.compute`."""19import warnings20from airflow.providers.google.cloud.operators.compute import (21    ComputeEngineBaseOperator,22    ComputeEngineCopyInstanceTemplateOperator,23    ComputeEngineInstanceGroupUpdateManagerTemplateOperator,24    ComputeEngineSetMachineTypeOperator,25    ComputeEngineStartInstanceOperator,26    ComputeEngineStopInstanceOperator,27)28warnings.warn(29    "This module is deprecated. Please use `airflow.providers.google.cloud.operators.compute`.",30    DeprecationWarning,31    stacklevel=2,32)33class GceBaseOperator(ComputeEngineBaseOperator):34    """35    This class is deprecated.36    Please use `airflow.providers.google.cloud.operators.compute.ComputeEngineBaseOperator`.37    """38    def __init__(self, *args, **kwargs):39        warnings.warn(40            """This class is deprecated.41            Please use `airflow.providers.google.cloud.operators.compute.ComputeEngineBaseOperator`.""",42            DeprecationWarning,43            stacklevel=3,44        )45        super().__init__(*args, **kwargs)46class GceInstanceGroupManagerUpdateTemplateOperator(ComputeEngineInstanceGroupUpdateManagerTemplateOperator):47    """48    This class is deprecated.49    Please use `airflow.providers.google.cloud.operators.compute50    .ComputeEngineInstanceGroupUpdateManagerTemplateOperator`.51    """52    def __init__(self, *args, **kwargs):53        warnings.warn(54            """This class is deprecated. Please use55            `airflow.providers.google.cloud.operators.compute56            .ComputeEngineInstanceGroupUpdateManagerTemplateOperator`.""",57            DeprecationWarning,58            stacklevel=3,59        )60        super().__init__(*args, **kwargs)61class GceInstanceStartOperator(ComputeEngineStartInstanceOperator):62    """63    This class is deprecated.64    Please use `airflow.providers.google.cloud.operators65    .compute.ComputeEngineStartInstanceOperator`.66    """67    def __init__(self, *args, **kwargs):68        warnings.warn(69            """This class is deprecated.70            Please use `airflow.providers.google.cloud.operators.compute71            .ComputeEngineStartInstanceOperator`.""",72            DeprecationWarning,73            stacklevel=3,74        )75        super().__init__(*args, **kwargs)76class GceInstanceStopOperator(ComputeEngineStopInstanceOperator):77    """78    This class is deprecated.79    Please use `airflow.providers.google.cloud.operators.compute.ComputeEngineStopInstanceOperator`.80    """81    def __init__(self, *args, **kwargs):82        warnings.warn(83            """This class is deprecated.84            Please use `airflow.providers.google.cloud.operators.compute85            .ComputeEngineStopInstanceOperator`.""",86            DeprecationWarning,87            stacklevel=3,88        )89        super().__init__(*args, **kwargs)90class GceInstanceTemplateCopyOperator(ComputeEngineCopyInstanceTemplateOperator):91    """92    This class is deprecated.93    Please use `airflow.providers.google.cloud.operators.compute.ComputeEngineCopyInstanceTemplateOperator`.94    """95    def __init__(self, *args, **kwargs):96        warnings.warn(97            """"This class is deprecated.98            Please use `airflow.providers.google.cloud.operators.compute99            .ComputeEngineCopyInstanceTemplateOperator`.""",100            DeprecationWarning,101            stacklevel=3,102        )103        super().__init__(*args, **kwargs)104class GceSetMachineTypeOperator(ComputeEngineSetMachineTypeOperator):105    """106    This class is deprecated.107    Please use `airflow.providers.google.cloud.operators.compute.ComputeEngineSetMachineTypeOperator`.108    """109    def __init__(self, *args, **kwargs):110        warnings.warn(111            """This class is deprecated.112            Please use `airflow.providers.google.cloud.operators.compute113            .ComputeEngineSetMachineTypeOperator`.""",114            DeprecationWarning,115            stacklevel=3,116        )...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.