How to use redirected method in Splinter

Best Python code snippet using splinter

test_redirects.py

Source:test_redirects.py

1# -*- coding: utf-8 -*-2from __future__ import absolute_import3import unittest4import six5from splash.tests.test_render import BaseRenderTest6from splash.tests.utils import NON_EXISTING_RESOLVABLE7class HttpRedirectTest(BaseRenderTest):8    def assertRedirectedResponse(self, resp, code):9        self.assertStatusCode(resp, 200)10        self.assertIn("GET request", resp.text)11        if six.PY3:12            self.assertIn("{b'http_code': [b'%s']}" % code, resp.text)13        else:14            self.assertIn("{'http_code': ['%s']}" % code, resp.text)15    def assertHttpRedirectWorks(self, code):16        resp = self.request({"url": self.mockurl("http-redirect?code=%s" % code)})17        self.assertRedirectedResponse(resp, code)18    def assertBaseurlHttpRedirectWorks(self, code):19        url = self.mockurl("http-redirect?code=%s" % code)20        resp = self.request({"url": url, "baseurl": url})21        self.assertRedirectedResponse(resp, code)22    def test_301(self):23        self.assertHttpRedirectWorks(301)24    def test_302(self):25        self.assertHttpRedirectWorks(302)26    def test_303(self):27        self.assertHttpRedirectWorks(303)28    def test_307(self):29        self.assertHttpRedirectWorks(307)30    def test_301_baseurl(self):31        self.assertBaseurlHttpRedirectWorks(301)32    def test_302_baseurl(self):33        self.assertBaseurlHttpRedirectWorks(302)34    def test_303_baseurl(self):35        self.assertBaseurlHttpRedirectWorks(303)36    def test_307_baseurl(self):37        self.assertBaseurlHttpRedirectWorks(307)38class MetaRedirectTest(BaseRenderTest):39    def assertRedirected(self, resp):40        self.assertStatusCode(resp, 200)41        self.assertIn("META REDIRECT TARGET", resp.text)42    def assertNotRedirected(self, resp):43        self.assertStatusCode(resp, 200)44        self.assertIn('<meta http-equiv="REFRESH"', resp.text)45    def test_meta_redirect_nowait(self):46        r = self.request({'url': self.mockurl('meta-redirect0')})47        self.assertNotRedirected(r)48    def test_meta_redirect_wait(self):49        r = self.request({'url': self.mockurl('meta-redirect0'), 'wait': '0.1'})50        self.assertRedirected(r)51    def test_meta_redirect_delay_wait(self):52        r = self.request({'url': self.mockurl('meta-redirect1'), 'wait': '0.1'})53        self.assertNotRedirected(r)54    def test_meta_redirect_delay_wait_enough(self):55        r = self.request({'url': self.mockurl('meta-redirect1'), 'wait': '0.3'})56        self.assertRedirected(r)57    def test_meta_redirect_slowload(self):58        r = self.request({'url': self.mockurl('meta-redirect-slowload')})59        self.assertNotRedirected(r)60    def test_meta_redirect_slowload_wait(self):61        r = self.request({62            'url': self.mockurl('meta-redirect-slowload'),63            'wait': '0.1',64        })65        self.assertRedirected(r)66    def test_meta_redirect_slowload_wait_more(self):67        r = self.request({68            'url': self.mockurl('meta-redirect-slowload'),69            'wait': '0.3',70        })71        self.assertRedirected(r)72    def test_meta_redirect_slowload2(self):73        r = self.request({'url': self.mockurl('meta-redirect-slowload2')})74        self.assertNotRedirected(r)75    def test_meta_redirect_slowload2_wait(self):76        r = self.request({77            'url': self.mockurl('meta-redirect-slowload2'),78            'wait': '0.1',79        })80        self.assertRedirected(r)81    def test_meta_redirect_slowload2_wait_more(self):82        r = self.request({83            'url': self.mockurl('meta-redirect-slowload2'),84            'wait': '0.3',85        })86        self.assertRedirected(r)87class JsRedirectTest(BaseRenderTest):88    def assertRedirected(self, resp):89        self.assertStatusCode(resp, 200)90        self.assertIn("JS REDIRECT TARGET", resp.text)91    def assertNotRedirected(self, resp):92        self.assertStatusCode(resp, 200)93        self.assertNotIn("JS REDIRECT TARGET", resp.text)94        self.assertIn("Redirecting", resp.text)95    def test_redirect_nowait(self):96        r = self.request({'url': self.mockurl('jsredirect')})97        self.assertNotRedirected(r)98    def test_redirect_wait(self):99        r = self.request({'url': self.mockurl('jsredirect'), 'wait': '0.1'})100        self.assertRedirected(r)101    def test_redirect_onload_nowait(self):102        r = self.request({'url': self.mockurl('jsredirect-onload')})103        self.assertNotRedirected(r)104    def test_redirect_onload_wait(self):105        r = self.request({'url': self.mockurl('jsredirect-onload'), 'wait': '0.1'})106        self.assertRedirected(r)107    def test_redirect_timer_nowait(self):108        r = self.request({'url': self.mockurl('jsredirect-timer')})109        self.assertNotRedirected(r)110    def test_redirect_timer_wait(self):111        # jsredirect-timer redirects after 0.1ms112        r = self.request({'url': self.mockurl('jsredirect-timer'), 'wait': '0.05'})113        self.assertNotRedirected(r)114    def test_redirect_timer_wait_enough(self):115        # jsredirect-timer redirects after 0.1s116        r = self.request({'url': self.mockurl('jsredirect-timer'), 'wait': '0.2'})117        self.assertRedirected(r)118    def test_redirect_chain_nowait(self):119        r = self.request({'url': self.mockurl('jsredirect-chain')})120        self.assertNotRedirected(r)121    def test_redirect_chain_wait(self):122        r = self.request({'url': self.mockurl('jsredirect-chain'), 'wait': '0.2'})123        self.assertRedirected(r)124    def test_redirect_slowimage_nowait(self):125        r = self.request({'url': self.mockurl('jsredirect-slowimage')})126        self.assertRedirected(r)127    def test_redirect_slowimage_wait(self):128        r = self.request({'url': self.mockurl('jsredirect-slowimage'), 'wait': '0.1'})129        self.assertRedirected(r)130    def test_redirect_slowimage_nowait_baseurl(self):131        r = self.request({132            'url': self.mockurl('jsredirect-slowimage'),133            'baseurl': self.mockurl('/'),134        })135        self.assertRedirected(r)136    def test_redirect_slowimage_wait_baseurl(self):137        r = self.request({138            'url': self.mockurl('jsredirect-slowimage'),139            'baseurl': self.mockurl('/'),140            'wait': '0.1'141        })142        self.assertRedirected(r)143    @unittest.skipIf(NON_EXISTING_RESOLVABLE, "non existing hosts are resolvable")144    def test_redirect_to_non_existing(self):145        r = self.request({146            "url": self.mockurl("jsredirect-non-existing"),147            "wait": '2.',148        })149        self.assertStatusCode(r, 502)...

redirect_relu.py

Source:redirect_relu.py

1import torch2import torchbearer3from torch import nn4class TemporaryRelu(nn.Module):5    """ Module to wrap ReLU and RedirectedReLU and call the correct on for the current stage of training.6        For the first epoch only the first 16 batches use redirection and all others use standard relu.7        Args:8            old_module: Standard ReLU module9            redirected_module: Redirected Module10            parent: Parent which tracks the stage in progress and sets parent.redirected11    """12    def __init__(self, old_module, redirected_module, parent):13        super().__init__()14        self.redirected_module = redirected_module15        self.old_module = old_module16        self.parent = [parent]17        self.module = redirected_module18    def __repr__(self):19        return self.module.__repr__()20    def forward(self, x):21        if self.parent[0].redirected:22            self.module = self.redirected_module23        else:24            self.module = self.old_module25        return self.module(x)26class RedirectReLUs(nn.Module):27    """Module that replaces all ReLU or ReLU6 modules in the model with28    `redirected ReLU <https://github.com/tensorflow/lucid/blob/master/lucid/misc/redirected_relu_grad.py>`__29    versions for the first 16 iterations.30    .. Note::31        - This doesn't apply to nn.functional ReLUs32        - This must be applied before an IntermediateLayerGetter module for both to correctly function33    Example::34        >>> import torchbearer35        >>> from torchbearer import Trial36        >>> from visual import RedirectReLUs37        >>> model = RedirectReLUs(torch.nn.Sequential(torch.nn.ReLU()))38        >>> @torchbearer.callbacks.on_sample39        ... def input_data(state):40        ...     state[torchbearer.X] = torch.rand(1, 1)41        >>> trial = Trial(model, callbacks=[input_data]).for_steps(1).run()42        >>> print(model)43        RedirectReLUs(44          (model): Sequential(45            (0): RedirectedReLU()46          )47        )48        >>> model = RedirectReLUs(torch.nn.Sequential(torch.nn.ReLU()))49        >>> trial = Trial(model, callbacks=[input_data]).for_steps(17).run()50        >>> print(model)51        RedirectReLUs(52          (model): Sequential(53            (0): ReLU()54          )55        )56    """57    def __init__(self, model):58        super(RedirectReLUs, self).__init__()59        self.relu_types = [torch.nn.ReLU]60        self.old_modules = {}61        self.model = self.replace_relu(model)62        self.batch = None63        self.redirected = True64    def replace_relu(self, model):65        new_modules = {}66        for i, mod in enumerate(model.named_children()):67            n, m = mod68            if type(m) == torch.nn.ReLU:69                self.old_modules[n] = m70                new_modules[n] = TemporaryRelu(m, RedirectedReLU(), self)71            elif type(m) == torch.nn.ReLU6:72                self.old_modules[n] = m73                new_modules[n] = TemporaryRelu(m, RedirectedReLU6(), self)74        for m in new_modules:75            model._modules[m] = new_modules[m]76        return model77    def forward(self, x, state=None):78        if state is not None:79            self.redirected = not (state[torchbearer.EPOCH] != 0 or state[torchbearer.BATCH] >= 16)80        return self.model(x)81class RedirectedReLU(torch.nn.Module):82    """ Module to wrap the redirected ReLU function.83    See `here <https://github.com/tensorflow/lucid/blob/master/lucid/misc/redirected_relu_grad.py>`__84    """85    def forward(self, x):86        return RedirectedReLUFunction.apply(x)87class RedirectedReLU6(torch.nn.Module):88    """ Module to wrap the redirected ReLU6 function.89    See `here <https://github.com/tensorflow/lucid/blob/master/lucid/misc/redirected_relu_grad.py>`__90    """91    def forward(self, x):92        return RedirectedReLU6Function.apply(x)93class RedirectedReLUFunction(torch.autograd.Function):94    """Reimplementation of the redirected ReLU from95    `tensorflows lucid library <https://github.com/tensorflow/lucid/blob/master/lucid/misc/redirected_relu_grad.py>`__.96    """97    @staticmethod98    def forward(ctx, input):99        ctx.save_for_backward(input)100        return input.clamp(min=0)101    @staticmethod102    def backward(ctx, grad_output):103        input, = ctx.saved_tensors104        grad_input = grad_output.clone()105        relu_grad = grad_input.clone()106        relu_grad[input < 0] = 0107        neg_pushing_lower = torch.lt(input, 0) & torch.gt(grad_input, 0)108        redirected_grad = grad_input109        redirected_grad[neg_pushing_lower] = 0110        batch = grad_input.shape[0]111        reshaped_relu_grad = relu_grad.view(batch, -1)112        relu_grad_mag = torch.norm(reshaped_relu_grad, p=2, dim=1)113        result_grad = relu_grad114        result_grad[relu_grad_mag == 0, :] = redirected_grad[relu_grad_mag == 0, :]115        return result_grad116class RedirectedReLU6Function(torch.autograd.Function):117    """Reimplementation of the redirected ReLU6 from118    `tensorflows lucid library <https://github.com/tensorflow/lucid/blob/master/lucid/misc/redirected_relu_grad.py>`__.119    """120    @staticmethod121    def forward(ctx, input):122        ctx.save_for_backward(input)123        return input.clamp(min=0, max=6)124    @staticmethod125    def backward(ctx, grad_output):126        input, = ctx.saved_tensors127        grad_input = grad_output.clone()128        relu_grad = grad_input.clone()129        relu_grad[input < 0] = 0130        relu_grad[input > 6] = 0131        neg_pushing_lower = torch.lt(input, 0) & torch.gt(grad_input, 0)132        pos_pushing_higher = torch.gt(input, 6) & torch.lt(grad_input, 0)133        redirected_grad = grad_input134        redirected_grad[neg_pushing_lower] = 0135        redirected_grad[pos_pushing_higher] = 0136        batch = grad_input.shape[0]137        reshaped_relu_grad = relu_grad.view(batch, -1)138        relu_grad_mag = torch.norm(reshaped_relu_grad, p=2, dim=1)139        result_grad = relu_grad140        result_grad[relu_grad_mag == 0, :] = redirected_grad[relu_grad_mag == 0, :]...

redirect_with_captcha.py

Source:redirect_with_captcha.py

1import logging2import random3import re4import time5from six.moves.urllib.parse import urljoin6from scrapy.http import FormRequest,Request7from scrapy.downloadermiddlewares.redirect import RedirectMiddleware8from scrapy.utils.python import to_native_str9from scrapy.exceptions import IgnoreRequest, NotConfigured10from scrapy.http import Request1112logger = logging.getLogger(__name__)1314class house_Redirect(RedirectMiddleware):15    """Handle redirection of requests based on response status and meta-refresh html tag"""1617    def _new_request_from_response(self,request):18        """ Make new request from old request"""19        new_request = request.copy() 20        new_request.dont_filter = True21        return new_request2223    def process_response(self, request, response, spider):24        #To determine whether the status code in the definition of the yuan ancestral25        #and in conjunction with the corresponding URL to record to error.txt2627        if (request.meta.get('dont_redirect', False) or28                response.status in getattr(spider, 'handle_httpstatus_list', []) or29                response.status in request.meta.get('handle_httpstatus_list', []) or30                request.meta.get('handle_httpstatus_all', False)):31            return response3233        allowed_status = (301, 302, 303, 307)34        if 'Location' not in response.headers or response.status not in allowed_status:35            return response3637        if response.status == 302 or 301:38            location_url = response.headers['location'] 39            logging.debug("Redirecting (%(status)s) to %(redirected)s from %(request)s",40                     {'status': response.status, 'redirected': location_url, 'request': request.url},41                     extra={'spider': spider})42            error = 'status:(' + str(response.status) + ') ' + str(request.url) + ' ' + str(location_url) + ' Using--:' + str(request.meta['proxy']) + '\n'43            file_text = open('302_error.txt', 'a+')44            file_text.write(error)45            file_text.close()4647        # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-848        location = to_native_str(response.headers['location'].decode('latin1'))#latin1 è¥¿æ¬§çä¸ç§ç¼ç æ ¼å¼åºäºascill4950        redirected_url = urljoin(request.url, location)# æé ä¸ä¸ªæ°çURL51        if response.status in (301, 307) or request.method == 'HEAD':# è¿ä¸ºä»ä¹æ²¡æç¶æç 30252            redirected = request.replace(url=redirected_url)53            return self._redirect(redirected, request, spider, response.status)5455        # redirected url has two kinds of redirected url if it related to captcha input56        # the first type of url has captcha string in it57        # the second type of url has not, and is the objective url which we care more about58        59        redirected = self._redirect_request_using_get(request, redirected_url)60        return self._redirect(redirected, request, spider, response.status)616263        # if re.search('captcha',redirected_url):64        #     redirected = self._redirect_request_using_post(request,redirected_url)65        #     # å«æcaptchaçéå®åURL66        # else:67        #     redirected = self._redirect_request_using_get(request,redirected_url)6869        # return self._redirect(redirected, request, spider, response.status)707172    def _redirect_request_using_get(self, request, redirect_url):73        74        redirected = Request(url=redirect_url,75                             method='GET',76                             meta=request.meta,77                             callback=request.callback,78                             headers=request.headers,79                             body='')80        request.dont_filter=True81        redirected.meta.pop('proxy_need_change',None)82        redirected.headers.pop('Content-Type', None)83        redirected.headers.pop('Content-Length', None)84        return redirected8586    # def _redirect(self, redirected, request, spider, reason):87    #     ttl = request.meta.setdefault('redirect_ttl', self.max_redirect_times)#æå¤§çéå®åæ¬¡æ°88    #     redirects = request.meta.get('redirect_times', 0) + 18990    #     if ttl and redirects <= self.max_redirect_times:91    #         redirected.meta['redirect_times'] = redirects92    #         redirected.meta['redirect_ttl'] = ttl - 193    #         redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \94    #             [request.url]95    #         # if it has only two layers of redirects, we get rid of the third or more96    #         if redirected.meta['redirect_urls'].__len__()>2:97    #             redirected.meta['redirect_urls'].clear()98    #         redirected.dont_filter = request.dont_filter99    #         redirected.priority = request.priority + self.priority_adjust100            101    #         logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",102    #                      {'reason': reason, 'redirected': redirected, 'request': request},103    #                      extra={'spider': spider})104    #         return redirected105    #     else:106    #         logger.debug("Discarding %(request)s: max redirections reached",107    #                      {'request': request}, extra={'spider': spider})
...

redirect_middleware.py

Source:redirect_middleware.py

1# -*- coding: UTF-8 -*-2'''3Created on 2015-11-304@author: Lixiaowei5'''6from urlparse import urljoin7from scrapy import log8from scrapy.http import HtmlResponse9from scrapy.utils.response import get_meta_refresh10from scrapy.exceptions import IgnoreRequest, NotConfigured11class BaseRedirectMiddleware(object):12    enabled_setting = 'REDIRECT_ENABLED'13    def __init__(self, settings):14        if not settings.getbool(self.enabled_setting):15            raise NotConfigured16        self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES')17        self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')18    @classmethod19    def from_crawler(cls, crawler):20        return cls(crawler.settings)21    def _redirect(self, redirected, request, spider, reason):22        ttl = request.meta.setdefault('redirect_ttl', self.max_redirect_times)23        redirects = request.meta.get('redirect_times', 0) + 124        if ttl and redirects <= self.max_redirect_times:25            redirected.meta['redirect_times'] = redirects26            redirected.meta['redirect_ttl'] = ttl - 127            redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \28                [request.url]29            redirected.dont_filter = request.dont_filter30            redirected.priority = request.priority + self.priority_adjust31            log.msg(format="Redirecting (%(reason)s) to %(redirected)s from %(request)s",32                    level=log.DEBUG, spider=spider, request=request,33                    redirected=redirected, reason=reason)34            return redirected35        else:36            log.msg(format="Discarding %(request)s: max redirections reached",37                    level=log.DEBUG, spider=spider, request=request)38            raise IgnoreRequest("max redirections reached")39    def _redirect_request_using_get(self, request, redirect_url):40        redirected = request.replace(url=redirect_url, method='GET', body='')41        redirected.headers.pop('Content-Type', None)42        redirected.headers.pop('Content-Length', None)43        return redirected44class MyRedirectMiddleware(BaseRedirectMiddleware):45    """Handle redirection of requests based on response status and meta-refresh html tag"""46    def process_response(self, request, response, spider):47        if 'dont_redirect' in request.meta:48            return response49        if request.method == 'HEAD':50            if response.status in [301, 302, 303, 307] and 'Location' in response.headers:51                redirected_url = urljoin(request.url, response.headers['location'])52                redirected = request.replace(url=redirected_url)53                return self._redirect(redirected, request, spider, response.status)54            else:55                return response56            57        if response.status in [302, 303] and 'Location' in response.headers:58            redirected_url = urljoin(request.url, response.headers['location'])59            redirected = self._redirect_request_using_get(request, redirected_url)60            return self._redirect(redirected, request, spider, response.status)61        if response.status in [301, 307] and 'Location' in response.headers:62            redirected_url = urljoin(request.url, response.headers['location'])63            redirected = request.replace(url=redirected_url)64            return self._redirect(redirected, request, spider, response.status)65        return response66class MetaRefreshMiddleware(BaseRedirectMiddleware):67    enabled_setting = 'METAREFRESH_ENABLED'68    def __init__(self, settings):69        super(MetaRefreshMiddleware, self).__init__(settings)70        self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY',71                                         settings.getint('METAREFRESH_MAXDELAY'))72    def process_response(self, request, response, spider):73        if 'dont_redirect' in request.meta or request.method == 'HEAD' or \74                not isinstance(response, HtmlResponse):75            return response76        if isinstance(response, HtmlResponse):77            interval, url = get_meta_refresh(response)78            if url and interval < self._maxdelay:79                redirected = self._redirect_request_using_get(request, url)80                return self._redirect(redirected, request, spider, 'meta refresh')81        return response...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.