How to use visit_page method in lettuce_webdriver

Best Python code snippet using lettuce_webdriver_python

scraper.py

Source:scraper.py

...46                    "page": str(page_num),47                    "url": self.ROOT_DOMAIN + pack_query_string48                }49                self.page_links.append(page_dict)50    def visit_page(self, product_list_url, page_num):51        print "Page-{}, {}".format(page_num, product_list_url)52        response = urllib2.urlopen(product_list_url)53        if response.code is self.HTTP_STATUS_OK:54            page = Page()55            page.set_data(response.read())56            page.set_url(product_list_url)57            page.set_out_file(self.output_filename + "-page-" + str(page_num) + "-")58            page.scrap()59            print "{} - Completed".format(product_list_url)60            # list_page.set_response(response)61            # products = self.page.scrap()62    def scrap_all_products(self):63        # total_links = len(self.page_links)64        # print "Total Links : " + str(total_links)65        #66        # chunks = total_links / self.THREAD_NUM67        # remaining = total_links % self.THREAD_NUM68        #69        # print "Loop Required : " + str(chunks)70        # print "Starting Thread Count :" + str(self.THREAD_NUM)71        # print "Remaining Count : " + str(remaining)72        #73        # for i in range(0, chunks):74        #     for idx in range(self.THREAD_NUM):75        #         # print idx76        #         link = self.page_links[idx]77        #         page_num = link["page"]78        #         page_url = link["url"]79        #         #80        #         print "Starting Thread for url: " + page_url81        #         # t = threading.Thread(target=self.visit_page, args=(page_url, page_num,))82        #         # self.threads.append(t)83        #         # t.start()84        #85        # start_index = self.THREAD_NUM * chunks86        # #87        # remaining_chunk = total_links % self.THREAD_NUM88        # #89        # for j in range(remaining_chunk):90        #     print "Starting Thread for remaining urls"91        #     link = self.page_links[start_index]92        #     page_num = link["page"]93        #     page_url = link["url"]94        #95        #     print page_url96        #97        #     # t = threading.Thread(target=self.visit_page, args=(page_url, page_num,))98        #     # self.threads.append(t)99        #     #100        #     # print "Starting Thread for url: " + page_url101        #     # t.start()102        #103        #     start_index += 1104        count = 0;105        for link in self.page_links:106            # print link["url"]107            page_num = link["page"]108            page_url = link["url"]109            if (count % 4) is 0:110                time.sleep(5)111            t = threading.Thread(target=self.visit_page, args=(page_url, page_num,))112            self.threads.append(t)113            print "Starting Thread for url: " + page_url114            count += 1115            t.start()116            # self.visit_page(product_list_url=page_url, page_num=page_num)117    def main(self):118        try:119            opener = urllib2.build_opener()120            # opener.add_headers = [121            #     ('User-agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.82 Safari/537.36'),122            #     ('Referer', 'http://www.amazon.com/?field-keywords=LED+Lights'),123            #     ('Host', 'www.amazon.com'),124            #     ('Content-Type', 'application/x-www-form-urlencoded'),125            #     ('X-Requested-With', 'XMLHttpRequest'),126            #     ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')127            # ]128            response = opener.open(self.get_url())129            # #130            # # print response.code...

linkchecker.py

Source:linkchecker.py

...17    @blockedBy(['970592','970593'])18    def visit_all_html(self):19        self.visited = {}20        with asserts.Check() as check:21            r = self.visit_page(check,'','',accepts='text/html',regex='<a href=\"([^\"]+)')22        self.log.info('DONE!! Visited %d pages' % len(self.visited))23    @test24    def visit_all_json(self):25        self.visited = {}26        with asserts.Check() as check:27            r = self.visit_page(check,'','',accepts='application/json',regex='href\"\:\"([^\"]+)')28        self.log.info('DONE!! Visited %d pages' % len(self.visited))29    @test30    def visit_all_xml(self):31        self.visited = {}32        with asserts.Check() as check:33            r = self.visit_page(check,'','',accepts='application/xml',regex='href=\"([^\"]+)')34        self.log.info('DONE!! Visited %d pages' % len(self.visited))35    36    def visit_page(self,check,referer,url,follow=True,accepts='text/html',regex=''):37        self.visited[url] = True38        r = self.get(url,accepts=accepts)39        self.log.info('Visiting %s'%url)40        check.equal(r.status_code,200,'Server returned %d on GET %s, refering from %s' %(r.status_code,url,referer))41        if follow:42            for m in re.finditer(regex,r.text,re.DOTALL | re.IGNORECASE):43                href = m.group(1)44                if href.find('reports') >= 0 and href.find('csv') <= 0:45                    # https://bugzilla.redhat.com/show_bug.cgi?id=97277446                    # reports are "just" in csv47                    continue48                if href.find('/') == 0:49                    href = self.server_url.rstrip('/')+href50                if href.find('http') < 0:51                    href = self.url(href)52                if href.find(self.server_url) >= 0 and not self.visited.has_key(href):...

page_factory.py

Source:page_factory.py

1from __future__ import absolute_import2from page_object.browser import Browser3def on(page, visit_page=False, url_params={}):4    """5    Creates PageObject with current instance of the selenium_browser.6    Args:7        page     (PageObject): PageObject class name.8        visit_page  (boolean): Navigate to page.9        url_params   (dictionary): url parameter object.10    Returns:11            New instance of page argument.12    """13    page_instance = page(Browser.selenium_browser())14    if hasattr(page_instance, 'define_elements'):15        page_instance.define_elements()16    if visit_page:17        page_instance.goto(url_params)18    if hasattr(page_instance, 'initialize_page'):19        page_instance.initialize_page()20    return page_instance21def visit(page, url_params={}):22    """23    Navigate to page url.24    Creates PageObject with current instance of the selenium_browser.25    Args:26        page     (PageObject): PageObject class name.27        url_params   (dictionary): url parameter object.28    Returns:29            New instance of page argument.30    """...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.