How to use stop_chunk method in Playwright Python

Best Python code snippet using playwright-python

sources.py

Source:sources.py Github

copy

Full Screen

1import os2import pickle3import re4import zipfile5from abc import ABC6from typing import List, Tuple7from Tools import ImageLoader8from container import Container9class Source(ABC):10 _chunk = -111 _chunks = []12 _dumping = True13 _chunk_size = 100014 def __init__(self, app: Container, chunk_size: int = 1000):15 self._chunk_size = chunk_size16 self.data = []17 self.length = 018 self.app = app19 self.loader = app.resolve(ImageLoader)20 def __iter__(self):21 self._current = 022 return self23 def __next__(self):24 index = self._current25 self._current += 126 if index >= self.length:27 raise StopIteration28 return self[index]29 def _slice_impl(self, item):30 full_list = []31 for chunk in self._chunks:32 full_list.extend(chunk)33 new_list = full_list[item.start:item.stop]34 source = Source(self.app)35 source._chunks = list(self.app.helper.list_chunk(new_list, self._chunk_size))36 source.length = len(new_list)37 return source38 def _item_impl(self, item):39 if self._chunk == -1 or self._chunk != int(item / self._chunk_size):40 self._chunk = int(item / self._chunk_size)41 self.data = self.loader(42 self._chunks[self._chunk]43 )44 image = self.data[item % self._chunk_size]45 return image.grayscale, image.thermal46 def __getitem__(self, item):47 if isinstance(item, slice):48 return self._slice_impl(item)49 return self._item_impl(item)50 def __len__(self):51 return self.length52 def dump(self, path: str):53 with open(path, 'wb') as bin_file:54 pickle.dump(self.data, bin_file)55 def load(self, path: str):56 with open(path, 'rb') as bin_file:57 self.data = pickle.load(bin_file)58 def chunks_dump(self):59 name = self.app.helper.time_name()60 os.mkdir(self.app.config('data.cache') + name)61 meta = []62 for i, datum in enumerate(self._chunks):63 self.data = self.loader(datum)64 self.dump(self.app.config('data.cache') + name + '/' + str(i) + '.pkl')65 meta.append((str(i) + '.pkl', len(self.data)))66 self.app.helper.dump_json(self.app.config('data.cache') + name + '/meta.json', meta)67class FileSource(Source):68 def __init__(self, app: Container, dir_name: str, dump: bool = True):69 super(FileSource, self).__init__(app)70 self.dir_name = dir_name71 self._dumping = dump72 self.fetch()73 def fetch(self):74 # Detect the zip files presented in given dir75 zip_files = []76 for file in os.listdir(self.dir_name):77 if re.match(r'(.*)\.zip', file):78 zip_files.append(79 zipfile.ZipFile(self.dir_name + '/' + file)80 )81 manifest = []82 # Load data from each zip file83 for zip_file in zip_files:84 manifest += self.load_list(zip_file)85 self.length = len(manifest)86 # 加载列表分块87 self._chunks = list(self.app.helper.list_chunk(manifest, self._chunk_size))88 if self._dumping:89 self.chunks_dump()90 @staticmethod91 def load_list(file: zipfile.ZipFile) -> list:92 nums = []93 # Determine the first file in zip file94 for name in file.namelist():95 # Extract every file from zip file96 file.extract(name, path='./storage/images')97 num = re.search(r'GS(.*)\.png', name)98 if num:99 nums.append(100 int(num.group(1))101 )102 start = min(nums)103 # Determine the name of extract dir104 dir_name = file.namelist()[0].split('/')[0]105 # Transform the result file into a load list106 with open('./storage/images/' + dir_name + '/result.txt', 'r') as results_file:107 results = results_file.readlines()108 manifest = []109 for i, result in enumerate(results, start):110 result = result.split()111 manifest.append((112 file.filename + ':{GS|RGB}' + str(i) + '.png',113 './storage/images/' + dir_name + '/GS' + str(i) + '.png',114 './storage/images/' + dir_name + '/RGB' + str(i) + '.png',115 float(result[0]),116 float(result[1]),117 ))118 return manifest119class DBSource(Source):120 _chunk = -1121 _chunks = []122 _dumping = True123 def __init__(124 self,125 app: Container,126 db_hostname: str,127 db_username: str,128 db_password: str,129 db_name: str,130 base_url: str,131 dump: bool = True132 ):133 """134 Database Source Implementation135 Args:136 db_hostname: The hostname of database server137 db_username: The username of database server138 db_password: The password of database user139 db_name: The name of database to use140 base_url: The base url prefix append before images path141 """142 super(DBSource, self).__init__(app)143 import pymysql144 import threading145 self._dumping = dump146 self.base_url = base_url147 self.download_tasks = []148 self.manifest = []149 self.downloading = False150 # Init the connection to database151 self.connection = pymysql.Connection(152 user=db_username,153 password=db_password,154 host=db_hostname,155 database=db_name156 )157 self.base_db_url = 'mysql://' + self.connection.get_host_info().split()[-1] + '/' + db_name + '/'158 # Init two task threads159 self.fetcher = threading.Thread(target=self.fetch)160 self.downloader = threading.Thread(target=self.download)161 self.fetcher.start()162 self.downloader.start()163 # Wait both threads exit164 while True:165 if self.fetcher.is_alive():166 continue167 elif self.download_tasks:168 continue169 else:170 self.downloading = False171 break172 self._chunks = list(self.app.helper.list_chunk(self.manifest, self._chunk_size))173 self.length = len(self.manifest)174 def fetch(self) -> None:175 # Two cursors to perform different queries176 cursor_1 = self.connection.cursor()177 cursor_2 = self.connection.cursor()178 course = 'SELECT * from `mtrl_info`' # Query all materials179 cursor_1.execute(course)180 materials = []181 while True:182 result = cursor_1.fetchone()183 if not result:184 break185 material_id = result[0]186 # Query all params belongs to current material187 course = 'SELECT * from `mtrl_para` where `mtrl_id` = %d' % material_id188 cursor_2.execute(course)189 params = cursor_2.fetchall()190 # Init a material param list and fill it191 material = [None, None, None, None, None]192 for param in params:193 material[194 param[2]195 ] = (param[3] if param[2] in [1, 2] else (self.base_url + param[7]))196 urls = material[3:]197 material[0] = self.base_db_url + 'mtrl_info/' + str(material_id)198 material[3] = 'images/' + material[3].split('/')[-1]199 material[4] = 'images/' + material[4].split('/')[-1]200 # Push download task into the download tasks list201 self.download_tasks.append({'url': urls[0], 'path': 'images/'})202 self.download_tasks.append({'url': urls[1], 'path': 'images/'})203 # Push the material params list into the materials list204 materials.append(material)205 self._chunks = list(self.app.helper.list_chunk(materials, self._chunk_size))206 self.length = len(materials)207 if self._dumping:208 self.chunks_dump()209 def download(self):210 import requests211 # Downloading signal212 self.downloading = True213 while self.downloading:214 if self.download_tasks:215 task = self.download_tasks.pop()216 url = task['url']217 path = task['path']218 response = requests.get(url)219 filename = url.split('/')[-1]220 with open(path + filename, 'wb') as file:221 file.write(response.content)222 else:223 continue224class SavedSource(Source):225 def __init__(self, app: Container, dir_name: str):226 """227 Load data from a previously bin source.228 Args:229 dir_name: The path to saves file230 """231 super(SavedSource, self).__init__(app)232 self.path = dir_name233 self.fetch()234 self.loader = self.load_chunk235 def refresh(self):236 length = 0237 for chunk in self._chunks:238 length += chunk[1] - chunk[2]239 self.length = length240 self._chunk = -1241 self.data = []242 return self243 def fetch(self):244 meta = self.app.helper.load_json(self.path + '/meta.json')245 self._chunks = []246 for chunk_desc in meta:247 self._chunks.append(tuple(chunk_desc) + (0,))248 self.length += chunk_desc[1]249 def load_chunk(self, chunk_desc):250 return self.load(self.path + '/' + chunk_desc[0])251 def locate_chunk(self, item):252 for _i, _chunk in enumerate(self._chunks):253 item -= _chunk[1] - _chunk[2]254 if item < 0:255 return _i, item + _chunk[1]256 def _slice_impl(self, item):257 new_source = SavedSource(self.app, self.path)258 if item.start is None:259 start_chunk, start_offset = 0, self._chunks[0][2]260 else:261 start_chunk, start_offset = self.locate_chunk(item.start)262 if item.stop is None:263 stop_chunk, stop_offset = len(self._chunks) - 1, self._chunks[-1][1]264 else:265 stop_chunk, stop_offset = self.locate_chunk(item.stop)266 if stop_offset == 0:267 stop_chunk -= 1268 stop_offset = self._chunks[stop_chunk][1]269 new_chunks = list(map(lambda x: list(x), self._chunks[start_chunk:stop_chunk + 1]))270 new_chunks[0][2] = start_offset271 new_chunks[-1][1] = stop_offset272 new_source._chunks = list(map(lambda x: tuple(x), new_chunks))273 new_source.refresh()274 return new_source275 def _item_impl(self, item):276 chunk, offset = self.locate_chunk(item)277 if self._chunk == -1 or self._chunk != chunk:278 self._chunk = chunk279 self.loader(self._chunks[self._chunk])280 image = self.data[offset]281 return image.grayscale, image.thermal282class TestSource(Source):283 resources = [284 ('tests/train-images-idx3-ubyte', 'tests/train-labels-idx1-ubyte'),285 ('tests/t10k-images-idx3-ubyte', 'tests/t10k-labels-idx1-ubyte')286 ]287 def __init__(self, app: Container):288 super(TestSource, self).__init__(app)289 self.data = self.fetch()290 pass291 def fetch(self) -> List[Tuple]:292 import struct293 from PIL import Image294 data = []295 for images, labels in self.resources:296 with open(images, 'br') as img_files, open(labels, 'br') as lab_files:297 # Read the magic number298 img_files.read(4)299 lab_files.read(8)300 # Read the file number301 num = img_files.read(4)302 num = struct.unpack('>i', num)[0]303 # Read the size of image304 height = struct.unpack('>i', img_files.read(4))[0]305 width = struct.unpack('>i', img_files.read(4))[0]306 size = width * height307 padding = b'\x00\x00\x00'308 # Read the images309 for i in range(num):310 img = img_files.read(size)311 lab = lab_files.read(1)312 # Convert format313 label = struct.unpack('>i', padding + lab)[0]314 image = Image.frombytes(315 mode='L',316 size=(28, 28),317 data=img318 )319 data.append((320 self.loader.loader(image).unsqueeze(0).cuda(),321 label322 ))...

Full Screen

Full Screen

artist.py.svn-base

Source:artist.py.svn-base Github

copy

Full Screen

1#!/usr/bin/env python2# encoding: utf-83"""4A Python interface to the The Echo Nest's web API. See5http://developer.echonest.com/ for details.6"""7from config import CACHE8import document9import util10class Artist(object):11 def __init__(self, identifier, name=None):12 if len(identifier)==18:13 identifier = 'music://id.echonest.com/~/AR/' + identifier14 self._identifier = identifier15 self._name = name16 self._audio = document.WebDocumentSet(identifier, 'get_audio')17 self._image = document.WebDocumentSet(identifier, 'get_images')18 self._biographies = document.WebDocumentSet(identifier, 'get_biographies')19 self._blogs = document.WebDocumentSet(identifier, 'get_blogs')20 self._news = document.WebDocumentSet(identifier, 'get_news')21 self._reviews = document.WebDocumentSet(identifier, 'get_reviews')22 self._similar = SimilarDocumentSet(identifier)23 self._video = document.WebDocumentSet(identifier, 'get_video')24 self._familiarity = None25 self._hotttnesss = None26 self._urls = None27 self._terms = None28 def audio(self, rows=15, start=0, refresh=False):29 if refresh or not CACHE:30 self._audio = document.WebDocumentSet(self._identifier, 'get_audio')31 return self._audio[start:start + rows]32 def images(self, rows=15, start=0, refresh=False):33 if refresh or not CACHE:34 self._images = document.WebDocumentSet(self._identifier, 'get_images')35 return self._image[start:start + rows]36 37 def biographies(self, rows=15, start=0, refresh=False):38 if refresh or not CACHE:39 self._biographies = document.WebDocumentSet(self._identifier, 'get_biographies')40 return self._biographies[start:start + rows]41 def blogs(self, rows=15, start=0, refresh=False):42 if refresh or not CACHE:43 self._blogs = document.WebDocumentSet(self._identifier, 'get_blogs')44 return self._blogs[start:start + rows]45 46 def news(self, rows=15, start=0, refresh=False):47 if refresh or not CACHE:48 self._news = document.WebDocumentSet(self._identifier, 'get_news')49 return self._news[start:start + rows]50 51 def reviews(self, rows=15, start=0, refresh=False):52 if refresh or not CACHE:53 self._reviews = document.WebDocumentSet(self._identifier, 'get_reviews')54 return self._reviews[start:start + rows]55 56 def similar(self, rows=15, start=0, refresh=False):57 if refresh or not CACHE:58 self._similar = SimilarDocumentSet(self._identifier)59 return self._similar[start:start + rows]60 61 def video(self, rows=15, start=0, refresh=False):62 if refresh or not CACHE:63 self._video = document.WebDocumentSet(self._identifier, 'get_video')64 return self._video[start:start + rows]65 def familiarity(self, refresh=False):66 """Returns our numerical estimation of how 67 familiar an artist currently is to the world."""68 if self._familiarity is None or not CACHE:69 try:70 params = {'id': self.identifier}71 response = util.call('get_familiarity', params).findtext('artist/familiarity')72 self._familiarity = float(response)73 except:74 self._familiarity = 075 return self._familiarity76 def hotttnesss(self, refresh=False):77 """Returns our numerical description of how 78 hottt an artist currently is."""79 if self._hotttnesss is None or not CACHE:80 try:81 params = {'id': self.identifier}82 response = util.call('get_hotttnesss', params).findtext('artist/hotttnesss')83 self._hotttnesss = float(response)84 except:85 self._hotttnesss = 086 return self._hotttnesss87 @property88 def name(self):89 if self._name is None or not CACHE:90 self._name = util.call('get_profile', {'id': self.identifier}).findtext('artist/name')91 return self._name92 93 @property94 def identifier(self):95 """A unique identifier for an artist.96 See http://developer.echonest.com/docs/datatypes/97 for more information"""98 return self._identifier99 100 @property101 def urls(self):102 """Get links to the artist's official site, MusicBrainz site, 103 MySpace site, Wikipedia article, Amazon list, and iTunes page."""104 if self._urls is None or not CACHE:105 response = util.call('get_urls', {'id': self.identifier}).find('artist').getchildren()106 self._urls = dict((url.tag[:-4], url.text) for url in response if url.tag[-4:] == '_url')107 return self._urls108 def terms(self):109 if self._terms is None or not CACHE:110 response = util.call('get_top_terms', {'id': self.identifier}).findall('terms/term')111 self._terms = [e.text for e in response]112 return self._terms113 def __repr__(self):114 return "<Artist '%s'>" % str(self)115 116 def __str__(self):117 return self.name if isinstance(self.name, str) else self.name.encode('utf-8')118 119 def clear_cache(self):120 pass121TRUTH = {True: 'Y', False: 'N'}122SEARCH_ARTISTS_CACHE = {}123def search_artists(name, exact=False, sounds_like=True, rows=15, refresh=False):124 """Search for an artist using a query on the artist name.125 This may perform a sounds-like search to correct common 126 spelling mistakes."""127 global SEARCH_ARTISTS_CACHE128 if CACHE and not refresh:129 try:130 return SEARCH_ARTISTS_CACHE[(name, exact, sounds_like, rows)]131 except KeyError:132 pass133 params = {'query': name, 'exact': TRUTH[exact], 134 'sounds_like': TRUTH[sounds_like], 'rows': rows}135 response = util.call('search_artists', params).findall('artists/artist')136 value = [Artist(a.findtext('id'), a.findtext('name')) for a in response]137 SEARCH_ARTISTS_CACHE[(name, exact, sounds_like, rows)] = value138 return value139TOP_HOTTT_ARTISTS_CACHE = []140def get_top_hottt_artists(rows=15, refresh=False):141 """Retrieves a list of the top hottt artists.142 Do not request this more than once an hour."""143 global TOP_HOTTT_ARTISTS_CACHE144 if TOP_HOTTT_ARTISTS_CACHE==[] or refresh or not CACHE:145 response = util.call('get_top_hottt_artists', {'rows': rows}).findall('artists/artist')146 TOP_HOTTT_ARTISTS_CACHE = [Artist(a.findtext('id'), a.findtext('name')) for a in response]147 return TOP_HOTTT_ARTISTS_CACHE148class SimilarDocumentSet(document.DocumentSet):149 def __init__(self, identifier):150 super(SimilarDocumentSet, self).__init__(identifier, 'get_similar', 'similar/artist')151 def __len__(self):152 if self._len is None:153 self._len = 100154 return self._len155 def __getitem__(self, k):156 if k.stop > 100:157 raise util.EchoNestAPIError(5, 'Invalid parameter: "rows" must be less than or equal to 100')158 chunk_and_index = lambda i: (i / self._cache.chunk_size, i % self._cache.chunk_size)159 start, start_chunk, start_index, stop = self.chunk_magic(k)160 if (stop + 1) > self._cache.chunk_size:161 self.__init__(self.identifier)162 self._cache.chunk_size = stop + 1163 chunk_and_index = lambda i: (i / self._cache.chunk_size, i % self._cache.chunk_size)164 start, start_chunk, start_index, stop = self.chunk_magic(k)165 stop_chunk, stop_index = chunk_and_index(stop)166 items = []167 for chunk in xrange(start_chunk, stop_chunk + 1):168 elements = self._cache[chunk].findall(self.element_path)169 if chunk == start_chunk and chunk == stop_chunk:170 elements = elements[start_index:stop_index + 1]171 elif chunk == start_chunk:172 elements = elements[start_index:]173 elif chunk == stop_chunk:174 elements = elements[:stop_index + 1]175 items.extend([self._parse_element(e) for e in elements])176 return items177 def chunk_magic(self, k):178 chunk_and_index = lambda i: (i / self._cache.chunk_size, i % self._cache.chunk_size)179 if not isinstance(k, slice):180 raise TypeError181 start = k.start or 0182 start_chunk, start_index = chunk_and_index(start)183 stop = min(k.stop or len(self), len(self)) - 1 # use inclusive to simplify logic184 return start, start_chunk, start_index, stop185 def _parse_element(self, element):...

Full Screen

Full Screen

artist.py

Source:artist.py Github

copy

Full Screen

1#!/usr/bin/env python2# encoding: utf-83"""4A Python interface to the The Echo Nest's web API. See5http://developer.echonest.com/ for details.6"""7from config import CACHE8import document9import util10class Artist(object):11 def __init__(self, identifier, name=None):12 if len(identifier)==18:13 identifier = 'music://id.echonest.com/~/AR/' + identifier14 self._identifier = identifier15 self._name = name16 self._audio = document.WebDocumentSet(identifier, 'get_audio')17 self._image = document.WebDocumentSet(identifier, 'get_images')18 self._biographies = document.WebDocumentSet(identifier, 'get_biographies')19 self._blogs = document.WebDocumentSet(identifier, 'get_blogs')20 self._news = document.WebDocumentSet(identifier, 'get_news')21 self._reviews = document.WebDocumentSet(identifier, 'get_reviews')22 self._similar = SimilarDocumentSet(identifier)23 self._video = document.WebDocumentSet(identifier, 'get_video')24 self._familiarity = None25 self._hotttnesss = None26 self._urls = None27 self._terms = None28 def audio(self, rows=15, start=0, refresh=False):29 if refresh or not CACHE:30 self._audio = document.WebDocumentSet(self._identifier, 'get_audio')31 return self._audio[start:start + rows]32 def images(self, rows=15, start=0, refresh=False):33 if refresh or not CACHE:34 self._images = document.WebDocumentSet(self._identifier, 'get_images')35 return self._image[start:start + rows]36 37 def biographies(self, rows=15, start=0, refresh=False):38 if refresh or not CACHE:39 self._biographies = document.WebDocumentSet(self._identifier, 'get_biographies')40 return self._biographies[start:start + rows]41 def blogs(self, rows=15, start=0, refresh=False):42 if refresh or not CACHE:43 self._blogs = document.WebDocumentSet(self._identifier, 'get_blogs')44 return self._blogs[start:start + rows]45 46 def news(self, rows=15, start=0, refresh=False):47 if refresh or not CACHE:48 self._news = document.WebDocumentSet(self._identifier, 'get_news')49 return self._news[start:start + rows]50 51 def reviews(self, rows=15, start=0, refresh=False):52 if refresh or not CACHE:53 self._reviews = document.WebDocumentSet(self._identifier, 'get_reviews')54 return self._reviews[start:start + rows]55 56 def similar(self, rows=15, start=0, refresh=False):57 if refresh or not CACHE:58 self._similar = SimilarDocumentSet(self._identifier)59 return self._similar[start:start + rows]60 61 def video(self, rows=15, start=0, refresh=False):62 if refresh or not CACHE:63 self._video = document.WebDocumentSet(self._identifier, 'get_video')64 return self._video[start:start + rows]65 def familiarity(self, refresh=False):66 """Returns our numerical estimation of how 67 familiar an artist currently is to the world."""68 if self._familiarity is None or not CACHE:69 try:70 params = {'id': self.identifier}71 response = util.call('get_familiarity', params).findtext('artist/familiarity')72 self._familiarity = float(response)73 except:74 self._familiarity = 075 return self._familiarity76 def hotttnesss(self, refresh=False):77 """Returns our numerical description of how 78 hottt an artist currently is."""79 if self._hotttnesss is None or not CACHE:80 try:81 params = {'id': self.identifier}82 response = util.call('get_hotttnesss', params).findtext('artist/hotttnesss')83 self._hotttnesss = float(response)84 except:85 self._hotttnesss = 086 return self._hotttnesss87 @property88 def name(self):89 if self._name is None or not CACHE:90 self._name = util.call('get_profile', {'id': self.identifier}).findtext('artist/name')91 return self._name92 93 @property94 def identifier(self):95 """A unique identifier for an artist.96 See http://developer.echonest.com/docs/datatypes/97 for more information"""98 return self._identifier99 100 @property101 def urls(self):102 """Get links to the artist's official site, MusicBrainz site, 103 MySpace site, Wikipedia article, Amazon list, and iTunes page."""104 if self._urls is None or not CACHE:105 response = util.call('get_urls', {'id': self.identifier}).find('artist').getchildren()106 self._urls = dict((url.tag[:-4], url.text) for url in response if url.tag[-4:] == '_url')107 return self._urls108 def terms(self):109 if self._terms is None or not CACHE:110 response = util.call('get_top_terms', {'id': self.identifier}).findall('terms/term')111 self._terms = [e.text for e in response]112 return self._terms113 def __repr__(self):114 return "<Artist '%s'>" % str(self)115 116 def __str__(self):117 return self.name if isinstance(self.name, str) else self.name.encode('utf-8')118 119 def clear_cache(self):120 pass121TRUTH = {True: 'Y', False: 'N'}122SEARCH_ARTISTS_CACHE = {}123def search_artists(name, exact=False, sounds_like=True, rows=15, refresh=False):124 """Search for an artist using a query on the artist name.125 This may perform a sounds-like search to correct common 126 spelling mistakes."""127 global SEARCH_ARTISTS_CACHE128 if CACHE and not refresh:129 try:130 return SEARCH_ARTISTS_CACHE[(name, exact, sounds_like, rows)]131 except KeyError:132 pass133 params = {'query': name, 'exact': TRUTH[exact], 134 'sounds_like': TRUTH[sounds_like], 'rows': rows}135 response = util.call('search_artists', params).findall('artists/artist')136 value = [Artist(a.findtext('id'), a.findtext('name')) for a in response]137 SEARCH_ARTISTS_CACHE[(name, exact, sounds_like, rows)] = value138 return value139TOP_HOTTT_ARTISTS_CACHE = []140def get_top_hottt_artists(rows=15, refresh=False):141 """Retrieves a list of the top hottt artists.142 Do not request this more than once an hour."""143 global TOP_HOTTT_ARTISTS_CACHE144 if TOP_HOTTT_ARTISTS_CACHE==[] or refresh or not CACHE:145 response = util.call('get_top_hottt_artists', {'rows': rows}).findall('artists/artist')146 TOP_HOTTT_ARTISTS_CACHE = [Artist(a.findtext('id'), a.findtext('name')) for a in response]147 return TOP_HOTTT_ARTISTS_CACHE148class SimilarDocumentSet(document.DocumentSet):149 def __init__(self, identifier):150 super(SimilarDocumentSet, self).__init__(identifier, 'get_similar', 'similar/artist')151 def __len__(self):152 if self._len is None:153 self._len = 100154 return self._len155 def __getitem__(self, k):156 if k.stop > 100:157 raise util.EchoNestAPIError(5, 'Invalid parameter: "rows" must be less than or equal to 100')158 chunk_and_index = lambda i: (i / self._cache.chunk_size, i % self._cache.chunk_size)159 start, start_chunk, start_index, stop = self.chunk_magic(k)160 if (stop + 1) > self._cache.chunk_size:161 self.__init__(self.identifier)162 self._cache.chunk_size = stop + 1163 chunk_and_index = lambda i: (i / self._cache.chunk_size, i % self._cache.chunk_size)164 start, start_chunk, start_index, stop = self.chunk_magic(k)165 stop_chunk, stop_index = chunk_and_index(stop)166 items = []167 for chunk in xrange(start_chunk, stop_chunk + 1):168 elements = self._cache[chunk].findall(self.element_path)169 if chunk == start_chunk and chunk == stop_chunk:170 elements = elements[start_index:stop_index + 1]171 elif chunk == start_chunk:172 elements = elements[start_index:]173 elif chunk == stop_chunk:174 elements = elements[:stop_index + 1]175 items.extend([self._parse_element(e) for e in elements])176 return items177 def chunk_magic(self, k):178 chunk_and_index = lambda i: (i / self._cache.chunk_size, i % self._cache.chunk_size)179 if not isinstance(k, slice):180 raise TypeError181 start = k.start or 0182 start_chunk, start_index = chunk_and_index(start)183 stop = min(k.stop or len(self), len(self)) - 1 # use inclusive to simplify logic184 return start, start_chunk, start_index, stop185 def _parse_element(self, element):...

Full Screen

Full Screen

support.py

Source:support.py Github

copy

Full Screen

1from numpy import ndarray, array2def fix_slicing(index, logical_length, allow_column=True):3 """4 Checks, and caps, the slices being used.5 :param index: The initial index, which may be int or slice.6 :param logical_length: The current logical length of the data. It is used to cap the indices.7 If None, indices will not be capped.8 :param allow_column: Tells whether allowing 1 or 2 dimensions indexing.9 :return: The fixed slicing, if no exception occurs.10 """11 column_index = None12 if isinstance(index, tuple):13 if allow_column:14 if len(index) == 2:15 index, column_index = index16 else:17 raise IndexError("At most two indices/slices (bi-dimensional indexing) are supported")18 else:19 raise IndexError("Only one index/slice (one-dimensional indexing) is supported")20 if isinstance(index, slice):21 start = index.start22 stop = index.stop23 step = index.step24 if start is None:25 start = 026 if stop is None:27 stop = logical_length28 if step and step != 1:29 raise IndexError("Slices with step != 1 are not supported")30 if start < 0 or (stop is not None and stop < 0):31 raise IndexError("Negative indices in slices are not supported")32 if stop is not None and stop < start:33 raise IndexError("Slices must have start <= stop indices")34 elif logical_length is None:35 return start, stop, column_index36 else:37 # Here, start will be <= stop. We will limit both by the logical_length, silently.38 return min(start, logical_length), min(stop, logical_length), column_index39 elif isinstance(index, int):40 if index < 0:41 raise IndexError("Negative indices are not supported")42 if logical_length is None:43 return index, None, column_index44 else:45 return min(index, logical_length), None, column_index46 else:47 raise TypeError("Only slices (non-negative, growing, and with step 1) or non-negative integer indices are "48 "supported")49def fix_input(index, expected_width, expected_length, expected_type, value):50 """51 Checks and fixes the value according to the given index type.52 :param index: The initial index, which may be int or slice.53 :param expected_width: The expected width of the value.54 :param expected_length: The expected length of the data. It will be ignored if the index is a number.55 :param expected_type: The expected type for array input.56 :param value: The given value, which may be scalar or array.57 :return: The fixed value, if no exception occurs.58 """59 # If the input is an iterable, convert it to an 1-dimensional array.60 if isinstance(value, (tuple, list)):61 value = array(value)62 if isinstance(index, slice):63 if not isinstance(value, ndarray) and value.dtype != expected_type:64 raise TypeError("When setting a slice, the value must be a numpy array of (stop - start)x(width) "65 "elements (if the width is 1, an uni-dimensional array of (stop-start) elements is "66 "also allowed)")67 if expected_width == 1 and len(value.shape) == 1:68 value = value[:]69 value.shape = (value.size, 1)70 if value.shape != (expected_length, expected_width):71 raise TypeError("When setting a slice, the value must be a numpy array of (stop - start)x(width) "72 "elements (if the width is 1, an uni-dimensional array of (stop-start) elements is "73 "also allowed)")74 elif isinstance(index, int):75 if expected_width > 1 and (not isinstance(value, ndarray) or value.shape != (expected_width,)):76 raise TypeError("When setting an index, the value must be a 1-dimensional numpy array of the appropriate "77 "size (=width), if the expected width is > 1")78 if expected_width == 1 and not isinstance(value, ndarray):79 value = array((value,))80 else:81 raise TypeError("Only slices (non-negative, growing, and with step 1) or non-negative integer indices are "82 "supported")83 return value84def chunked_slicing(slice_start, slice_stop, chunk_size):85 """86 Iterator that yields, every time, a data structure like this:87 - Current chunk index88 - Start index in chunk89 - Stop index in chunk90 - Start index in source/destination91 - Stop index in source/destination92 It is guaranteed: 0 <= slice_start <= slice_stop <= logical length <= chunk_size * chunk_count.93 The algorithm goes like this:94 - Before: we know the starter bin, and the end bin.95 :param slice_start: The overall start.96 :param slice_stop: The overall stop.97 :param chunk_size: The chunk size.98 :return: A generator.99 """100 start_chunk = slice_start // chunk_size101 stop_chunk = slice_stop // chunk_size102 if start_chunk == stop_chunk:103 # This is the easiest case.104 # The data indices will be 0 and stop_chunk - start_chunk.105 # The chunk index will be start_chunk.106 # the start index in chunk, and end index in chunk, will both involve modulo.107 data_indices = (0, slice_stop - slice_start)108 chunk_indices = (slice_start % chunk_size, slice_stop % chunk_size)109 yield data_indices, start_chunk, chunk_indices110 else:111 # In this case, start chunk will always be lower than end chunk.112 chunk_start_index = slice_start % chunk_size113 chunk_stop_index = slice_stop % chunk_size114 first_iteration = True115 data_index = 0116 current_chunk = start_chunk117 # We know that, in the first iteration:118 # - chunk_start_index >= 0119 # - start_chunk < stop_chunk, strictly120 # And in further iterations:121 # - chunk_start_index == 0122 # - start_chunk <= stop_chunk123 while True:124 # The chunk upper bound will be:125 # - The chunk stop index, if in last chunk.126 # - The chunk size, otherwise.127 if current_chunk == stop_chunk:128 # Another check here: if the chunk stop index is 0, just break.129 if chunk_stop_index == 0:130 return131 current_chunk_ubound = chunk_stop_index132 else:133 current_chunk_ubound = chunk_size134 # The chunk lower bound will be:135 # - The chunk start index, if in first chunk.136 # - 0, otherwise.137 if first_iteration:138 current_chunk_lbound = chunk_start_index139 else:140 current_chunk_lbound = 0141 # Now we know the chunk start and chunk end bounds.142 # We also know the current chunk index.143 # We also know the current data index.144 # Also we can compute the current data length145 length = current_chunk_ubound - current_chunk_lbound146 # We can yield all the data now.147 data_indices = (data_index, data_index + length)148 chunk_indices = (current_chunk_lbound, current_chunk_ubound)149 yield data_indices, current_chunk, chunk_indices150 # If this is the last chunk, we must exit.151 if current_chunk == stop_chunk:152 return153 # If not, then we move the data index and the current chunk.154 current_chunk += 1155 data_index += length156 # Finish the first iteration....

Full Screen

Full Screen

Playwright tutorial

LambdaTest’s Playwright tutorial will give you a broader idea about the Playwright automation framework, its unique features, and use cases with examples to exceed your understanding of Playwright testing. This tutorial will give A to Z guidance, from installing the Playwright framework to some best practices and advanced concepts.

Chapters:

  1. What is Playwright : Playwright is comparatively new but has gained good popularity. Get to know some history of the Playwright with some interesting facts connected with it.
  2. How To Install Playwright : Learn in detail about what basic configuration and dependencies are required for installing Playwright and run a test. Get a step-by-step direction for installing the Playwright automation framework.
  3. Playwright Futuristic Features: Launched in 2020, Playwright gained huge popularity quickly because of some obliging features such as Playwright Test Generator and Inspector, Playwright Reporter, Playwright auto-waiting mechanism and etc. Read up on those features to master Playwright testing.
  4. What is Component Testing: Component testing in Playwright is a unique feature that allows a tester to test a single component of a web application without integrating them with other elements. Learn how to perform Component testing on the Playwright automation framework.
  5. Inputs And Buttons In Playwright: Every website has Input boxes and buttons; learn about testing inputs and buttons with different scenarios and examples.
  6. Functions and Selectors in Playwright: Learn how to launch the Chromium browser with Playwright. Also, gain a better understanding of some important functions like “BrowserContext,” which allows you to run multiple browser sessions, and “newPage” which interacts with a page.
  7. Handling Alerts and Dropdowns in Playwright : Playwright interact with different types of alerts and pop-ups, such as simple, confirmation, and prompt, and different types of dropdowns, such as single selector and multi-selector get your hands-on with handling alerts and dropdown in Playright testing.
  8. Playwright vs Puppeteer: Get to know about the difference between two testing frameworks and how they are different than one another, which browsers they support, and what features they provide.
  9. Run Playwright Tests on LambdaTest: Playwright testing with LambdaTest leverages test performance to the utmost. You can run multiple Playwright tests in Parallel with the LammbdaTest test cloud. Get a step-by-step guide to run your Playwright test on the LambdaTest platform.
  10. Playwright Python Tutorial: Playwright automation framework support all major languages such as Python, JavaScript, TypeScript, .NET and etc. However, there are various advantages to Python end-to-end testing with Playwright because of its versatile utility. Get the hang of Playwright python testing with this chapter.
  11. Playwright End To End Testing Tutorial: Get your hands on with Playwright end-to-end testing and learn to use some exciting features such as TraceViewer, Debugging, Networking, Component testing, Visual testing, and many more.
  12. Playwright Video Tutorial: Watch the video tutorials on Playwright testing from experts and get a consecutive in-depth explanation of Playwright automation testing.

Run Playwright Python automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

... View sample repo... View DocumentationSign up Free
›
_

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

...
Sign up with GoogleSign up with Email
Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful