How to use substr method in Lemoncheesecake
Best Python code snippet using lemoncheesecake
test_bigmem.py
Source:test_bigmem.py 
1from test import test_support2from test.test_support import bigmemtest, _1G, _2G34import unittest5import operator6import string7import sys89# Bigmem testing houserules:10#11# - Try not to allocate too many large objects. It's okay to rely on12# refcounting semantics, but don't forget that 's = create_largestring()'13# doesn't release the old 's' (if it exists) until well after its new14# value has been created. Use 'del s' before the create_largestring call.15#16# - Do *not* compare large objects using assertEquals or similar. It's a17# lengty operation and the errormessage will be utterly useless due to18# its size. To make sure whether a result has the right contents, better19# to use the strip or count methods, or compare meaningful slices.20#21# - Don't forget to test for large indices, offsets and results and such,22# in addition to large sizes.23#24# - When repeating an object (say, a substring, or a small list) to create25# a large object, make the subobject of a length that is not a power of26# 2. That way, int-wrapping problems are more easily detected.27#28# - While the bigmemtest decorator speaks of 'minsize', all tests will29# actually be called with a much smaller number too, in the normal30# test run (5Kb currently.) This is so the tests themselves get frequent31# testing. Consequently, always make all large allocations based on the32# passed-in 'size', and don't rely on the size being very large. Also,33# memuse-per-size should remain sane (less than a few thousand); if your34# test uses more, adjust 'size' upward, instead.3536class StrTest(unittest.TestCase):37 @bigmemtest(minsize=_2G, memuse=2)38 def test_capitalize(self, size):39 SUBSTR = ' abc def ghi'40 s = '-' * size + SUBSTR41 caps = s.capitalize()42 self.assertEquals(caps[-len(SUBSTR):],43 SUBSTR.capitalize())44 self.assertEquals(caps.lstrip('-'), SUBSTR)4546 @bigmemtest(minsize=_2G + 10, memuse=1)47 def test_center(self, size):48 SUBSTR = ' abc def ghi'49 s = SUBSTR.center(size)50 self.assertEquals(len(s), size)51 lpadsize = rpadsize = (len(s) - len(SUBSTR)) // 252 if len(s) % 2:53 lpadsize += 154 self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)55 self.assertEquals(s.strip(), SUBSTR.strip())5657 @bigmemtest(minsize=_2G, memuse=2)58 def test_count(self, size):59 SUBSTR = ' abc def ghi'60 s = '.' * size + SUBSTR61 self.assertEquals(s.count('.'), size)62 s += '.'63 self.assertEquals(s.count('.'), size + 1)64 self.assertEquals(s.count(' '), 3)65 self.assertEquals(s.count('i'), 1)66 self.assertEquals(s.count('j'), 0)6768 @bigmemtest(minsize=_2G + 2, memuse=3)69 def test_decode(self, size):70 s = '.' * size71 self.assertEquals(len(s.decode('utf-8')), size)7273 @bigmemtest(minsize=_2G + 2, memuse=3)74 def test_encode(self, size):75 s = u'.' * size76 self.assertEquals(len(s.encode('utf-8')), size)7778 @bigmemtest(minsize=_2G, memuse=2)79 def test_endswith(self, size):80 SUBSTR = ' abc def ghi'81 s = '-' * size + SUBSTR82 self.failUnless(s.endswith(SUBSTR))83 self.failUnless(s.endswith(s))84 s2 = '...' + s85 self.failUnless(s2.endswith(s))86 self.failIf(s.endswith('a' + SUBSTR))87 self.failIf(SUBSTR.endswith(s))8889 @bigmemtest(minsize=_2G + 10, memuse=2)90 def test_expandtabs(self, size):91 s = '-' * size92 tabsize = 893 self.assertEquals(s.expandtabs(), s)94 del s95 slen, remainder = divmod(size, tabsize)96 s = ' \t' * slen97 s = s.expandtabs(tabsize)98 self.assertEquals(len(s), size - remainder)99 self.assertEquals(len(s.strip(' ')), 0)100101 @bigmemtest(minsize=_2G, memuse=2)102 def test_find(self, size):103 SUBSTR = ' abc def ghi'104 sublen = len(SUBSTR)105 s = ''.join([SUBSTR, '-' * size, SUBSTR])106 self.assertEquals(s.find(' '), 0)107 self.assertEquals(s.find(SUBSTR), 0)108 self.assertEquals(s.find(' ', sublen), sublen + size)109 self.assertEquals(s.find(SUBSTR, len(SUBSTR)), sublen + size)110 self.assertEquals(s.find('i'), SUBSTR.find('i'))111 self.assertEquals(s.find('i', sublen),112 sublen + size + SUBSTR.find('i'))113 self.assertEquals(s.find('i', size),114 sublen + size + SUBSTR.find('i'))115 self.assertEquals(s.find('j'), -1)116117 @bigmemtest(minsize=_2G, memuse=2)118 def test_index(self, size):119 SUBSTR = ' abc def ghi'120 sublen = len(SUBSTR)121 s = ''.join([SUBSTR, '-' * size, SUBSTR])122 self.assertEquals(s.index(' '), 0)123 self.assertEquals(s.index(SUBSTR), 0)124 self.assertEquals(s.index(' ', sublen), sublen + size)125 self.assertEquals(s.index(SUBSTR, sublen), sublen + size)126 self.assertEquals(s.index('i'), SUBSTR.index('i'))127 self.assertEquals(s.index('i', sublen),128 sublen + size + SUBSTR.index('i'))129 self.assertEquals(s.index('i', size),130 sublen + size + SUBSTR.index('i'))131 self.assertRaises(ValueError, s.index, 'j')132133 @bigmemtest(minsize=_2G, memuse=2)134 def test_isalnum(self, size):135 SUBSTR = '123456'136 s = 'a' * size + SUBSTR137 self.failUnless(s.isalnum())138 s += '.'139 self.failIf(s.isalnum())140141 @bigmemtest(minsize=_2G, memuse=2)142 def test_isalpha(self, size):143 SUBSTR = 'zzzzzzz'144 s = 'a' * size + SUBSTR145 self.failUnless(s.isalpha())146 s += '.'147 self.failIf(s.isalpha())148149 @bigmemtest(minsize=_2G, memuse=2)150 def test_isdigit(self, size):151 SUBSTR = '123456'152 s = '9' * size + SUBSTR153 self.failUnless(s.isdigit())154 s += 'z'155 self.failIf(s.isdigit())156157 @bigmemtest(minsize=_2G, memuse=2)158 def test_islower(self, size):159 chars = ''.join([ chr(c) for c in range(255) if not chr(c).isupper() ])160 repeats = size // len(chars) + 2161 s = chars * repeats162 self.failUnless(s.islower())163 s += 'A'164 self.failIf(s.islower())165166 @bigmemtest(minsize=_2G, memuse=2)167 def test_isspace(self, size):168 whitespace = ' \f\n\r\t\v'169 repeats = size // len(whitespace) + 2170 s = whitespace * repeats171 self.failUnless(s.isspace())172 s += 'j'173 self.failIf(s.isspace())174175 @bigmemtest(minsize=_2G, memuse=2)176 def test_istitle(self, size):177 SUBSTR = '123456'178 s = ''.join(['A', 'a' * size, SUBSTR])179 self.failUnless(s.istitle())180 s += 'A'181 self.failUnless(s.istitle())182 s += 'aA'183 self.failIf(s.istitle())184185 @bigmemtest(minsize=_2G, memuse=2)186 def test_isupper(self, size):187 chars = ''.join([ chr(c) for c in range(255) if not chr(c).islower() ])188 repeats = size // len(chars) + 2189 s = chars * repeats190 self.failUnless(s.isupper())191 s += 'a'192 self.failIf(s.isupper())193194 @bigmemtest(minsize=_2G, memuse=2)195 def test_join(self, size):196 s = 'A' * size197 x = s.join(['aaaaa', 'bbbbb'])198 self.assertEquals(x.count('a'), 5)199 self.assertEquals(x.count('b'), 5)200 self.failUnless(x.startswith('aaaaaA'))201 self.failUnless(x.endswith('Abbbbb'))202203 @bigmemtest(minsize=_2G + 10, memuse=1)204 def test_ljust(self, size):205 SUBSTR = ' abc def ghi'206 s = SUBSTR.ljust(size)207 self.failUnless(s.startswith(SUBSTR + ' '))208 self.assertEquals(len(s), size)209 self.assertEquals(s.strip(), SUBSTR.strip())210211 @bigmemtest(minsize=_2G + 10, memuse=2)212 def test_lower(self, size):213 s = 'A' * size214 s = s.lower()215 self.assertEquals(len(s), size)216 self.assertEquals(s.count('a'), size)217218 @bigmemtest(minsize=_2G + 10, memuse=1)219 def test_lstrip(self, size):220 SUBSTR = 'abc def ghi'221 s = SUBSTR.rjust(size)222 self.assertEquals(len(s), size)223 self.assertEquals(s.lstrip(), SUBSTR.lstrip())224 del s225 s = SUBSTR.ljust(size)226 self.assertEquals(len(s), size)227 stripped = s.lstrip()228 self.failUnless(stripped is s)229230 @bigmemtest(minsize=_2G + 10, memuse=2)231 def test_replace(self, size):232 replacement = 'a'233 s = ' ' * size234 s = s.replace(' ', replacement)235 self.assertEquals(len(s), size)236 self.assertEquals(s.count(replacement), size)237 s = s.replace(replacement, ' ', size - 4)238 self.assertEquals(len(s), size)239 self.assertEquals(s.count(replacement), 4)240 self.assertEquals(s[-10:], ' aaaa')241242 @bigmemtest(minsize=_2G, memuse=2)243 def test_rfind(self, size):244 SUBSTR = ' abc def ghi'245 sublen = len(SUBSTR)246 s = ''.join([SUBSTR, '-' * size, SUBSTR])247 self.assertEquals(s.rfind(' '), sublen + size + SUBSTR.rfind(' '))248 self.assertEquals(s.rfind(SUBSTR), sublen + size)249 self.assertEquals(s.rfind(' ', 0, size), SUBSTR.rfind(' '))250 self.assertEquals(s.rfind(SUBSTR, 0, sublen + size), 0)251 self.assertEquals(s.rfind('i'), sublen + size + SUBSTR.rfind('i'))252 self.assertEquals(s.rfind('i', 0, sublen), SUBSTR.rfind('i'))253 self.assertEquals(s.rfind('i', 0, sublen + size),254 SUBSTR.rfind('i'))255 self.assertEquals(s.rfind('j'), -1)256257 @bigmemtest(minsize=_2G, memuse=2)258 def test_rindex(self, size):259 SUBSTR = ' abc def ghi'260 sublen = len(SUBSTR)261 s = ''.join([SUBSTR, '-' * size, SUBSTR])262 self.assertEquals(s.rindex(' '),263 sublen + size + SUBSTR.rindex(' '))264 self.assertEquals(s.rindex(SUBSTR), sublen + size)265 self.assertEquals(s.rindex(' ', 0, sublen + size - 1),266 SUBSTR.rindex(' '))267 self.assertEquals(s.rindex(SUBSTR, 0, sublen + size), 0)268 self.assertEquals(s.rindex('i'),269 sublen + size + SUBSTR.rindex('i'))270 self.assertEquals(s.rindex('i', 0, sublen), SUBSTR.rindex('i'))271 self.assertEquals(s.rindex('i', 0, sublen + size),272 SUBSTR.rindex('i'))273 self.assertRaises(ValueError, s.rindex, 'j')274275 @bigmemtest(minsize=_2G + 10, memuse=1)276 def test_rjust(self, size):277 SUBSTR = ' abc def ghi'278 s = SUBSTR.ljust(size)279 self.failUnless(s.startswith(SUBSTR + ' '))280 self.assertEquals(len(s), size)281 self.assertEquals(s.strip(), SUBSTR.strip())282283 @bigmemtest(minsize=_2G + 10, memuse=1)284 def test_rstrip(self, size):285 SUBSTR = ' abc def ghi'286 s = SUBSTR.ljust(size)287 self.assertEquals(len(s), size)288 self.assertEquals(s.rstrip(), SUBSTR.rstrip())289 del s290 s = SUBSTR.rjust(size)291 self.assertEquals(len(s), size)292 stripped = s.rstrip()293 self.failUnless(stripped is s)294295 # The test takes about size bytes to build a string, and then about296 # sqrt(size) substrings of sqrt(size) in size and a list to297 # hold sqrt(size) items. It's close but just over 2x size.298 @bigmemtest(minsize=_2G, memuse=2.1)299 def test_split_small(self, size):300 # Crudely calculate an estimate so that the result of s.split won't301 # take up an inordinate amount of memory302 chunksize = int(size ** 0.5 + 2)303 SUBSTR = 'a' + ' ' * chunksize304 s = SUBSTR * chunksize305 l = s.split()306 self.assertEquals(len(l), chunksize)307 self.assertEquals(set(l), set(['a']))308 del l309 l = s.split('a')310 self.assertEquals(len(l), chunksize + 1)311 self.assertEquals(set(l), set(['', ' ' * chunksize]))312313 # Allocates a string of twice size (and briefly two) and a list of314 # size. Because of internal affairs, the s.split() call produces a315 # list of size times the same one-character string, so we only316 # suffer for the list size. (Otherwise, it'd cost another 48 times317 # size in bytes!) Nevertheless, a list of size takes318 # 8*size bytes.319 @bigmemtest(minsize=_2G + 5, memuse=10)320 def test_split_large(self, size):321 s = ' a' * size + ' '322 l = s.split()323 self.assertEquals(len(l), size)324 self.assertEquals(set(l), set(['a']))325 del l326 l = s.split('a')327 self.assertEquals(len(l), size + 1)328 self.assertEquals(set(l), set([' ']))329330 @bigmemtest(minsize=_2G, memuse=2.1)331 def test_splitlines(self, size):332 # Crudely calculate an estimate so that the result of s.split won't333 # take up an inordinate amount of memory334 chunksize = int(size ** 0.5 + 2) // 2335 SUBSTR = ' ' * chunksize + '\n' + ' ' * chunksize + '\r\n'336 s = SUBSTR * chunksize337 l = s.splitlines()338 self.assertEquals(len(l), chunksize * 2)339 self.assertEquals(set(l), set([' ' * chunksize]))340341 @bigmemtest(minsize=_2G, memuse=2)342 def test_startswith(self, size):343 SUBSTR = ' abc def ghi'344 s = '-' * size + SUBSTR345 self.failUnless(s.startswith(s))346 self.failUnless(s.startswith('-' * size))347 self.failIf(s.startswith(SUBSTR))348349 @bigmemtest(minsize=_2G, memuse=1)350 def test_strip(self, size):351 SUBSTR = ' abc def ghi '352 s = SUBSTR.rjust(size)353 self.assertEquals(len(s), size)354 self.assertEquals(s.strip(), SUBSTR.strip())355 del s356 s = SUBSTR.ljust(size)357 self.assertEquals(len(s), size)358 self.assertEquals(s.strip(), SUBSTR.strip())359360 @bigmemtest(minsize=_2G, memuse=2)361 def test_swapcase(self, size):362 SUBSTR = "aBcDeFG12.'\xa9\x00"363 sublen = len(SUBSTR)364 repeats = size // sublen + 2365 s = SUBSTR * repeats366 s = s.swapcase()367 self.assertEquals(len(s), sublen * repeats)368 self.assertEquals(s[:sublen * 3], SUBSTR.swapcase() * 3)369 self.assertEquals(s[-sublen * 3:], SUBSTR.swapcase() * 3)370371 @bigmemtest(minsize=_2G, memuse=2)372 def test_title(self, size):373 SUBSTR = 'SpaaHAaaAaham'374 s = SUBSTR * (size // len(SUBSTR) + 2)375 s = s.title()376 self.failUnless(s.startswith((SUBSTR * 3).title()))377 self.failUnless(s.endswith(SUBSTR.lower() * 3))378379 @bigmemtest(minsize=_2G, memuse=2)380 def test_translate(self, size):381 trans = string.maketrans('.aZ', '-!$')382 SUBSTR = 'aZz.z.Aaz.'383 sublen = len(SUBSTR)384 repeats = size // sublen + 2385 s = SUBSTR * repeats386 s = s.translate(trans)387 self.assertEquals(len(s), repeats * sublen)388 self.assertEquals(s[:sublen], SUBSTR.translate(trans))389 self.assertEquals(s[-sublen:], SUBSTR.translate(trans))390 self.assertEquals(s.count('.'), 0)391 self.assertEquals(s.count('!'), repeats * 2)392 self.assertEquals(s.count('z'), repeats * 3)393394 @bigmemtest(minsize=_2G + 5, memuse=2)395 def test_upper(self, size):396 s = 'a' * size397 s = s.upper()398 self.assertEquals(len(s), size)399 self.assertEquals(s.count('A'), size)400401 @bigmemtest(minsize=_2G + 20, memuse=1)402 def test_zfill(self, size):403 SUBSTR = '-568324723598234'404 s = SUBSTR.zfill(size)405 self.failUnless(s.endswith('0' + SUBSTR[1:]))406 self.failUnless(s.startswith('-0'))407 self.assertEquals(len(s), size)408 self.assertEquals(s.count('0'), size - len(SUBSTR))409410 @bigmemtest(minsize=_2G + 10, memuse=2)411 def test_format(self, size):412 s = '-' * size413 sf = '%s' % (s,)414 self.failUnless(s == sf)415 del sf416 sf = '..%s..' % (s,)417 self.assertEquals(len(sf), len(s) + 4)418 self.failUnless(sf.startswith('..-'))419 self.failUnless(sf.endswith('-..'))420 del s, sf421422 size //= 2423 edge = '-' * size424 s = ''.join([edge, '%s', edge])425 del edge426 s = s % '...'427 self.assertEquals(len(s), size * 2 + 3)428 self.assertEquals(s.count('.'), 3)429 self.assertEquals(s.count('-'), size * 2)430431 @bigmemtest(minsize=_2G + 10, memuse=2)432 def test_repr_small(self, size):433 s = '-' * size434 s = repr(s)435 self.assertEquals(len(s), size + 2)436 self.assertEquals(s[0], "'")437 self.assertEquals(s[-1], "'")438 self.assertEquals(s.count('-'), size)439 del s440 # repr() will create a string four times as large as this 'binary441 # string', but we don't want to allocate much more than twice442 # size in total. (We do extra testing in test_repr_large())443 size = size // 5 * 2444 s = '\x00' * size445 s = repr(s)446 self.assertEquals(len(s), size * 4 + 2)447 self.assertEquals(s[0], "'")448 self.assertEquals(s[-1], "'")449 self.assertEquals(s.count('\\'), size)450 self.assertEquals(s.count('0'), size * 2)451452 @bigmemtest(minsize=_2G + 10, memuse=5)453 def test_repr_large(self, size):454 s = '\x00' * size455 s = repr(s)456 self.assertEquals(len(s), size * 4 + 2)457 self.assertEquals(s[0], "'")458 self.assertEquals(s[-1], "'")459 self.assertEquals(s.count('\\'), size)460 self.assertEquals(s.count('0'), size * 2)461462 # This test is meaningful even with size < 2G, as long as the463 # doubled string is > 2G (but it tests more if both are > 2G :)464 @bigmemtest(minsize=_1G + 2, memuse=3)465 def test_concat(self, size):466 s = '.' * size467 self.assertEquals(len(s), size)468 s = s + s469 self.assertEquals(len(s), size * 2)470 self.assertEquals(s.count('.'), size * 2)471472 # This test is meaningful even with size < 2G, as long as the473 # repeated string is > 2G (but it tests more if both are > 2G :)474 @bigmemtest(minsize=_1G + 2, memuse=3)475 def test_repeat(self, size):476 s = '.' * size477 self.assertEquals(len(s), size)478 s = s * 2479 self.assertEquals(len(s), size * 2)480 self.assertEquals(s.count('.'), size * 2)481482 @bigmemtest(minsize=_2G + 20, memuse=1)483 def test_slice_and_getitem(self, size):484 SUBSTR = '0123456789'485 sublen = len(SUBSTR)486 s = SUBSTR * (size // sublen)487 stepsize = len(s) // 100488 stepsize = stepsize - (stepsize % sublen)489 for i in range(0, len(s) - stepsize, stepsize):490 self.assertEquals(s[i], SUBSTR[0])491 self.assertEquals(s[i:i + sublen], SUBSTR)492 self.assertEquals(s[i:i + sublen:2], SUBSTR[::2])493 if i > 0:494 self.assertEquals(s[i + sublen - 1:i - 1:-3],495 SUBSTR[sublen::-3])496 # Make sure we do some slicing and indexing near the end of the497 # string, too.498 self.assertEquals(s[len(s) - 1], SUBSTR[-1])499 self.assertEquals(s[-1], SUBSTR[-1])500 self.assertEquals(s[len(s) - 10], SUBSTR[0])501 self.assertEquals(s[-sublen], SUBSTR[0])502 self.assertEquals(s[len(s):], '')503 self.assertEquals(s[len(s) - 1:], SUBSTR[-1])504 self.assertEquals(s[-1:], SUBSTR[-1])505 self.assertEquals(s[len(s) - sublen:], SUBSTR)506 self.assertEquals(s[-sublen:], SUBSTR)507 self.assertEquals(len(s[:]), len(s))508 self.assertEquals(len(s[:len(s) - 5]), len(s) - 5)509 self.assertEquals(len(s[5:-5]), len(s) - 10)510511 self.assertRaises(IndexError, operator.getitem, s, len(s))512 self.assertRaises(IndexError, operator.getitem, s, len(s) + 1)513 self.assertRaises(IndexError, operator.getitem, s, len(s) + 1<<31)514515 @bigmemtest(minsize=_2G, memuse=2)516 def test_contains(self, size):517 SUBSTR = '0123456789'518 edge = '-' * (size // 2)519 s = ''.join([edge, SUBSTR, edge])520 del edge521 self.failUnless(SUBSTR in s)522 self.failIf(SUBSTR * 2 in s)523 self.failUnless('-' in s)524 self.failIf('a' in s)525 s += 'a'526 self.failUnless('a' in s)527528 @bigmemtest(minsize=_2G + 10, memuse=2)529 def test_compare(self, size):530 s1 = '-' * size531 s2 = '-' * size532 self.failUnless(s1 == s2)533 del s2534 s2 = s1 + 'a'535 self.failIf(s1 == s2)536 del s2537 s2 = '.' * size538 self.failIf(s1 == s2)539540 @bigmemtest(minsize=_2G + 10, memuse=1)541 def test_hash(self, size):542 # Not sure if we can do any meaningful tests here... Even if we543 # start relying on the exact algorithm used, the result will be544 # different depending on the size of the C 'long int'. Even this545 # test is dodgy (there's no *guarantee* that the two things should546 # have a different hash, even if they, in the current547 # implementation, almost always do.)548 s = '\x00' * size549 h1 = hash(s)550 del s551 s = '\x00' * (size + 1)552 self.failIf(h1 == hash(s))553554class TupleTest(unittest.TestCase):555556 # Tuples have a small, fixed-sized head and an array of pointers to557 # data. Since we're testing 64-bit addressing, we can assume that the558 # pointers are 8 bytes, and that thus that the tuples take up 8 bytes559 # per size.560561 # As a side-effect of testing long tuples, these tests happen to test562 # having more than 2<<31 references to any given object. Hence the563 # use of different types of objects as contents in different tests.564565 @bigmemtest(minsize=_2G + 2, memuse=16)566 def test_compare(self, size):567 t1 = (u'',) * size568 t2 = (u'',) * size569 self.failUnless(t1 == t2)570 del t2571 t2 = (u'',) * (size + 1)572 self.failIf(t1 == t2)573 del t2574 t2 = (1,) * size575 self.failIf(t1 == t2)576577 # Test concatenating into a single tuple of more than 2G in length,578 # and concatenating a tuple of more than 2G in length separately, so579 # the smaller test still gets run even if there isn't memory for the580 # larger test (but we still let the tester know the larger test is581 # skipped, in verbose mode.)582 def basic_concat_test(self, size):583 t = ((),) * size584 self.assertEquals(len(t), size)585 t = t + t586 self.assertEquals(len(t), size * 2)587588 @bigmemtest(minsize=_2G // 2 + 2, memuse=24)589 def test_concat_small(self, size):590 return self.basic_concat_test(size)591592 @bigmemtest(minsize=_2G + 2, memuse=24)593 def test_concat_large(self, size):594 return self.basic_concat_test(size)595596 @bigmemtest(minsize=_2G // 5 + 10, memuse=8 * 5)597 def test_contains(self, size):598 t = (1, 2, 3, 4, 5) * size599 self.assertEquals(len(t), size * 5)600 self.failUnless(5 in t)601 self.failIf((1, 2, 3, 4, 5) in t)602 self.failIf(0 in t)603604 @bigmemtest(minsize=_2G + 10, memuse=8)605 def test_hash(self, size):606 t1 = (0,) * size607 h1 = hash(t1)608 del t1609 t2 = (0,) * (size + 1)610 self.failIf(h1 == hash(t2))611612 @bigmemtest(minsize=_2G + 10, memuse=8)613 def test_index_and_slice(self, size):614 t = (None,) * size615 self.assertEquals(len(t), size)616 self.assertEquals(t[-1], None)617 self.assertEquals(t[5], None)618 self.assertEquals(t[size - 1], None)619 self.assertRaises(IndexError, operator.getitem, t, size)620 self.assertEquals(t[:5], (None,) * 5)621 self.assertEquals(t[-5:], (None,) * 5)622 self.assertEquals(t[20:25], (None,) * 5)623 self.assertEquals(t[-25:-20], (None,) * 5)624 self.assertEquals(t[size - 5:], (None,) * 5)625 self.assertEquals(t[size - 5:size], (None,) * 5)626 self.assertEquals(t[size - 6:size - 2], (None,) * 4)627 self.assertEquals(t[size:size], ())628 self.assertEquals(t[size:size+5], ())629630 # Like test_concat, split in two.631 def basic_test_repeat(self, size):632 t = ('',) * size633 self.assertEquals(len(t), size)634 t = t * 2635 self.assertEquals(len(t), size * 2)636637 @bigmemtest(minsize=_2G // 2 + 2, memuse=24)638 def test_repeat_small(self, size):639 return self.basic_test_repeat(size)640641 @bigmemtest(minsize=_2G + 2, memuse=24)642 def test_repeat_large(self, size):643 return self.basic_test_repeat(size)644645 # Like test_concat, split in two.646 def basic_test_repr(self, size):647 t = (0,) * size648 s = repr(t)649 # The repr of a tuple of 0's is exactly three times the tuple length.650 self.assertEquals(len(s), size * 3)651 self.assertEquals(s[:5], '(0, 0')652 self.assertEquals(s[-5:], '0, 0)')653 self.assertEquals(s.count('0'), size)654655 @bigmemtest(minsize=_2G // 3 + 2, memuse=8 + 3)656 def test_repr_small(self, size):657 return self.basic_test_repr(size)658659 @bigmemtest(minsize=_2G + 2, memuse=8 + 3)660 def test_repr_large(self, size):661 return self.basic_test_repr(size)662663class ListTest(unittest.TestCase):664665 # Like tuples, lists have a small, fixed-sized head and an array of666 # pointers to data, so 8 bytes per size. Also like tuples, we make the667 # lists hold references to various objects to test their refcount668 # limits.669670 @bigmemtest(minsize=_2G + 2, memuse=16)671 def test_compare(self, size):672 l1 = [u''] * size673 l2 = [u''] * size674 self.failUnless(l1 == l2)675 del l2676 l2 = [u''] * (size + 1)677 self.failIf(l1 == l2)678 del l2679 l2 = [2] * size680 self.failIf(l1 == l2)681682 # Test concatenating into a single list of more than 2G in length,683 # and concatenating a list of more than 2G in length separately, so684 # the smaller test still gets run even if there isn't memory for the685 # larger test (but we still let the tester know the larger test is686 # skipped, in verbose mode.)687 def basic_test_concat(self, size):688 l = [[]] * size689 self.assertEquals(len(l), size)690 l = l + l691 self.assertEquals(len(l), size * 2)692693 @bigmemtest(minsize=_2G // 2 + 2, memuse=24)694 def test_concat_small(self, size):695 return self.basic_test_concat(size)696697 @bigmemtest(minsize=_2G + 2, memuse=24)698 def test_concat_large(self, size):699 return self.basic_test_concat(size)700701 def basic_test_inplace_concat(self, size):702 l = [sys.stdout] * size703 l += l704 self.assertEquals(len(l), size * 2)705 self.failUnless(l[0] is l[-1])706 self.failUnless(l[size - 1] is l[size + 1])707708 @bigmemtest(minsize=_2G // 2 + 2, memuse=24)709 def test_inplace_concat_small(self, size):710 return self.basic_test_inplace_concat(size)711712 @bigmemtest(minsize=_2G + 2, memuse=24)713 def test_inplace_concat_large(self, size):714 return self.basic_test_inplace_concat(size)715716 @bigmemtest(minsize=_2G // 5 + 10, memuse=8 * 5)717 def test_contains(self, size):718 l = [1, 2, 3, 4, 5] * size719 self.assertEquals(len(l), size * 5)720 self.failUnless(5 in l)721 self.failIf([1, 2, 3, 4, 5] in l)722 self.failIf(0 in l)723724 @bigmemtest(minsize=_2G + 10, memuse=8)725 def test_hash(self, size):726 l = [0] * size727 self.failUnlessRaises(TypeError, hash, l)728729 @bigmemtest(minsize=_2G + 10, memuse=8)730 def test_index_and_slice(self, size):731 l = [None] * size732 self.assertEquals(len(l), size)733 self.assertEquals(l[-1], None)734 self.assertEquals(l[5], None)735 self.assertEquals(l[size - 1], None)736 self.assertRaises(IndexError, operator.getitem, l, size)737 self.assertEquals(l[:5], [None] * 5)738 self.assertEquals(l[-5:], [None] * 5)739 self.assertEquals(l[20:25], [None] * 5)740 self.assertEquals(l[-25:-20], [None] * 5)741 self.assertEquals(l[size - 5:], [None] * 5)742 self.assertEquals(l[size - 5:size], [None] * 5)743 self.assertEquals(l[size - 6:size - 2], [None] * 4)744 self.assertEquals(l[size:size], [])745 self.assertEquals(l[size:size+5], [])746747 l[size - 2] = 5748 self.assertEquals(len(l), size)749 self.assertEquals(l[-3:], [None, 5, None])750 self.assertEquals(l.count(5), 1)751 self.assertRaises(IndexError, operator.setitem, l, size, 6)752 self.assertEquals(len(l), size)753754 l[size - 7:] = [1, 2, 3, 4, 5]755 size -= 2756 self.assertEquals(len(l), size)757 self.assertEquals(l[-7:], [None, None, 1, 2, 3, 4, 5])758759 l[:7] = [1, 2, 3, 4, 5]760 size -= 2761 self.assertEquals(len(l), size)762 self.assertEquals(l[:7], [1, 2, 3, 4, 5, None, None])763764 del l[size - 1]765 size -= 1766 self.assertEquals(len(l), size)767 self.assertEquals(l[-1], 4)768769 del l[-2:]770 size -= 2771 self.assertEquals(len(l), size)772 self.assertEquals(l[-1], 2)773774 del l[0]775 size -= 1776 self.assertEquals(len(l), size)777 self.assertEquals(l[0], 2)778779 del l[:2]780 size -= 2781 self.assertEquals(len(l), size)782 self.assertEquals(l[0], 4)783784 # Like test_concat, split in two.785 def basic_test_repeat(self, size):786 l = [] * size787 self.failIf(l)788 l = [''] * size789 self.assertEquals(len(l), size)790 l = l * 2791 self.assertEquals(len(l), size * 2)792793 @bigmemtest(minsize=_2G // 2 + 2, memuse=24)794 def test_repeat_small(self, size):795 return self.basic_test_repeat(size)796797 @bigmemtest(minsize=_2G + 2, memuse=24)798 def test_repeat_large(self, size):799 return self.basic_test_repeat(size)800801 def basic_test_inplace_repeat(self, size):802 l = ['']803 l *= size804 self.assertEquals(len(l), size)805 self.failUnless(l[0] is l[-1])806 del l807808 l = [''] * size809 l *= 2810 self.assertEquals(len(l), size * 2)811 self.failUnless(l[size - 1] is l[-1])812813 @bigmemtest(minsize=_2G // 2 + 2, memuse=16)814 def test_inplace_repeat_small(self, size):815 return self.basic_test_inplace_repeat(size)816817 @bigmemtest(minsize=_2G + 2, memuse=16)818 def test_inplace_repeat_large(self, size):819 return self.basic_test_inplace_repeat(size)820821 def basic_test_repr(self, size):822 l = [0] * size823 s = repr(l)824 # The repr of a list of 0's is exactly three times the list length.825 self.assertEquals(len(s), size * 3)826 self.assertEquals(s[:5], '[0, 0')827 self.assertEquals(s[-5:], '0, 0]')828 self.assertEquals(s.count('0'), size)829830 @bigmemtest(minsize=_2G // 3 + 2, memuse=8 + 3)831 def test_repr_small(self, size):832 return self.basic_test_repr(size)833834 @bigmemtest(minsize=_2G + 2, memuse=8 + 3)835 def test_repr_large(self, size):836 return self.basic_test_repr(size)837838 # list overallocates ~1/8th of the total size (on first expansion) so839 # the single list.append call puts memuse at 9 bytes per size.840 @bigmemtest(minsize=_2G, memuse=9)841 def test_append(self, size):842 l = [object()] * size843 l.append(object())844 self.assertEquals(len(l), size+1)845 self.failUnless(l[-3] is l[-2])846 self.failIf(l[-2] is l[-1])847848 @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5)849 def test_count(self, size):850 l = [1, 2, 3, 4, 5] * size851 self.assertEquals(l.count(1), size)852 self.assertEquals(l.count("1"), 0)853854 def basic_test_extend(self, size):855 l = [file] * size856 l.extend(l)857 self.assertEquals(len(l), size * 2)858 self.failUnless(l[0] is l[-1])859 self.failUnless(l[size - 1] is l[size + 1])860861 @bigmemtest(minsize=_2G // 2 + 2, memuse=16)862 def test_extend_small(self, size):863 return self.basic_test_extend(size)864865 @bigmemtest(minsize=_2G + 2, memuse=16)866 def test_extend_large(self, size):867 return self.basic_test_extend(size)868869 @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5)870 def test_index(self, size):871 l = [1L, 2L, 3L, 4L, 5L] * size872 size *= 5873 self.assertEquals(l.index(1), 0)874 self.assertEquals(l.index(5, size - 5), size - 1)875 self.assertEquals(l.index(5, size - 5, size), size - 1)876 self.assertRaises(ValueError, l.index, 1, size - 4, size)877 self.assertRaises(ValueError, l.index, 6L)878879 # This tests suffers from overallocation, just like test_append.880 @bigmemtest(minsize=_2G + 10, memuse=9)881 def test_insert(self, size):882 l = [1.0] * size883 l.insert(size - 1, "A")884 size += 1885 self.assertEquals(len(l), size)886 self.assertEquals(l[-3:], [1.0, "A", 1.0])887888 l.insert(size + 1, "B")889 size += 1890 self.assertEquals(len(l), size)891 self.assertEquals(l[-3:], ["A", 1.0, "B"])892893 l.insert(1, "C")894 size += 1895 self.assertEquals(len(l), size)896 self.assertEquals(l[:3], [1.0, "C", 1.0])897 self.assertEquals(l[size - 3:], ["A", 1.0, "B"])898899 @bigmemtest(minsize=_2G // 5 + 4, memuse=8 * 5)900 def test_pop(self, size):901 l = [u"a", u"b", u"c", u"d", u"e"] * size902 size *= 5903 self.assertEquals(len(l), size)904905 item = l.pop()906 size -= 1907 self.assertEquals(len(l), size)908 self.assertEquals(item, u"e")909 self.assertEquals(l[-2:], [u"c", u"d"])910911 item = l.pop(0)912 size -= 1913 self.assertEquals(len(l), size)914 self.assertEquals(item, u"a")915 self.assertEquals(l[:2], [u"b", u"c"])916917 item = l.pop(size - 2)918 size -= 1919 self.assertEquals(len(l), size)920 self.assertEquals(item, u"c")921 self.assertEquals(l[-2:], [u"b", u"d"])922923 @bigmemtest(minsize=_2G + 10, memuse=8)924 def test_remove(self, size):925 l = [10] * size926 self.assertEquals(len(l), size)927928 l.remove(10)929 size -= 1930 self.assertEquals(len(l), size)931932 # Because of the earlier l.remove(), this append doesn't trigger933 # a resize.934 l.append(5)935 size += 1936 self.assertEquals(len(l), size)937 self.assertEquals(l[-2:], [10, 5])938 l.remove(5)939 size -= 1940 self.assertEquals(len(l), size)941 self.assertEquals(l[-2:], [10, 10])942943 @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5)944 def test_reverse(self, size):945 l = [1, 2, 3, 4, 5] * size946 l.reverse()947 self.assertEquals(len(l), size * 5)948 self.assertEquals(l[-5:], [5, 4, 3, 2, 1])949 self.assertEquals(l[:5], [5, 4, 3, 2, 1])950951 @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5)952 def test_sort(self, size):953 l = [1, 2, 3, 4, 5] * size954 l.sort()955 self.assertEquals(len(l), size * 5)956 self.assertEquals(l.count(1), size)957 self.assertEquals(l[:10], [1] * 10)958 self.assertEquals(l[-10:], [5] * 10)959960def test_main():961 test_support.run_unittest(StrTest, TupleTest, ListTest)962963if __name__ == '__main__':964 if len(sys.argv) > 1:965 test_support.set_memlimit(sys.argv[1])
...
substr_op_test.py
Source:substr_op_test.py
...25 test_string = b"Hello"26 position = np.array(1, dtype)27 length = np.array(3, dtype)28 expected_value = b"ell"29 substr_op = string_ops.substr(test_string, position, length)30 with self.test_session():31 substr = substr_op.eval()32 self.assertAllEqual(substr, expected_value)33 # position is equal to the length of string.34 test_string = b""35 position = np.array(0, dtype)36 length = np.array(2, dtype)37 expected_value = b""38 substr_op = string_ops.substr(test_string, position, length)39 with self.test_session():40 substr = substr_op.eval()41 self.assertAllEqual(substr, expected_value)42 def _testVectorStrings(self, dtype):43 test_string = [b"Hello", b"World"]44 position = np.array(1, dtype)45 length = np.array(3, dtype)46 expected_value = [b"ell", b"orl"]47 substr_op = string_ops.substr(test_string, position, length)48 with self.test_session():49 substr = substr_op.eval()50 self.assertAllEqual(substr, expected_value)51 def _testMatrixStrings(self, dtype):52 test_string = [[b"ten", b"eleven", b"twelve"],53 [b"thirteen", b"fourteen", b"fifteen"],54 [b"sixteen", b"seventeen", b"eighteen"]]55 position = np.array(1, dtype)56 length = np.array(4, dtype)57 expected_value = [[b"en", b"leve", b"welv"], [b"hirt", b"ourt", b"ifte"],58 [b"ixte", b"even", b"ight"]]59 substr_op = string_ops.substr(test_string, position, length)60 with self.test_session():61 substr = substr_op.eval()62 self.assertAllEqual(substr, expected_value)63 def _testElementWisePosLen(self, dtype):64 test_string = [[b"ten", b"eleven", b"twelve"],65 [b"thirteen", b"fourteen", b"fifteen"],66 [b"sixteen", b"seventeen", b"eighteen"]]67 position = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3]], dtype)68 length = np.array([[2, 3, 4], [4, 3, 2], [5, 5, 5]], dtype)69 expected_value = [[b"en", b"eve", b"lve"], [b"hirt", b"urt", b"te"],70 [b"ixtee", b"vente", b"hteen"]]71 substr_op = string_ops.substr(test_string, position, length)72 with self.test_session():73 substr = substr_op.eval()74 self.assertAllEqual(substr, expected_value)75 def _testBroadcast(self, dtype):76 # Broadcast pos/len onto input string77 test_string = [[b"ten", b"eleven", b"twelve"],78 [b"thirteen", b"fourteen", b"fifteen"],79 [b"sixteen", b"seventeen", b"eighteen"],80 [b"nineteen", b"twenty", b"twentyone"]]81 position = np.array([1, 2, 3], dtype)82 length = np.array([1, 2, 3], dtype)83 expected_value = [[b"e", b"ev", b"lve"], [b"h", b"ur", b"tee"],84 [b"i", b"ve", b"hte"], [b"i", b"en", b"nty"]]85 substr_op = string_ops.substr(test_string, position, length)86 with self.test_session():87 substr = substr_op.eval()88 self.assertAllEqual(substr, expected_value)89 # Broadcast input string onto pos/len90 test_string = [b"thirteen", b"fourteen", b"fifteen"]91 position = np.array([[1, 2, 3], [3, 2, 1], [5, 5, 5]], dtype)92 length = np.array([[3, 2, 1], [1, 2, 3], [2, 2, 2]], dtype)93 expected_value = [[b"hir", b"ur", b"t"], [b"r", b"ur", b"ift"],94 [b"ee", b"ee", b"en"]]95 substr_op = string_ops.substr(test_string, position, length)96 with self.test_session():97 substr = substr_op.eval()98 self.assertAllEqual(substr, expected_value)99 # Test 1D broadcast100 test_string = b"thirteen"101 position = np.array([1, 5, 7], dtype)102 length = np.array([3, 2, 1], dtype)103 expected_value = [b"hir", b"ee", b"n"]104 substr_op = string_ops.substr(test_string, position, length)105 with self.test_session():106 substr = substr_op.eval()107 self.assertAllEqual(substr, expected_value)108 def _testBadBroadcast(self, dtype):109 test_string = [[b"ten", b"eleven", b"twelve"],110 [b"thirteen", b"fourteen", b"fifteen"],111 [b"sixteen", b"seventeen", b"eighteen"]]112 position = np.array([1, 2, 3, 4], dtype)113 length = np.array([1, 2, 3, 4], dtype)114 expected_value = [[b"e", b"ev", b"lve"], [b"h", b"ur", b"tee"],115 [b"i", b"ve", b"hte"]]116 with self.assertRaises(ValueError):117 substr_op = string_ops.substr(test_string, position, length)118 def _testOutOfRangeError(self, dtype):119 # Scalar/Scalar120 test_string = b"Hello"121 position = np.array(7, dtype)122 length = np.array(3, dtype)123 substr_op = string_ops.substr(test_string, position, length)124 with self.test_session():125 with self.assertRaises(errors_impl.InvalidArgumentError):126 substr = substr_op.eval()127 # Vector/Scalar128 test_string = [b"good", b"good", b"bad", b"good"]129 position = np.array(4, dtype)130 length = np.array(1, dtype)131 substr_op = string_ops.substr(test_string, position, length)132 with self.test_session():133 with self.assertRaises(errors_impl.InvalidArgumentError):134 substr = substr_op.eval()135 # Negative pos136 test_string = b"Hello"137 position = np.array(-1, dtype)138 length = np.array(3, dtype)139 substr_op = string_ops.substr(test_string, position, length)140 with self.test_session():141 with self.assertRaises(errors_impl.InvalidArgumentError):142 substr = substr_op.eval()143 # Matrix/Matrix144 test_string = [[b"good", b"good", b"good"], [b"good", b"good", b"bad"],145 [b"good", b"good", b"good"]]146 position = np.array([[1, 2, 3], [1, 2, 4], [1, 2, 3]], dtype)147 length = np.array([[3, 2, 1], [1, 2, 3], [2, 2, 2]], dtype)148 substr_op = string_ops.substr(test_string, position, length)149 with self.test_session():150 with self.assertRaises(errors_impl.InvalidArgumentError):151 substr = substr_op.eval()152 # Broadcast153 test_string = [[b"good", b"good", b"good"], [b"good", b"good", b"bad"]]154 position = np.array([1, 2, 4], dtype)155 length = np.array([1, 2, 3], dtype)156 substr_op = string_ops.substr(test_string, position, length)157 with self.test_session():158 with self.assertRaises(errors_impl.InvalidArgumentError):159 substr = substr_op.eval()160 def _testMismatchPosLenShapes(self, dtype):161 test_string = [[b"ten", b"eleven", b"twelve"],162 [b"thirteen", b"fourteen", b"fifteen"],163 [b"sixteen", b"seventeen", b"eighteen"]]164 position = np.array([[1, 2, 3]], dtype)165 length = np.array([2, 3, 4], dtype)166 # Should fail: position/length have different rank167 with self.assertRaises(ValueError):168 substr_op = string_ops.substr(test_string, position, length)169 position = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3]], dtype)170 length = np.array([[2, 3, 4]], dtype)171 # Should fail: position/length have different dimensionality172 with self.assertRaises(ValueError):173 substr_op = string_ops.substr(test_string, position, length)174 def _testAll(self, dtype):175 self._testScalarString(dtype)176 self._testVectorStrings(dtype)177 self._testMatrixStrings(dtype)178 self._testElementWisePosLen(dtype)179 self._testBroadcast(dtype)180 self._testBadBroadcast(dtype)181 self._testOutOfRangeError(dtype)182 self._testMismatchPosLenShapes(dtype)183 def testInt32(self):184 self._testAll(np.int32)185 def testInt64(self):186 self._testAll(np.int64)187 def testWrongDtype(self):188 with self.test_session():189 with self.assertRaises(TypeError):190 string_ops.substr(b"test", 3.0, 1)191 with self.assertRaises(TypeError):192 string_ops.substr(b"test", 3, 1.0)193if __name__ == "__main__":...
errors.py
Source:errors.py
1#!/usr/bin/env python2# ***** BEGIN LICENSE BLOCK *****3# This Source Code Form is subject to the terms of the Mozilla Public4# License, v. 2.0. If a copy of the MPL was not distributed with this file,5# You can obtain one at http://mozilla.org/MPL/2.0/.6# ***** END LICENSE BLOCK *****7"""Generic error lists.8Error lists are used to parse output in mozharness.base.log.OutputParser.9Each line of output is matched against each substring or regular expression10in the error list. On a match, we determine the 'level' of that line,11whether IGNORE, DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL.12TODO: Context lines (requires work on the OutputParser side)13TODO: We could also create classes that generate these, but with the14appropriate level (please don't die on any errors; please die on any15warning; etc.) or platform or language or whatever.16"""17import re18from mozharness.base.log import DEBUG, WARNING, ERROR, CRITICAL, FATAL19# Exceptions20class VCSException(Exception):21 pass22# ErrorLists {{{123BaseErrorList = [{24 'substr': r'''command not found''',25 'level': ERROR26}]27# For ssh, scp, rsync over ssh28SSHErrorList = BaseErrorList + [{29 'substr': r'''Name or service not known''',30 'level': ERROR31}, {32 'substr': r'''Could not resolve hostname''',33 'level': ERROR34}, {35 'substr': r'''POSSIBLE BREAK-IN ATTEMPT''',36 'level': WARNING37}, {38 'substr': r'''Network error:''',39 'level': ERROR40}, {41 'substr': r'''Access denied''',42 'level': ERROR43}, {44 'substr': r'''Authentication refused''',45 'level': ERROR46}, {47 'substr': r'''Out of memory''',48 'level': ERROR49}, {50 'substr': r'''Connection reset by peer''',51 'level': WARNING52}, {53 'substr': r'''Host key verification failed''',54 'level': ERROR55}, {56 'substr': r'''WARNING:''',57 'level': WARNING58}, {59 'substr': r'''rsync error:''',60 'level': ERROR61}, {62 'substr': r'''Broken pipe:''',63 'level': ERROR64}, {65 'substr': r'''Permission denied:''',66 'level': ERROR67}, {68 'substr': r'''connection unexpectedly closed''',69 'level': ERROR70}, {71 'substr': r'''Warning: Identity file''',72 'level': ERROR73}, {74 'substr': r'''command-line line 0: Missing argument''',75 'level': ERROR76}]77HgErrorList = BaseErrorList + [{78 'regex': re.compile(r'''^abort:'''),79 'level': ERROR,80 'explanation': 'Automation Error: hg not responding'81}, {82 'substr': r'''unknown exception encountered''',83 'level': ERROR,84 'explanation': 'Automation Error: python exception in hg'85}, {86 'substr': r'''failed to import extension''',87 'level': WARNING,88 'explanation': 'Automation Error: hg extension missing'89}]90GitErrorList = BaseErrorList + [91 {'substr': r'''Permission denied (publickey).''', 'level': ERROR},92 {'substr': r'''fatal: The remote end hung up unexpectedly''', 'level': ERROR},93 {'substr': r'''does not appear to be a git repository''', 'level': ERROR},94 {'substr': r'''error: src refspec''', 'level': ERROR},95 {'substr': r'''invalid author/committer line -''', 'level': ERROR},96 {'substr': r'''remote: fatal: Error in object''', 'level': ERROR},97 {'substr': r'''fatal: sha1 file '<stdout>' write error: Broken pipe''', 'level': ERROR},98 {'substr': r'''error: failed to push some refs to ''', 'level': ERROR},99 {'substr': r'''remote: error: denying non-fast-forward ''', 'level': ERROR},100 {'substr': r'''! [remote rejected] ''', 'level': ERROR},101 {'regex': re.compile(r'''remote:.*No such file or directory'''), 'level': ERROR},102]103PythonErrorList = BaseErrorList + [104 {'regex': re.compile(r'''Warning:.*Error: '''), 'level': WARNING},105 {'substr': r'''Traceback (most recent call last)''', 'level': ERROR},106 {'substr': r'''SyntaxError: ''', 'level': ERROR},107 {'substr': r'''TypeError: ''', 'level': ERROR},108 {'substr': r'''NameError: ''', 'level': ERROR},109 {'substr': r'''ZeroDivisionError: ''', 'level': ERROR},110 {'regex': re.compile(r'''raise \w*Exception: '''), 'level': CRITICAL},111 {'regex': re.compile(r'''raise \w*Error: '''), 'level': CRITICAL},112]113VirtualenvErrorList = [114 {'substr': r'''not found or a compiler error:''', 'level': WARNING},115 {'regex': re.compile('''\d+: error: '''), 'level': ERROR},116 {'regex': re.compile('''\d+: warning: '''), 'level': WARNING},117 {'regex': re.compile(r'''Downloading .* \(.*\): *([0-9]+%)? *[0-9\.]+[kmKM]b'''), 'level': DEBUG},118] + PythonErrorList119# We may need to have various MakefileErrorLists for differing amounts of120# warning-ignoring-ness.121MakefileErrorList = BaseErrorList + PythonErrorList + [122 {'substr': r'''No rule to make target ''', 'level': ERROR},123 {'regex': re.compile(r'''akefile.*was not found\.'''), 'level': ERROR},124 {'regex': re.compile(r'''Stop\.$'''), 'level': ERROR},125 {'regex': re.compile(r''':\d+: error:'''), 'level': ERROR},126 {'regex': re.compile(r'''make\[\d+\]: \*\*\* \[.*\] Error \d+'''), 'level': ERROR},127 {'regex': re.compile(r''':\d+: warning:'''), 'level': WARNING},128 {'regex': re.compile(r'''make(?:\[\d+\])?: \*\*\*/'''), 'level': ERROR},129 {'substr': r'''Warning: ''', 'level': WARNING},130]131TarErrorList = BaseErrorList + [132 {'substr': r'''(stdin) is not a bzip2 file.''', 'level': ERROR},133 {'regex': re.compile(r'''Child returned status [1-9]'''), 'level': ERROR},134 {'substr': r'''Error exit delayed from previous errors''', 'level': ERROR},135 {'substr': r'''stdin: unexpected end of file''', 'level': ERROR},136 {'substr': r'''stdin: not in gzip format''', 'level': ERROR},137 {'substr': r'''Cannot exec: No such file or directory''', 'level': ERROR},138 {'substr': r''': Error is not recoverable: exiting now''', 'level': ERROR},139]140ADBErrorList = BaseErrorList + [141 {'substr': r'''INSTALL_FAILED_''', 'level': ERROR},142 {'substr': r'''Android Debug Bridge version''', 'level': ERROR},143 {'substr': r'''error: protocol fault''', 'level': ERROR},144 {'substr': r'''unable to connect to ''', 'level': ERROR},145]146JarsignerErrorList = [{147 'substr': r'''command not found''',148 'level': FATAL149}, {150 'substr': r'''jarsigner error: java.lang.RuntimeException: keystore load: Keystore was tampered with, or password was incorrect''',151 'level': FATAL,152 'explanation': r'''The store passphrase is probably incorrect!''',153}, {154 'regex': re.compile(r'''jarsigner: key associated with .* not a private key'''),155 'level': FATAL,156 'explanation': r'''The key passphrase is probably incorrect!''',157}, {158 'regex': re.compile(r'''jarsigner error: java.lang.RuntimeException: keystore load: .* .No such file or directory'''),159 'level': FATAL,160 'explanation': r'''The keystore doesn't exist!''',161}, {162 'substr': r'''jarsigner: unable to open jar file:''',163 'level': FATAL,164 'explanation': r'''The apk is missing!''',165}]166ZipErrorList = BaseErrorList + [{167 'substr': r'''zip warning:''',168 'level': WARNING,169}, {170 'substr': r'''zip error:''',171 'level': ERROR,172}, {173 'substr': r'''Cannot open file: it does not appear to be a valid archive''',174 'level': ERROR,175}]176ZipalignErrorList = BaseErrorList + [{177 'regex': re.compile(r'''Unable to open .* as a zip archive'''),178 'level': ERROR,179}, {180 'regex': re.compile(r'''Output file .* exists'''),181 'level': ERROR,182}, {183 'substr': r'''Input and output can't be the same file''',184 'level': ERROR,185}]186# __main__ {{{1187if __name__ == '__main__':188 '''TODO: unit tests.189 '''...
read_ner.py
Source:read_ner.py
1from __future__ import unicode_literals2import os3from os import path4import re5def split(text):6 """Split an annotation file by sentence. Each sentence's annotation should7 be a single string."""8 return text.strip().split('\n')[1:-1]9 10def parse(string, strip_bad_periods=False):11 """Given a sentence's annotation string, return a list of word strings,12 and a list of named entities, where each entity is a (start, end, label)13 triple."""14 tokens = []15 tags = []16 open_tag = None17 # Arbitrary corrections to promote alignment, and ensure that entities18 # begin at a space. This allows us to treat entities as tokens, making it19 # easier to return the list of entities.20 string = string.replace('... .', '...')21 string = string.replace('U.S.</ENAMEX> .', 'U.S.</ENAMEX>')22 string = string.replace('Co.</ENAMEX> .', 'Co.</ENAMEX>')23 string = string.replace('U.S. .', 'U.S.')24 string = string.replace('<ENAMEX ', '<ENAMEX')25 string = string.replace(' E_OFF="', 'E_OFF="')26 string = string.replace(' S_OFF="', 'S_OFF="')27 string = string.replace('units</ENAMEX>-<ENAMEX', 'units</ENAMEX> - <ENAMEX')28 string = string.replace('<ENAMEXTYPE="PERSON"E_OFF="1">Paula</ENAMEX> Zahn', 'Paula Zahn')29 string = string.replace('<ENAMEXTYPE="CARDINAL"><ENAMEXTYPE="CARDINAL">little</ENAMEX> drain</ENAMEX>', 'little drain')30 for substr in string.strip().split():31 substr = _fix_inner_entities(substr)32 tokens.append(_get_text(substr))33 try:34 tag, open_tag = _get_tag(substr, open_tag)35 except:36 raise37 tags.append(tag)38 return tokens, tags39tag_re = re.compile(r'<ENAMEXTYPE="[^"]+">')40def _fix_inner_entities(substr):41 tags = tag_re.findall(substr)42 if '</ENAMEX' in substr and not substr.endswith('</ENAMEX'):43 substr = substr.replace('</ENAMEX>', '') + '</ENAMEX>'44 if tags:45 substr = tag_re.sub('', substr)46 return tags[0] + substr47 else:48 return substr49def _get_tag(substr, tag):50 if substr.startswith('<'):51 tag = substr.split('"')[1]52 if substr.endswith('>'):53 return 'U-' + tag, None54 else:55 return 'B-%s' % tag, tag56 elif substr.endswith('>'):57 return 'L-' + tag, None58 elif tag is not None:59 return 'I-' + tag, tag60 else:61 return 'O', None62def _get_text(substr):63 if substr.startswith('<'):64 substr = substr.split('>', 1)[1]65 if substr.endswith('>'):66 substr = substr.split('<')[0]67 return reform_string(substr)68def tags_to_entities(tags):69 entities = []70 start = None71 for i, tag in enumerate(tags):72 if tag.startswith('O'):73 # TODO: We shouldn't be getting these malformed inputs. Fix this.74 if start is not None:75 start = None76 continue77 elif tag == '-':78 continue79 elif tag.startswith('I'):80 assert start is not None, tags[:i]81 continue82 if tag.startswith('U'):83 entities.append((tag[2:], i, i))84 elif tag.startswith('B'):85 start = i86 elif tag.startswith('L'):87 entities.append((tag[2:], start, i))88 start = None89 else:90 raise Exception(tag)91 return entities92def reform_string(tok):93 tok = tok.replace("``", '"')94 tok = tok.replace("`", "'")95 tok = tok.replace("''", '"')96 tok = tok.replace('\\', '')97 tok = tok.replace('-LCB-', '{')98 tok = tok.replace('-RCB-', '}')99 tok = tok.replace('-RRB-', ')')100 tok = tok.replace('-LRB-', '(')101 tok = tok.replace("'T-", "'T")102 tok = tok.replace('-AMP-', '&')...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run Lemoncheesecake automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
