Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More
How to use test_base_url method in Selene
Best Python code snippet using selene_python
test_tree.py
Source:test_tree.py 
1import datetime2import difflib3import textwrap4from decimal import Decimal5from email.utils import format_datetime6from unittest import TestCase7import requests_mock8from dateutil.tz import tzoffset9from tests.helpers import gzip10from usp.log import create_logger11from usp.objects.page import (12 SitemapPage,13 SitemapNewsStory,14 SitemapPageChangeFrequency,15)16from usp.objects.sitemap import (17 IndexRobotsTxtSitemap,18 PagesXMLSitemap,19 IndexXMLSitemap,20 InvalidSitemap,21 PagesTextSitemap,22 IndexWebsiteSitemap,23 PagesRSSSitemap,24 PagesAtomSitemap,25)26from usp.tree import sitemap_tree_for_homepage27# FIXME various exotic properties28# FIXME XML vulnerabilities with Expat29# FIXME max. recursion level30# FIXME tests responses that are too big31log = create_logger(__name__)32class TestSitemapTree(TestCase):33 TEST_BASE_URL = 'http://test_ultimate-sitemap-parser.com' # mocked by HTTPretty34 # Publication / "last modified" date35 TEST_DATE_DATETIME = datetime.datetime(36 year=2009, month=12, day=17, hour=12, minute=4, second=56,37 tzinfo=tzoffset(None, 7200),38 )39 TEST_DATE_STR_ISO8601 = TEST_DATE_DATETIME.isoformat()40 """Test string date formatted as ISO 8601 (for XML and Atom 0.3 / 1.0 sitemaps)."""41 TEST_DATE_STR_RFC2822 = format_datetime(TEST_DATE_DATETIME)42 """Test string date formatted as RFC 2822 (for RSS 2.0 sitemaps)."""43 TEST_PUBLICATION_NAME = 'Test publication'44 TEST_PUBLICATION_LANGUAGE = 'en'45 @staticmethod46 def fallback_to_404_not_found_matcher(request):47 """Reply with "404 Not Found" to unmatched URLs instead of throwing NoMockAddress."""48 return requests_mock.create_response(49 request,50 status_code=404,51 reason='Not Found',52 headers={'Content-Type': 'text/html'},53 text="<h1>404 Not Found!</h1>",54 )55 # noinspection DuplicatedCode56 def test_sitemap_tree_for_homepage(self):57 """Test sitemap_tree_for_homepage()."""58 with requests_mock.Mocker() as m:59 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)60 m.get(61 self.TEST_BASE_URL + '/',62 text='This is a homepage.',63 )64 m.get(65 self.TEST_BASE_URL + '/robots.txt',66 headers={'Content-Type': 'text/plain'},67 text=textwrap.dedent("""68 User-agent: *69 Disallow: /whatever70 71 Sitemap: {base_url}/sitemap_pages.xml72 73 # Intentionally spelled as "Site-map" as Google tolerates this:74 # https://github.com/google/robotstxt/blob/master/robots.cc#L703 75 Site-map: {base_url}/sitemap_news_index_1.xml76 """.format(base_url=self.TEST_BASE_URL)).strip(),77 )78 # One sitemap for random static pages79 m.get(80 self.TEST_BASE_URL + '/sitemap_pages.xml',81 headers={'Content-Type': 'application/xml'},82 text=textwrap.dedent("""83 <?xml version="1.0" encoding="UTF-8"?>84 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">85 <url>86 <loc>{base_url}/about.html</loc>87 <lastmod>{last_modified_date}</lastmod>88 <changefreq>monthly</changefreq>89 <priority>0.8</priority>90 </url>91 <url>92 <loc>{base_url}/contact.html</loc>93 <lastmod>{last_modified_date}</lastmod>94 95 <!-- Invalid change frequency -->96 <changefreq>when we feel like it</changefreq>97 98 <!-- Invalid priority -->99 <priority>1.1</priority>100 101 </url>102 </urlset>103 """.format(base_url=self.TEST_BASE_URL, last_modified_date=self.TEST_DATE_STR_ISO8601)).strip(),104 )105 # Index sitemap pointing to sitemaps with stories106 m.get(107 self.TEST_BASE_URL + '/sitemap_news_index_1.xml',108 headers={'Content-Type': 'application/xml'},109 text=textwrap.dedent("""110 <?xml version="1.0" encoding="UTF-8"?>111 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">112 <sitemap>113 <loc>{base_url}/sitemap_news_1.xml</loc>114 <lastmod>{last_modified}</lastmod>115 </sitemap>116 <sitemap>117 <loc>{base_url}/sitemap_news_index_2.xml</loc>118 <lastmod>{last_modified}</lastmod>119 </sitemap>120 </sitemapindex>121 """.format(base_url=self.TEST_BASE_URL, last_modified=self.TEST_DATE_STR_ISO8601)).strip(),122 )123 # First sitemap with actual stories124 m.get(125 self.TEST_BASE_URL + '/sitemap_news_1.xml',126 headers={'Content-Type': 'application/xml'},127 text=textwrap.dedent("""128 <?xml version="1.0" encoding="UTF-8"?>129 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"130 xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"131 xmlns:xhtml="http://www.w3.org/1999/xhtml">132 133 <url>134 <loc>{base_url}/news/foo.html</loc>135 136 <!-- Element present but empty -->137 <lastmod />138 139 <!-- Some other XML namespace -->140 <xhtml:link rel="alternate"141 media="only screen and (max-width: 640px)"142 href="{base_url}/news/foo.html?mobile=1" />143 144 <news:news>145 <news:publication>146 <news:name>{publication_name}</news:name>147 <news:language>{publication_language}</news:language>148 </news:publication>149 <news:publication_date>{publication_date}</news:publication_date>150 <news:title>Foo <foo></news:title> <!-- HTML entity decoding -->151 </news:news>152 </url>153 154 <!-- Has a duplicate story in /sitemap_news_2.xml -->155 <url>156 <loc>{base_url}/news/bar.html</loc>157 <xhtml:link rel="alternate"158 media="only screen and (max-width: 640px)"159 href="{base_url}/news/bar.html?mobile=1" />160 <news:news>161 <news:publication>162 <news:name>{publication_name}</news:name>163 <news:language>{publication_language}</news:language>164 </news:publication>165 <news:publication_date>{publication_date}</news:publication_date>166 <news:title>Bar & bar</news:title>167 </news:news>168 </url>169 170 </urlset>171 """.format(172 base_url=self.TEST_BASE_URL,173 publication_name=self.TEST_PUBLICATION_NAME,174 publication_language=self.TEST_PUBLICATION_LANGUAGE,175 publication_date=self.TEST_DATE_STR_ISO8601,176 )).strip(),177 )178 # Another index sitemap pointing to a second sitemaps with stories179 m.get(180 self.TEST_BASE_URL + '/sitemap_news_index_2.xml',181 headers={'Content-Type': 'application/xml'},182 text=textwrap.dedent("""183 <?xml version="1.0" encoding="UTF-8"?>184 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">185 186 <sitemap>187 <!-- Extra whitespace added around URL -->188 <loc> {base_url}/sitemap_news_2.xml </loc>189 <lastmod>{last_modified}</lastmod>190 </sitemap>191 192 <!-- Nonexistent sitemap -->193 <sitemap>194 <loc>{base_url}/sitemap_news_missing.xml</loc>195 <lastmod>{last_modified}</lastmod>196 </sitemap>197 198 </sitemapindex>199 """.format(base_url=self.TEST_BASE_URL, last_modified=self.TEST_DATE_STR_ISO8601)).strip(),200 )201 # Second sitemap with actual stories202 m.get(203 self.TEST_BASE_URL + '/sitemap_news_2.xml',204 headers={'Content-Type': 'application/xml'},205 text=textwrap.dedent("""206 <?xml version="1.0" encoding="UTF-8"?>207 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"208 xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"209 xmlns:xhtml="http://www.w3.org/1999/xhtml">210 211 <!-- Has a duplicate story in /sitemap_news_1.xml -->212 <url>213 <!-- Extra whitespace added around URL -->214 <loc> {base_url}/news/bar.html </loc>215 <xhtml:link rel="alternate"216 media="only screen and (max-width: 640px)"217 href="{base_url}/news/bar.html?mobile=1#fragment_is_to_be_removed" />218 <news:news>219 <news:publication>220 <news:name>{publication_name}</news:name>221 <news:language>{publication_language}</news:language>222 </news:publication>223 <news:publication_date>{publication_date}</news:publication_date>224 225 <tag_without_inner_character_data name="value" />226 227 <news:title>Bar & bar</news:title>228 </news:news>229 </url>230 231 <url>232 <loc>{base_url}/news/baz.html</loc>233 <xhtml:link rel="alternate"234 media="only screen and (max-width: 640px)"235 href="{base_url}/news/baz.html?mobile=1" />236 <news:news>237 <news:publication>238 <news:name>{publication_name}</news:name>239 <news:language>{publication_language}</news:language>240 </news:publication>241 <news:publication_date>{publication_date}</news:publication_date>242 <news:title><![CDATA[BÄ
ž]]></news:title> <!-- CDATA and UTF-8 -->243 </news:news>244 </url>245 246 </urlset>247 """.format(248 base_url=self.TEST_BASE_URL,249 publication_name=self.TEST_PUBLICATION_NAME,250 publication_language=self.TEST_PUBLICATION_LANGUAGE,251 publication_date=self.TEST_DATE_STR_ISO8601,252 )).strip(),253 )254 # Nonexistent sitemap255 m.get(256 self.TEST_BASE_URL + '/sitemap_news_missing.xml',257 status_code=404,258 reason='Not Found',259 headers={'Content-Type': 'text/html'},260 text="<h1>404 Not Found!</h1>",261 )262 expected_sitemap_tree = IndexWebsiteSitemap(263 url='{}/'.format(self.TEST_BASE_URL),264 sub_sitemaps=[265 IndexRobotsTxtSitemap(266 url='{}/robots.txt'.format(self.TEST_BASE_URL),267 sub_sitemaps=[268 PagesXMLSitemap(269 url='{}/sitemap_pages.xml'.format(self.TEST_BASE_URL),270 pages=[271 SitemapPage(272 url='{}/about.html'.format(self.TEST_BASE_URL),273 last_modified=self.TEST_DATE_DATETIME,274 news_story=None,275 change_frequency=SitemapPageChangeFrequency.MONTHLY,276 priority=Decimal('0.8'),277 ),278 SitemapPage(279 url='{}/contact.html'.format(self.TEST_BASE_URL),280 last_modified=self.TEST_DATE_DATETIME,281 news_story=None,282 # Invalid input -- should be reset to "always"283 change_frequency=SitemapPageChangeFrequency.ALWAYS,284 # Invalid input -- should be reset to 0.5 (the default as per the spec)285 priority=Decimal('0.5'),286 )287 ],288 ),289 IndexXMLSitemap(290 url='{}/sitemap_news_index_1.xml'.format(self.TEST_BASE_URL),291 sub_sitemaps=[292 PagesXMLSitemap(293 url='{}/sitemap_news_1.xml'.format(self.TEST_BASE_URL),294 pages=[295 SitemapPage(296 url='{}/news/foo.html'.format(self.TEST_BASE_URL),297 news_story=SitemapNewsStory(298 title='Foo <foo>',299 publish_date=self.TEST_DATE_DATETIME,300 publication_name=self.TEST_PUBLICATION_NAME,301 publication_language=self.TEST_PUBLICATION_LANGUAGE,302 ),303 ),304 SitemapPage(305 url='{}/news/bar.html'.format(self.TEST_BASE_URL),306 news_story=SitemapNewsStory(307 title='Bar & bar',308 publish_date=self.TEST_DATE_DATETIME,309 publication_name=self.TEST_PUBLICATION_NAME,310 publication_language=self.TEST_PUBLICATION_LANGUAGE,311 ),312 ),313 ]314 ),315 IndexXMLSitemap(316 url='{}/sitemap_news_index_2.xml'.format(self.TEST_BASE_URL),317 sub_sitemaps=[318 PagesXMLSitemap(319 url='{}/sitemap_news_2.xml'.format(self.TEST_BASE_URL),320 pages=[321 SitemapPage(322 url='{}/news/bar.html'.format(self.TEST_BASE_URL),323 news_story=SitemapNewsStory(324 title='Bar & bar',325 publish_date=self.TEST_DATE_DATETIME,326 publication_name=self.TEST_PUBLICATION_NAME,327 publication_language=self.TEST_PUBLICATION_LANGUAGE,328 ),329 ),330 SitemapPage(331 url='{}/news/baz.html'.format(self.TEST_BASE_URL),332 news_story=SitemapNewsStory(333 title='BÄ
ž',334 publish_date=self.TEST_DATE_DATETIME,335 publication_name=self.TEST_PUBLICATION_NAME,336 publication_language=self.TEST_PUBLICATION_LANGUAGE,337 ),338 ),339 ],340 ),341 InvalidSitemap(342 url='{}/sitemap_news_missing.xml'.format(self.TEST_BASE_URL),343 reason=(344 'Unable to fetch sitemap from {base_url}/sitemap_news_missing.xml: '345 '404 Not Found'346 ).format(base_url=self.TEST_BASE_URL),347 ),348 ],349 ),350 ],351 ),352 ],353 )354 ]355 )356 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)357 expected_lines = str(expected_sitemap_tree).split()358 actual_lines = str(actual_sitemap_tree).split()359 diff = difflib.ndiff(expected_lines, actual_lines)360 diff_str = '\n'.join(diff)361 assert expected_sitemap_tree == actual_sitemap_tree, diff_str362 assert len(list(actual_sitemap_tree.all_pages())) == 6363 def test_sitemap_tree_for_homepage_gzip(self):364 """Test sitemap_tree_for_homepage() with gzipped sitemaps."""365 with requests_mock.Mocker() as m:366 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)367 m.get(368 self.TEST_BASE_URL + '/',369 text='This is a homepage.',370 )371 m.get(372 self.TEST_BASE_URL + '/robots.txt',373 headers={'Content-Type': 'text/plain'},374 text=textwrap.dedent("""375 User-agent: *376 Disallow: /whatever377 378 Sitemap: {base_url}/sitemap_1.gz379 Sitemap: {base_url}/sitemap_2.dat380 Sitemap: {base_url}/sitemap_3.xml.gz381 """.format(base_url=self.TEST_BASE_URL)).strip(),382 )383 # Gzipped sitemap without correct HTTP header but with .gz extension384 m.get(385 self.TEST_BASE_URL + '/sitemap_1.gz',386 content=gzip(textwrap.dedent("""387 <?xml version="1.0" encoding="UTF-8"?>388 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"389 xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">390 <url>391 <loc>{base_url}/news/foo.html</loc>392 <news:news>393 <news:publication>394 <news:name>{publication_name}</news:name>395 <news:language>{publication_language}</news:language>396 </news:publication>397 <news:publication_date>{publication_date}</news:publication_date>398 <news:title>Foo <foo></news:title> <!-- HTML entity decoding -->399 </news:news>400 </url>401 </urlset>402 """.format(403 base_url=self.TEST_BASE_URL,404 publication_name=self.TEST_PUBLICATION_NAME,405 publication_language=self.TEST_PUBLICATION_LANGUAGE,406 publication_date=self.TEST_DATE_STR_ISO8601,407 )).strip()),408 )409 # Gzipped sitemap with correct HTTP header but without .gz extension410 m.get(411 self.TEST_BASE_URL + '/sitemap_2.dat',412 headers={'Content-Type': 'application/x-gzip'},413 content=gzip(textwrap.dedent("""414 <?xml version="1.0" encoding="UTF-8"?>415 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"416 xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">417 <url>418 <loc>{base_url}/news/bar.html</loc>419 <news:news>420 <news:publication>421 <news:name>{publication_name}</news:name>422 <news:language>{publication_language}</news:language>423 </news:publication>424 <news:publication_date>{publication_date}</news:publication_date>425 <news:title><![CDATA[BÄ
r]]></news:title> <!-- CDATA and UTF-8 -->426 </news:news>427 </url>428 </urlset>429 """.format(430 base_url=self.TEST_BASE_URL,431 publication_name=self.TEST_PUBLICATION_NAME,432 publication_language=self.TEST_PUBLICATION_LANGUAGE,433 publication_date=self.TEST_DATE_STR_ISO8601,434 )).strip()),435 )436 # Sitemap which appears to be gzipped (due to extension and Content-Type) but really isn't437 m.get(438 self.TEST_BASE_URL + '/sitemap_3.xml.gz',439 headers={'Content-Type': 'application/x-gzip'},440 text=textwrap.dedent("""441 <?xml version="1.0" encoding="UTF-8"?>442 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"443 xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">444 <url>445 <loc>{base_url}/news/baz.html</loc>446 <news:news>447 <news:publication>448 <news:name>{publication_name}</news:name>449 <news:language>{publication_language}</news:language>450 </news:publication>451 <news:publication_date>{publication_date}</news:publication_date>452 <news:title><![CDATA[BÄ
ž]]></news:title> <!-- CDATA and UTF-8 -->453 </news:news>454 </url>455 </urlset>456 """.format(457 base_url=self.TEST_BASE_URL,458 publication_name=self.TEST_PUBLICATION_NAME,459 publication_language=self.TEST_PUBLICATION_LANGUAGE,460 publication_date=self.TEST_DATE_STR_ISO8601,461 )).strip(),462 )463 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)464 # Don't do an in-depth check, we just need to make sure that gunzip works465 assert isinstance(actual_sitemap_tree, IndexWebsiteSitemap)466 assert len(actual_sitemap_tree.sub_sitemaps) == 1467 assert isinstance(actual_sitemap_tree.sub_sitemaps[0], IndexRobotsTxtSitemap)468 # noinspection PyUnresolvedReferences469 assert len(actual_sitemap_tree.sub_sitemaps[0].sub_sitemaps) == 3470 # noinspection PyUnresolvedReferences471 sitemap_1 = actual_sitemap_tree.sub_sitemaps[0].sub_sitemaps[0]472 assert isinstance(sitemap_1, PagesXMLSitemap)473 assert len(sitemap_1.pages) == 1474 # noinspection PyUnresolvedReferences475 sitemap_2 = actual_sitemap_tree.sub_sitemaps[0].sub_sitemaps[1]476 assert isinstance(sitemap_2, PagesXMLSitemap)477 assert len(sitemap_2.pages) == 1478 # noinspection PyUnresolvedReferences479 sitemap_3 = actual_sitemap_tree.sub_sitemaps[0].sub_sitemaps[2]480 assert isinstance(sitemap_3, PagesXMLSitemap)481 assert len(sitemap_3.pages) == 1482 def test_sitemap_tree_for_homepage_plain_text(self):483 """Test sitemap_tree_for_homepage() with plain text sitemaps."""484 with requests_mock.Mocker() as m:485 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)486 m.get(487 self.TEST_BASE_URL + '/',488 text='This is a homepage.',489 )490 m.get(491 self.TEST_BASE_URL + '/robots.txt',492 headers={'Content-Type': 'text/plain'},493 text=textwrap.dedent("""494 User-agent: *495 Disallow: /whatever496 497 Sitemap: {base_url}/sitemap_1.txt498 Sitemap: {base_url}/sitemap_2.txt.dat499 """.format(base_url=self.TEST_BASE_URL)).strip(),500 )501 # Plain text uncompressed sitemap (no Content-Type header)502 m.get(503 self.TEST_BASE_URL + '/sitemap_1.txt',504 text=textwrap.dedent("""505 506 {base_url}/news/foo.html507 508 509 {base_url}/news/bar.html510 511 Some other stuff which totally doesn't look like an URL512 """.format(base_url=self.TEST_BASE_URL)).strip(),513 )514 # Plain text compressed sitemap without .gz extension515 m.get(516 self.TEST_BASE_URL + '/sitemap_2.txt.dat',517 headers={'Content-Type': 'application/x-gzip'},518 content=gzip(textwrap.dedent("""519 {base_url}/news/bar.html520 {base_url}/news/baz.html521 """.format(base_url=self.TEST_BASE_URL)).strip()),522 )523 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)524 assert isinstance(actual_sitemap_tree, IndexWebsiteSitemap)525 assert len(actual_sitemap_tree.sub_sitemaps) == 1526 assert isinstance(actual_sitemap_tree.sub_sitemaps[0], IndexRobotsTxtSitemap)527 # noinspection PyUnresolvedReferences528 assert len(actual_sitemap_tree.sub_sitemaps[0].sub_sitemaps) == 2529 # noinspection PyUnresolvedReferences530 sitemap_1 = actual_sitemap_tree.sub_sitemaps[0].sub_sitemaps[0]531 assert isinstance(sitemap_1, PagesTextSitemap)532 assert len(sitemap_1.pages) == 2533 # noinspection PyUnresolvedReferences534 sitemap_2 = actual_sitemap_tree.sub_sitemaps[0].sub_sitemaps[1]535 assert isinstance(sitemap_2, PagesTextSitemap)536 assert len(sitemap_2.pages) == 2537 pages = list(actual_sitemap_tree.all_pages())538 assert len(pages) == 4539 assert SitemapPage(url='{}/news/foo.html'.format(self.TEST_BASE_URL)) in pages540 assert SitemapPage(url='{}/news/bar.html'.format(self.TEST_BASE_URL)) in pages541 assert SitemapPage(url='{}/news/baz.html'.format(self.TEST_BASE_URL)) in pages542 # noinspection DuplicatedCode543 def test_sitemap_tree_for_homepage_rss_atom(self):544 """Test sitemap_tree_for_homepage() with RSS 2.0 / Atom 0.3 / Atom 1.0 feeds."""545 with requests_mock.Mocker() as m:546 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)547 m.get(548 self.TEST_BASE_URL + '/',549 text='This is a homepage.',550 )551 m.get(552 self.TEST_BASE_URL + '/robots.txt',553 headers={'Content-Type': 'text/plain'},554 text=textwrap.dedent("""555 User-agent: *556 Disallow: /whatever557 Sitemap: {base_url}/sitemap_rss.xml558 Sitemap: {base_url}/sitemap_atom_0_3.xml559 Sitemap: {base_url}/sitemap_atom_1_0.xml560 """.format(base_url=self.TEST_BASE_URL)).strip(),561 )562 # RSS 2.0 sitemap563 m.get(564 self.TEST_BASE_URL + '/sitemap_rss.xml',565 headers={'Content-Type': 'application/rss+xml'},566 text=textwrap.dedent("""567 <?xml version="1.0" encoding="UTF-8"?>568 <rss version="2.0">569 <channel>570 <title>Test RSS 2.0 feed</title>571 <description>This is a test RSS 2.0 feed.</description>572 <link>{base_url}</link>573 <pubDate>{pub_date}</pubDate>574 <item>575 <title>Test RSS 2.0 story #1</title>576 <description>This is a test RSS 2.0 story #1.</description>577 <link>{base_url}/rss_story_1.html</link>578 <guid isPermaLink="true">{base_url}/rss_story_1.html</guid>579 <pubDate>{pub_date}</pubDate>580 </item>581 <item>582 <title>Test RSS 2.0 story #2</title>583 <description>This is a test RSS 2.0 story #2.</description>584 <link>{base_url}/rss_story_2.html</link>585 <guid isPermaLink="true">{base_url}/rss_story_2.html</guid>586 <pubDate>{pub_date}</pubDate>587 </item>588 </channel>589 </rss>590 """.format(base_url=self.TEST_BASE_URL, pub_date=self.TEST_DATE_STR_RFC2822)).strip(),591 )592 # Atom 0.3 sitemap593 m.get(594 self.TEST_BASE_URL + '/sitemap_atom_0_3.xml',595 headers={'Content-Type': 'application/atom+xml'},596 text=textwrap.dedent("""597 <?xml version="1.0" encoding="UTF-8"?>598 <feed version="0.3" xmlns="http://purl.org/atom/ns#">599 <title>Test Atom 0.3 feed</title>600 <link rel="alternate" type="text/html" href="{base_url}" />601 <modified>{pub_date}</modified>602 <entry>603 <title>Test Atom 0.3 story #1</title>604 <link rel="alternate" type="text/html" href="{base_url}/atom_0_3_story_1.html" />605 <id>{base_url}/atom_0_3_story_1.html</id>606 <issued>{pub_date}</issued>607 </entry>608 <entry>609 <title>Test Atom 0.3 story #2</title>610 <link rel="alternate" type="text/html" href="{base_url}/atom_0_3_story_2.html" />611 <id>{base_url}/atom_0_3_story_2.html</id>612 <issued>{pub_date}</issued>613 </entry>614 </feed>615 """.format(base_url=self.TEST_BASE_URL, pub_date=self.TEST_DATE_STR_ISO8601)).strip(),616 )617 # Atom 1.0 sitemap618 m.get(619 self.TEST_BASE_URL + '/sitemap_atom_1_0.xml',620 headers={'Content-Type': 'application/atom+xml'},621 text=textwrap.dedent("""622 <?xml version="1.0" encoding="UTF-8"?>623 <feed xmlns="http://www.w3.org/2005/Atom">624 <title>Test Atom 1.0 feed</title>625 <subtitle>This is a test Atom 1.0 feed.</subtitle>626 <link href="{base_url}/sitemap_atom_1_0.xml" rel="self" />627 <link href="{base_url}" />628 <id>{base_url}</id>629 <updated>{pub_date}</updated>630 <entry>631 <title>Test Atom 1.0 story #1</title>632 <link href="{base_url}/atom_1_0_story_1.html" />633 <link rel="alternate" type="text/html" href="{base_url}/atom_1_0_story_1.html?alt" />634 <link rel="edit" href="{base_url}/atom_1_0_story_1.html?edit" />635 <id>{base_url}/atom_1_0_story_1.html</id>636 <updated>{pub_date}</updated>637 <summary>This is test atom 1.0 story #1.</summary>638 <content type="xhtml">639 <div xmlns="http://www.w3.org/1999/xhtml">640 <p>This is test atom 1.0 story #1.</p>641 </div>642 </content>643 <author>644 <name>John Doe</name>645 <email>johndoe@example.com</email>646 </author>647 </entry>648 <entry>649 <title>Test Atom 1.0 story #2</title>650 <link href="{base_url}/atom_1_0_story_2.html" />651 <link rel="alternate" type="text/html" href="{base_url}/atom_1_0_story_2.html?alt" />652 <link rel="edit" href="{base_url}/atom_1_0_story_2.html?edit" />653 <id>{base_url}/atom_1_0_story_2.html</id>654 <updated>{pub_date}</updated>655 <summary>This is test atom 1.0 story #2.</summary>656 <content type="xhtml">657 <div xmlns="http://www.w3.org/1999/xhtml">658 <p>This is test atom 1.0 story #2.</p>659 </div>660 </content>661 <author>662 <name>John Doe</name>663 <email>johndoe@example.com</email>664 </author>665 </entry>666 </feed>667 """.format(base_url=self.TEST_BASE_URL, pub_date=self.TEST_DATE_STR_ISO8601)).strip(),668 )669 expected_sitemap_tree = IndexWebsiteSitemap(670 url='{}/'.format(self.TEST_BASE_URL),671 sub_sitemaps=[672 IndexRobotsTxtSitemap(673 url='{}/robots.txt'.format(self.TEST_BASE_URL),674 sub_sitemaps=[675 PagesRSSSitemap(676 url='{}/sitemap_rss.xml'.format(self.TEST_BASE_URL),677 pages=[678 SitemapPage(679 url='{}/rss_story_1.html'.format(self.TEST_BASE_URL),680 news_story=SitemapNewsStory(681 title='Test RSS 2.0 story #1',682 publish_date=self.TEST_DATE_DATETIME,683 ),684 ),685 SitemapPage(686 url='{}/rss_story_2.html'.format(self.TEST_BASE_URL),687 news_story=SitemapNewsStory(688 title='Test RSS 2.0 story #2',689 publish_date=self.TEST_DATE_DATETIME,690 )691 )692 ]693 ),694 PagesAtomSitemap(695 url='{}/sitemap_atom_0_3.xml'.format(self.TEST_BASE_URL),696 pages=[697 SitemapPage(698 url='{}/atom_0_3_story_1.html'.format(self.TEST_BASE_URL),699 news_story=SitemapNewsStory(700 title='Test Atom 0.3 story #1',701 publish_date=self.TEST_DATE_DATETIME,702 ),703 ),704 SitemapPage(705 url='{}/atom_0_3_story_2.html'.format(self.TEST_BASE_URL),706 news_story=SitemapNewsStory(707 title='Test Atom 0.3 story #2',708 publish_date=self.TEST_DATE_DATETIME,709 )710 )711 ]712 ),713 PagesAtomSitemap(714 url='{}/sitemap_atom_1_0.xml'.format(self.TEST_BASE_URL),715 pages=[716 SitemapPage(717 url='{}/atom_1_0_story_1.html'.format(self.TEST_BASE_URL),718 news_story=SitemapNewsStory(719 title='Test Atom 1.0 story #1',720 publish_date=self.TEST_DATE_DATETIME,721 ),722 ),723 SitemapPage(724 url='{}/atom_1_0_story_2.html'.format(self.TEST_BASE_URL),725 news_story=SitemapNewsStory(726 title='Test Atom 1.0 story #2',727 publish_date=self.TEST_DATE_DATETIME,728 )729 )730 ]731 ),732 ]733 )734 ]735 )736 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)737 expected_lines = str(expected_sitemap_tree).split()738 actual_lines = str(actual_sitemap_tree).split()739 diff = difflib.ndiff(expected_lines, actual_lines)740 diff_str = '\n'.join(diff)741 assert expected_sitemap_tree == actual_sitemap_tree, diff_str742 assert len(list(actual_sitemap_tree.all_pages())) == 6743 def test_sitemap_tree_for_homepage_rss_atom_empty(self):744 """Test sitemap_tree_for_homepage() with empty RSS 2.0 / Atom 0.3 / Atom 1.0 feeds."""745 with requests_mock.Mocker() as m:746 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)747 m.get(748 self.TEST_BASE_URL + '/',749 text='This is a homepage.',750 )751 m.get(752 self.TEST_BASE_URL + '/robots.txt',753 headers={'Content-Type': 'text/plain'},754 text=textwrap.dedent("""755 User-agent: *756 Disallow: /whatever757 Sitemap: {base_url}/sitemap_rss.xml758 Sitemap: {base_url}/sitemap_atom_0_3.xml759 Sitemap: {base_url}/sitemap_atom_1_0.xml760 """.format(base_url=self.TEST_BASE_URL)).strip(),761 )762 # RSS 2.0 sitemap763 m.get(764 self.TEST_BASE_URL + '/sitemap_rss.xml',765 headers={'Content-Type': 'application/rss+xml'},766 text=textwrap.dedent("""767 <?xml version="1.0" encoding="UTF-8"?>768 <rss version="2.0">769 <channel>770 <title>Test RSS 2.0 feed</title>771 <description>This is a test RSS 2.0 feed.</description>772 <link>{base_url}</link>773 <pubDate>{pub_date}</pubDate>774 </channel>775 </rss>776 """.format(base_url=self.TEST_BASE_URL, pub_date=self.TEST_DATE_STR_RFC2822)).strip(),777 )778 # Atom 0.3 sitemap779 m.get(780 self.TEST_BASE_URL + '/sitemap_atom_0_3.xml',781 headers={'Content-Type': 'application/atom+xml'},782 text=textwrap.dedent("""783 <?xml version="1.0" encoding="UTF-8"?>784 <feed version="0.3" xmlns="http://purl.org/atom/ns#">785 <title>Test Atom 0.3 feed</title>786 <link rel="alternate" type="text/html" href="{base_url}" />787 <modified>{pub_date}</modified>788 </feed>789 """.format(base_url=self.TEST_BASE_URL, pub_date=self.TEST_DATE_STR_ISO8601)).strip(),790 )791 # Atom 1.0 sitemap792 m.get(793 self.TEST_BASE_URL + '/sitemap_atom_1_0.xml',794 headers={'Content-Type': 'application/atom+xml'},795 text=textwrap.dedent("""796 <?xml version="1.0" encoding="UTF-8"?>797 <feed xmlns="http://www.w3.org/2005/Atom">798 <title>Test Atom 1.0 feed</title>799 <subtitle>This is a test Atom 1.0 feed.</subtitle>800 <link href="{base_url}/sitemap_atom_1_0.xml" rel="self" />801 <link href="{base_url}" />802 <id>{base_url}</id>803 <updated>{pub_date}</updated>804 </feed>805 """.format(base_url=self.TEST_BASE_URL, pub_date=self.TEST_DATE_STR_ISO8601)).strip(),806 )807 expected_sitemap_tree = IndexWebsiteSitemap(808 url='{}/'.format(self.TEST_BASE_URL),809 sub_sitemaps=[810 IndexRobotsTxtSitemap(811 url='{}/robots.txt'.format(self.TEST_BASE_URL),812 sub_sitemaps=[813 PagesRSSSitemap(814 url='{}/sitemap_rss.xml'.format(self.TEST_BASE_URL),815 pages=[]816 ),817 PagesAtomSitemap(818 url='{}/sitemap_atom_0_3.xml'.format(self.TEST_BASE_URL),819 pages=[]820 ),821 PagesAtomSitemap(822 url='{}/sitemap_atom_1_0.xml'.format(self.TEST_BASE_URL),823 pages=[]824 ),825 ]826 )827 ]828 )829 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)830 assert expected_sitemap_tree == actual_sitemap_tree831 assert len(list(actual_sitemap_tree.all_pages())) == 0832 def test_sitemap_tree_for_homepage_prematurely_ending_xml(self):833 """Test sitemap_tree_for_homepage() with clipped XML.834 Some webservers are misconfigured to limit the request length to a certain number of seconds, in which time the835 server is unable to generate and compress a 50 MB sitemap XML. Google News doesn't seem to have a problem with836 this behavior, so we have to support this too.837 """838 with requests_mock.Mocker() as m:839 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)840 m.get(841 self.TEST_BASE_URL + '/',842 text='This is a homepage.',843 )844 m.get(845 self.TEST_BASE_URL + '/robots.txt',846 headers={'Content-Type': 'text/plain'},847 text=textwrap.dedent("""848 User-agent: *849 Disallow: /whatever850 851 Sitemap: {base_url}/sitemap.xml852 """.format(base_url=self.TEST_BASE_URL)).strip(),853 )854 m.get(855 self.TEST_BASE_URL + '/sitemap.xml',856 text=textwrap.dedent("""857 <?xml version="1.0" encoding="UTF-8"?>858 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"859 xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">860 <url>861 <loc>{base_url}/news/first.html</loc>862 <news:news>863 <news:publication>864 <news:name>{publication_name}</news:name>865 <news:language>{publication_language}</news:language>866 </news:publication>867 <news:publication_date>{publication_date}</news:publication_date>868 <news:title>First story</news:title>869 </news:news>870 </url>871 <url>872 <loc>{base_url}/news/second.html</loc>873 <news:news>874 <news:publication>875 <news:name>{publication_name}</news:name>876 <news:language>{publication_language}</news:language>877 </news:publication>878 <news:publication_date>{publication_date}</news:publication_date>879 <news:title>Second story</news:title>880 </news:news>881 </url>882 883 <!-- The following story shouldn't get added as the XML ends prematurely -->884 <url>885 <loc>{base_url}/news/third.html</loc>886 <news:news>887 <news:publication>888 <news:name>{publication_name}</news:name>889 <news:language>{publication_language}</news:language>890 </news:publication>891 <news:publicat892 """.format(893 base_url=self.TEST_BASE_URL,894 publication_name=self.TEST_PUBLICATION_NAME,895 publication_language=self.TEST_PUBLICATION_LANGUAGE,896 publication_date=self.TEST_DATE_STR_ISO8601,897 )).strip(),898 )899 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)900 assert isinstance(actual_sitemap_tree, IndexWebsiteSitemap)901 assert len(actual_sitemap_tree.sub_sitemaps) == 1902 assert isinstance(actual_sitemap_tree.sub_sitemaps[0], IndexRobotsTxtSitemap)903 # noinspection PyUnresolvedReferences904 assert len(actual_sitemap_tree.sub_sitemaps[0].sub_sitemaps) == 1905 # noinspection PyUnresolvedReferences906 sitemap = actual_sitemap_tree.sub_sitemaps[0].sub_sitemaps[0]907 assert isinstance(sitemap, PagesXMLSitemap)908 assert len(sitemap.pages) == 2909 def test_sitemap_tree_for_homepage_no_sitemap(self):910 """Test sitemap_tree_for_homepage() with no sitemaps listed in robots.txt."""911 with requests_mock.Mocker() as m:912 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)913 m.get(914 self.TEST_BASE_URL + '/',915 text='This is a homepage.',916 )917 m.get(918 self.TEST_BASE_URL + '/robots.txt',919 headers={'Content-Type': 'text/plain'},920 text=textwrap.dedent("""921 User-agent: *922 Disallow: /whatever923 """.format(base_url=self.TEST_BASE_URL)).strip(),924 )925 expected_sitemap_tree = IndexWebsiteSitemap(926 url='{}/'.format(self.TEST_BASE_URL),927 sub_sitemaps=[928 IndexRobotsTxtSitemap(929 url='{}/robots.txt'.format(self.TEST_BASE_URL),930 sub_sitemaps=[],931 )932 ]933 )934 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)935 assert expected_sitemap_tree == actual_sitemap_tree936 def test_sitemap_tree_for_homepage_unpublished_sitemap(self):937 """Test sitemap_tree_for_homepage() with some sitemaps not published in robots.txt."""938 with requests_mock.Mocker() as m:939 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)940 m.get(941 self.TEST_BASE_URL + '/',942 text='This is a homepage.',943 )944 m.get(945 self.TEST_BASE_URL + '/robots.txt',946 headers={'Content-Type': 'text/plain'},947 text=textwrap.dedent("""948 User-agent: *949 Disallow: /whatever950 951 Sitemap: {base_url}/sitemap_public.xml952 """.format(base_url=self.TEST_BASE_URL)).strip(),953 )954 # Public sitemap (linked to from robots.txt)955 m.get(956 self.TEST_BASE_URL + '/sitemap_public.xml',957 text=textwrap.dedent("""958 <?xml version="1.0" encoding="UTF-8"?>959 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">960 <url>961 <loc>{base_url}/news/public.html</loc>962 </url>963 </urlset>964 """.format(965 base_url=self.TEST_BASE_URL,966 publication_name=self.TEST_PUBLICATION_NAME,967 publication_language=self.TEST_PUBLICATION_LANGUAGE,968 publication_date=self.TEST_DATE_STR_ISO8601,969 )).strip(),970 )971 # Private sitemap (to be discovered by trying out a few paths)972 m.get(973 self.TEST_BASE_URL + '/sitemap_index.xml',974 text=textwrap.dedent("""975 <?xml version="1.0" encoding="UTF-8"?>976 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">977 <url>978 <loc>{base_url}/news/private.html</loc>979 </url>980 </urlset>981 """.format(982 base_url=self.TEST_BASE_URL,983 publication_name=self.TEST_PUBLICATION_NAME,984 publication_language=self.TEST_PUBLICATION_LANGUAGE,985 publication_date=self.TEST_DATE_STR_ISO8601,986 )).strip(),987 )988 expected_sitemap_tree = IndexWebsiteSitemap(989 url='{}/'.format(self.TEST_BASE_URL),990 sub_sitemaps=[991 IndexRobotsTxtSitemap(992 url='{}/robots.txt'.format(self.TEST_BASE_URL),993 sub_sitemaps=[994 PagesXMLSitemap(995 url='{}/sitemap_public.xml'.format(self.TEST_BASE_URL),996 pages=[997 SitemapPage(998 url='{}/news/public.html'.format(self.TEST_BASE_URL),999 ),1000 ],1001 ),1002 ],1003 ),1004 PagesXMLSitemap(1005 url='{}/sitemap_index.xml'.format(self.TEST_BASE_URL),1006 pages=[1007 SitemapPage(1008 url='{}/news/private.html'.format(self.TEST_BASE_URL),1009 ),1010 ],1011 ),1012 ]1013 )1014 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)1015 assert expected_sitemap_tree == actual_sitemap_tree1016 def test_sitemap_tree_for_homepage_robots_txt_no_content_type(self):1017 """Test sitemap_tree_for_homepage() with no Content-Type in robots.txt."""1018 with requests_mock.Mocker() as m:1019 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)1020 m.get(1021 self.TEST_BASE_URL + '/',1022 text='This is a homepage.',1023 )1024 m.get(1025 self.TEST_BASE_URL + '/robots.txt',1026 headers={'Content-Type': ''},1027 text=textwrap.dedent("""1028 User-agent: *1029 Disallow: /whatever1030 """.format(base_url=self.TEST_BASE_URL)).strip(),1031 )1032 expected_sitemap_tree = IndexWebsiteSitemap(1033 url='{}/'.format(self.TEST_BASE_URL),1034 sub_sitemaps=[1035 IndexRobotsTxtSitemap(1036 url='{}/robots.txt'.format(self.TEST_BASE_URL),1037 sub_sitemaps=[],1038 )1039 ]1040 )1041 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)1042 assert expected_sitemap_tree == actual_sitemap_tree1043 def test_sitemap_tree_for_homepage_no_robots_txt(self):1044 """Test sitemap_tree_for_homepage() with no robots.txt."""1045 with requests_mock.Mocker() as m:1046 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)1047 m.get(1048 self.TEST_BASE_URL + '/',1049 text='This is a homepage.',1050 )1051 # Nonexistent robots.txt1052 m.get(1053 self.TEST_BASE_URL + '/robots.txt',1054 status_code=404,1055 reason='Not Found',1056 headers={'Content-Type': 'text/html'},1057 text="<h1>404 Not Found!</h1>",1058 )1059 expected_sitemap_tree = IndexWebsiteSitemap(1060 url='{}/'.format(self.TEST_BASE_URL),1061 sub_sitemaps=[1062 InvalidSitemap(1063 url='{}/robots.txt'.format(self.TEST_BASE_URL),1064 reason=(1065 'Unable to fetch sitemap from {base_url}/robots.txt: 404 Not Found'1066 ).format(base_url=self.TEST_BASE_URL),1067 )1068 ]1069 )1070 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)1071 assert expected_sitemap_tree == actual_sitemap_tree1072 def test_sitemap_tree_for_homepage_huge_sitemap(self):1073 """Test sitemap_tree_for_homepage() with a huge sitemap (mostly for profiling)."""1074 page_count = 10001075 sitemap_xml = """<?xml version="1.0" encoding="UTF-8"?>1076 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"1077 xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"1078 xmlns:xhtml="http://www.w3.org/1999/xhtml">1079 """1080 for x in range(page_count):1081 sitemap_xml += """1082 <url>1083 <loc>{base_url}/news/page_{x}.html</loc>1084 <!-- Element present but empty -->1085 <lastmod />1086 <!-- Some other XML namespace -->1087 <xhtml:link rel="alternate"1088 media="only screen and (max-width: 640px)"1089 href="{base_url}/news/page_{x}.html?mobile=1" />1090 <news:news>1091 <news:publication>1092 <news:name>{publication_name}</news:name>1093 <news:language>{publication_language}</news:language>1094 </news:publication>1095 <news:publication_date>{publication_date}</news:publication_date>1096 <news:title>Foo <foo></news:title> <!-- HTML entity decoding -->1097 </news:news>1098 </url>1099 """.format(1100 x=x,1101 base_url=self.TEST_BASE_URL,1102 publication_name=self.TEST_PUBLICATION_NAME,1103 publication_language=self.TEST_PUBLICATION_LANGUAGE,1104 publication_date=self.TEST_DATE_STR_ISO8601,1105 )1106 sitemap_xml += "</urlset>"1107 with requests_mock.Mocker() as m:1108 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)1109 m.get(1110 self.TEST_BASE_URL + '/',1111 text='This is a homepage.',1112 )1113 m.get(1114 self.TEST_BASE_URL + '/robots.txt',1115 headers={'Content-Type': 'text/plain'},1116 text=textwrap.dedent("""1117 User-agent: *1118 Disallow: /whatever1119 1120 Sitemap: {base_url}/sitemap.xml.gz1121 """.format(base_url=self.TEST_BASE_URL)).strip(),1122 )1123 m.get(1124 self.TEST_BASE_URL + '/sitemap.xml.gz',1125 headers={'Content-Type': 'application/x-gzip'},1126 content=gzip(sitemap_xml),1127 )1128 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)1129 assert len(list(actual_sitemap_tree.all_pages())) == page_count1130 def test_sitemap_tree_for_homepage_robots_txt_weird_spacing(self):1131 """Test sitemap_tree_for_homepage() with weird (but valid) spacing."""1132 with requests_mock.Mocker() as m:1133 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)1134 m.get(1135 self.TEST_BASE_URL + '/',1136 text='This is a homepage.',1137 )1138 robots_txt_body = ""1139 robots_txt_body += "User-agent: *\n"1140 # Extra space before "Sitemap:", no space after "Sitemap:", and extra space after sitemap URL1141 robots_txt_body += " Sitemap:{base_url}/sitemap.xml ".format(base_url=self.TEST_BASE_URL)1142 m.get(1143 self.TEST_BASE_URL + '/robots.txt',1144 headers={'Content-Type': 'text/plain'},1145 text=robots_txt_body,1146 )1147 m.get(1148 self.TEST_BASE_URL + '/sitemap.xml',1149 text=textwrap.dedent("""1150 <?xml version="1.0" encoding="UTF-8"?>1151 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"1152 xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">1153 <url>1154 <loc>{base_url}/news/first.html</loc>1155 <news:news>1156 <news:publication>1157 <news:name>{publication_name}</news:name>1158 <news:language>{publication_language}</news:language>1159 </news:publication>1160 <news:publication_date>{publication_date}</news:publication_date>1161 <news:title>First story</news:title>1162 </news:news>1163 </url>1164 </urlset>1165 """.format(1166 base_url=self.TEST_BASE_URL,1167 publication_name=self.TEST_PUBLICATION_NAME,1168 publication_language=self.TEST_PUBLICATION_LANGUAGE,1169 publication_date=self.TEST_DATE_STR_ISO8601,1170 )).strip(),1171 )1172 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)1173 assert len(list(actual_sitemap_tree.all_pages())) == 11174 def test_sitemap_tree_for_homepage_utf8_bom(self):1175 """Test sitemap_tree_for_homepage() with UTF-8 BOM in both robots.txt and sitemap."""1176 robots_txt_body = textwrap.dedent("""1177 User-agent: *1178 Disallow: /whatever1179 Sitemap: {base_url}/sitemap.xml1180 """.format(base_url=self.TEST_BASE_URL)).strip()1181 sitemap_xml_body = textwrap.dedent("""1182 <?xml version="1.0" encoding="UTF-8"?>1183 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"1184 xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">1185 <url>1186 <loc>{base_url}/news/first.html</loc>1187 <news:news>1188 <news:publication>1189 <news:name>{publication_name}</news:name>1190 <news:language>{publication_language}</news:language>1191 </news:publication>1192 <news:publication_date>{publication_date}</news:publication_date>1193 <news:title>First story</news:title>1194 </news:news>1195 </url>1196 </urlset>1197 """.format(1198 base_url=self.TEST_BASE_URL,1199 publication_name=self.TEST_PUBLICATION_NAME,1200 publication_language=self.TEST_PUBLICATION_LANGUAGE,1201 publication_date=self.TEST_DATE_STR_ISO8601,1202 )).strip()1203 robots_txt_body_encoded = robots_txt_body.encode('utf-8-sig')1204 sitemap_xml_body_encoded = sitemap_xml_body.encode('utf-8-sig')1205 with requests_mock.Mocker() as m:1206 m.add_matcher(TestSitemapTree.fallback_to_404_not_found_matcher)1207 m.get(1208 self.TEST_BASE_URL + '/',1209 text='This is a homepage.',1210 )1211 m.get(1212 self.TEST_BASE_URL + '/robots.txt',1213 headers={'Content-Type': 'text/plain'},1214 content=robots_txt_body_encoded,1215 )1216 m.get(1217 self.TEST_BASE_URL + '/sitemap.xml',1218 content=sitemap_xml_body_encoded,1219 )1220 actual_sitemap_tree = sitemap_tree_for_homepage(homepage_url=self.TEST_BASE_URL)...
combinations.py
Source:combinations.py
1'''Test cases arguments combinations.'''2import os3import sys4import inflection5TEST_DIR = os.path.abspath(os.path.dirname(__file__))6if TEST_DIR not in sys.path:7 sys.path.append(TEST_DIR)8from consts import TEMPDIR, TEST_BASE_URL # noqa: E4029from http_request_codegen.hrc_string import replace_multiple # noqa: E40210def argument_combination_to_filename(combination_name, index):11 return '{}.{}.expect.txt'.format(12 str(index).zfill(3),13 inflection.parameterize(14 replace_multiple(15 combination_name, replacements={16 '"': '-double-quote-',17 '\'': '-single-quote-',18 },19 ),20 ),21 )22def combination_arguments_to_kwargs(arguments):23 kwargs = {}24 for key, value in arguments.items():25 if key == 'kwargs':26 kwargs.update(value)27 else:28 kwargs[key] = value29 return kwargs30def get_argument_combinations(31 method='GET', include_filenames=True,32 dirpath=None,33):34 response = [35 {36 'name': 'URL',37 'arguments': {38 'url': TEST_BASE_URL,39 },40 },41 {42 'name': 'URL wrapping (no wrap)',43 'arguments': {44 'url': TEST_BASE_URL,45 'wrap': 99999,46 },47 },48 {49 'name': 'URL wrapping (wrap 15)',50 'arguments': {51 'url': TEST_BASE_URL,52 'wrap': 15,53 },54 },55 {56 'name': 'Parameter',57 'arguments': {58 'url': TEST_BASE_URL,59 'parameters': [60 {61 'name': 'param-1',62 'value': 'value-1',63 },64 ],65 },66 },67 {68 'name': 'Parameters',69 'arguments': {70 'url': TEST_BASE_URL,71 'parameters': [72 {73 'name': 'param-1',74 'value': 'foo',75 },76 {77 'name': 'param-2',78 'value': 1,79 },80 {81 'name': 'param-3',82 'value': .777,83 },84 {85 'name': 'param-4',86 'value': True,87 },88 ],89 },90 },91 {92 'name': 'Parameter wrapping value',93 'arguments': {94 'url': TEST_BASE_URL,95 'parameters': [96 {97 'name': 'param-1',98 'value': 'foo-bar-baz' * 50,99 },100 ],101 },102 },103 {104 'name': 'Parameters, one wrapping value',105 'arguments': {106 'url': TEST_BASE_URL,107 'parameters': [108 {109 'name': 'param-1',110 'value': 'foo-bar-baz' * 50,111 },112 {113 'name': 'param-2',114 'value': 'value-2',115 },116 ],117 },118 },119 {120 'name': 'Parameter escaping quotes',121 'arguments': {122 'url': TEST_BASE_URL,123 'parameters': [124 {125 'name': 'param-1-with-\'\'-quotes',126 'value': 'value-1-with-\'\'-quotes',127 },128 ],129 },130 },131 {132 'name': 'URL + header',133 'arguments': {134 'url': TEST_BASE_URL,135 'headers': {136 'Content-Type': 'application/json',137 },138 },139 },140 {141 'name': 'URL + headers',142 'arguments': {143 'url': TEST_BASE_URL,144 'headers': {145 'Content-Type': 'application/json',146 'Accept-Language': 'es',147 },148 },149 },150 {151 'name': 'URL + header wrapping value',152 'arguments': {153 'url': TEST_BASE_URL,154 'headers': {155 'Content-Type': 'application/json' * 5,156 },157 },158 },159 {160 'name': 'URL + headers, one wrapping value',161 'arguments': {162 'url': TEST_BASE_URL,163 'headers': {164 'Content-Type': 'application/json' * 5,165 'Accept-Language': '*',166 },167 },168 },169 {170 'name': 'URL + header escaping quotes',171 'arguments': {172 'url': TEST_BASE_URL,173 'headers': {174 'Accept-Language': 'Header value with \'\' quotes',175 },176 },177 },178 {179 'name': 'URL + kwarg',180 'arguments': {181 'url': TEST_BASE_URL,182 'kwargs': {183 'timeout': 5,184 },185 },186 },187 {188 'name': 'URL + kwargs',189 'arguments': {190 'url': TEST_BASE_URL,191 'kwargs': {192 'timeout': 5,193 'stream': True,194 },195 },196 },197 {198 'name': 'URL + kwarg escaping quotes',199 'arguments': {200 'url': TEST_BASE_URL,201 'kwargs': {202 'cookies': {203 'foo': 'value with \'\' quotes',204 },205 },206 },207 },208 {209 'name': 'URL + kwarg wrapping value',210 'arguments': {211 'url': TEST_BASE_URL,212 'kwargs': {213 'cookies': {214 'bar': 'foo bar baz ' * 50,215 },216 },217 },218 },219 {220 'name': 'URL + kwargs, one wrapping value',221 'arguments': {222 'url': TEST_BASE_URL,223 'kwargs': {224 'cookies': {225 'bar': 'foo bar baz ' * 50,226 },227 'stream': True,228 },229 },230 },231 {232 'name': 'Parameter + header',233 'arguments': {234 'url': TEST_BASE_URL,235 'parameters': [236 {237 'name': 'param-1',238 'value': 'value-1',239 },240 ],241 'headers': {242 'Content-Type': 'application/json',243 },244 },245 },246 {247 'name': 'Parameter + header (oneline)',248 'arguments': {249 'url': TEST_BASE_URL,250 'parameters': [251 {252 'name': 'param-1',253 'value': 'value-1',254 },255 ],256 'headers': {257 'Content-Type': 'application/json',258 },259 'oneline': True,260 },261 },262 {263 'name': 'Parameters + header',264 'arguments': {265 'url': TEST_BASE_URL,266 'parameters': [267 {268 'name': 'param-1',269 'value': 'value-1',270 },271 {272 'name': 'param-2',273 'value': 'value-2',274 },275 ],276 'headers': {277 'Content-Type': 'application/json',278 },279 },280 },281 {282 'name': 'Parameter + headers',283 'arguments': {284 'url': TEST_BASE_URL,285 'parameters': [286 {287 'name': 'param-1',288 'value': 'value-1',289 },290 ],291 'headers': {292 'Content-Type': 'application/json',293 'Accept-Language': '*',294 },295 },296 },297 {298 'name': 'Parameters + headers',299 'arguments': {300 'url': TEST_BASE_URL,301 'parameters': [302 {303 'name': 'param-1',304 'value': 'value-1',305 },306 {307 'name': 'param-2',308 'value': 'value-2',309 },310 ],311 'headers': {312 'Content-Type': 'application/json',313 'Accept-Language': '*',314 },315 },316 },317 {318 'name': 'Parameter + kwarg',319 'arguments': {320 'url': TEST_BASE_URL,321 'parameters': [322 {323 'name': 'param-1',324 'value': 'value-1',325 },326 ],327 'kwargs': {328 'timeout': 10,329 },330 },331 },332 {333 'name': 'Parameter + kwarg (oneline)',334 'arguments': {335 'url': TEST_BASE_URL,336 'parameters': [337 {338 'name': 'a',339 'value': 'b',340 },341 ],342 'kwargs': {343 'timeout': 10,344 },345 'oneline': True,346 },347 },348 {349 'name': 'Parameters + kwarg',350 'arguments': {351 'url': TEST_BASE_URL,352 'parameters': [353 {354 'name': 'param-1',355 'value': 'value-1',356 },357 {358 'name': 'param-2',359 'value': 'value-2',360 },361 ],362 'kwargs': {363 'timeout': 10,364 },365 },366 },367 {368 'name': 'Parameter + kwargs',369 'arguments': {370 'url': TEST_BASE_URL,371 'parameters': [372 {373 'name': 'param-1',374 'value': 'value-1',375 },376 ],377 'kwargs': {378 'timeout': 10,379 'stream': True,380 },381 },382 },383 {384 'name': 'Parameters + kwargs',385 'arguments': {386 'url': TEST_BASE_URL,387 'parameters': [388 {389 'name': 'param-1',390 'value': 'value-1',391 },392 {393 'name': 'param-2',394 'value': 'value-2',395 },396 ],397 'kwargs': {398 'timeout': 10,399 'stream': True,400 },401 },402 },403 {404 'name': 'URL + header + kwarg',405 'arguments': {406 'url': TEST_BASE_URL,407 'headers': {408 'Content-Type': 'application/json',409 },410 'kwargs': {411 'timeout': 5,412 },413 },414 },415 {416 'name': 'URL + header + kwarg (oneline)',417 'arguments': {418 'url': TEST_BASE_URL,419 'headers': {420 'Content-Type': 'application/json',421 },422 'kwargs': {423 'timeout': 5,424 },425 'oneline': True,426 },427 },428 {429 'name': 'URL + headers + kwarg',430 'arguments': {431 'url': TEST_BASE_URL,432 'headers': {433 'Content-Type': 'application/json',434 'Accept-Language': '*',435 },436 'kwargs': {437 'timeout': 5,438 },439 },440 },441 {442 'name': 'URL + header + kwargs',443 'arguments': {444 'url': TEST_BASE_URL,445 'headers': {446 'Accept-Language': '*',447 },448 'kwargs': {449 'timeout': 5,450 'stream': False,451 },452 },453 },454 {455 'name': 'URL + headers + kwargs',456 'arguments': {457 'url': TEST_BASE_URL,458 'headers': {459 'Content-Type': 'application/json',460 'Accept-Language': '*',461 },462 'kwargs': {463 'timeout': 5,464 'stream': False,465 },466 },467 },468 {469 'name': 'Parameter + header + kwarg',470 'arguments': {471 'url': TEST_BASE_URL,472 'parameters': [473 {474 'name': 'param-1',475 'value': 'value-1',476 },477 ],478 'headers': {479 'Content-Type': 'application/json',480 },481 'kwargs': {482 'timeout': 5,483 },484 },485 },486 {487 'name': 'Parameter + header + kwargs',488 'arguments': {489 'url': TEST_BASE_URL,490 'parameters': [491 {492 'name': 'param-1',493 'value': 'value-1',494 },495 ],496 'headers': {497 'Content-Type': 'application/json',498 },499 'kwargs': {500 'timeout': 5,501 'stream': True,502 },503 },504 },505 {506 'name': 'Parameters + header + kwarg',507 'arguments': {508 'url': TEST_BASE_URL,509 'parameters': [510 {511 'name': 'param-1',512 'value': 'value-1',513 },514 {515 'name': 'param-2',516 'value': 7.77,517 },518 ],519 'headers': {520 'Content-Type': 'application/json',521 },522 'kwargs': {523 'timeout': 5,524 },525 },526 },527 {528 'name': 'Parameters + header + kwargs',529 'arguments': {530 'url': TEST_BASE_URL,531 'parameters': [532 {533 'name': 'param-1',534 'value': 'value-1',535 },536 {537 'name': 'param-2',538 'value': 7.77,539 },540 ],541 'headers': {542 'Content-Type': 'application/json',543 },544 'kwargs': {545 'timeout': 5,546 'stream': False,547 },548 },549 },550 {551 'name': 'Parameters + headers + kwarg',552 'arguments': {553 'url': TEST_BASE_URL,554 'parameters': [555 {556 'name': 'param-1',557 'value': 'value-1',558 },559 {560 'name': 'param-2',561 'value': 7.77,562 },563 ],564 'headers': {565 'Content-Type': 'application/json',566 'Accept-Language': 'fr',567 },568 'kwargs': {569 'timeout': 5,570 },571 },572 },573 {574 'name': 'Parameters + headers + kwargs',575 'arguments': {576 'url': TEST_BASE_URL,577 'parameters': [578 {579 'name': 'param-1',580 'value': 'value-1',581 },582 {583 'name': 'param-2',584 'value': 7.77,585 },586 ],587 'headers': {588 'Content-Type': 'application/json',589 'Accept-Language': 'fr',590 },591 'kwargs': {592 'timeout': 5,593 'stream': True,594 },595 },596 },597 {598 'name': 'Setup',599 'arguments': {600 'url': TEST_BASE_URL,601 'setup': True,602 },603 },604 {605 'name': 'No setup',606 'arguments': {607 'url': TEST_BASE_URL,608 'setup': False,609 },610 },611 {612 'name': 'Custom setup',613 'arguments': {614 'url': TEST_BASE_URL,615 'setup': 'custom_setup=1\n\n',616 },617 },618 {619 'name': 'Custom teardown',620 'arguments': {621 'url': TEST_BASE_URL,622 'teardown': '\n\ncustom_teardown=1',623 },624 },625 {626 'name': 'Quote character \'',627 'arguments': {628 'url': TEST_BASE_URL,629 'quote_char': '\'',630 },631 },632 {633 'name': 'Quote character "',634 'arguments': {635 'url': TEST_BASE_URL,636 'quote_char': '"',637 },638 },639 {640 'name': 'Indent 2 spaces',641 'arguments': {642 'url': TEST_BASE_URL,643 'indent': ' ',644 'headers': {645 'Accept-Language': 'es en fr * ' * 20,646 },647 },648 },649 {650 'name': 'Indent 4 spaces',651 'arguments': {652 'url': TEST_BASE_URL,653 'indent': ' ',654 'headers': {655 'Accept-Language': 'es en fr * ' * 20,656 },657 },658 },659 {660 'name': 'One line',661 'arguments': {662 'url': TEST_BASE_URL,663 'oneline': True,664 },665 },666 {667 'name': 'One line + no setup',668 'arguments': {669 'url': TEST_BASE_URL,670 'oneline': True,671 'setup': False,672 },673 },674 {675 'name': 'Wrap 0',676 'arguments': {677 'url': TEST_BASE_URL,678 'wrap': 0,679 },680 },681 {682 'name': 'Wrap 1',683 'arguments': {684 'url': TEST_BASE_URL,685 'wrap': 1,686 },687 },688 {689 'name': 'Wrap 10',690 'arguments': {691 'url': TEST_BASE_URL,692 'wrap': 10,693 },694 },695 {696 'name': 'Wrap 20',697 'arguments': {698 'url': TEST_BASE_URL,699 'wrap': 20,700 },701 },702 {703 'name': 'Wrap 25',704 'arguments': {705 'url': TEST_BASE_URL,706 'wrap': 25,707 },708 },709 {710 'name': 'Wrap 30',711 'arguments': {712 'url': TEST_BASE_URL,713 'wrap': 30,714 },715 },716 {717 'name': 'Wrap 35',718 'arguments': {719 'url': TEST_BASE_URL,720 'wrap': 35,721 },722 },723 {724 'name': 'Wrap 40',725 'arguments': {726 'url': TEST_BASE_URL,727 'wrap': 40,728 },729 },730 {731 'name': 'Wrap infinite',732 'arguments': {733 'url': TEST_BASE_URL,734 'wrap': float('inf'),735 },736 },737 {738 'name': 'Wrap null is infinite',739 'arguments': {740 'url': TEST_BASE_URL,741 'wrap': None,742 },743 },744 ]745 if method.lower() == 'post':746 response.extend([747 {748 'name': 'Data by parameter (text/plain)',749 'arguments': {750 'url': TEST_BASE_URL,751 'parameters': [752 {753 'name': '',754 'value': 'foo bar baz ' * 3,755 },756 ],757 'headers': {758 'Content-Type': 'text/plain',759 },760 },761 },762 {763 'name': 'Data by parameter (text/plain) wrapping value',764 'arguments': {765 'url': TEST_BASE_URL,766 'parameters': [767 {768 'name': '',769 'value': 'foo bar baz ' * 30,770 },771 ],772 'headers': {773 'Content-Type': 'text/plain',774 },775 },776 },777 {778 'name': 'Data by parameter (application/json)',779 'arguments': {780 'url': TEST_BASE_URL,781 'parameters': [782 {783 'name': 'param-1',784 'value': 'value-1',785 },786 ],787 'headers': {788 'Content-Type': 'application/json',789 },790 },791 },792 {793 'name': 'Data by parameters (application/json)',794 'arguments': {795 'url': TEST_BASE_URL,796 'parameters': [797 {798 'name': 'param-int',799 'value': 1,800 },801 {802 'name': 'param-float',803 'value': .777,804 },805 {806 'name': 'param-bool',807 'value': True,808 },809 ],810 'headers': {811 'Content-Type': 'application/json',812 },813 },814 },815 {816 'name': (817 'Data by parameter'818 ' (application/x-www-form-urlencoded)'819 ),820 'arguments': {821 'url': TEST_BASE_URL,822 'parameters': [823 {824 'name': 'param-1',825 'value': 'value-1',826 },827 ],828 'headers': {829 'Content-Type': 'application/x-www-form-urlencoded',830 },831 },832 },833 {834 'name': (835 'Data by parameters'836 ' (application/x-www-form-urlencoded)'837 ),838 'arguments': {839 'url': TEST_BASE_URL,840 'parameters': [841 {842 'name': 'param-int',843 'value': 1,844 },845 {846 'name': 'param-float',847 'value': .777,848 },849 {850 'name': 'param-bool',851 'value': True,852 },853 ],854 'headers': {855 'Content-Type': 'application/x-www-form-urlencoded',856 },857 },858 },859 {860 'name': 'File by filepath (multipart/form-data)',861 'arguments': {862 'url': TEST_BASE_URL,863 'files': {864 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),865 },866 },867 },868 {869 'name': 'Files by filepath (multipart/form-data)',870 'arguments': {871 'url': TEST_BASE_URL,872 'files': {873 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),874 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),875 },876 },877 },878 {879 'name': 'File by filepath (multipart/form-data) wrapping',880 'arguments': {881 'url': TEST_BASE_URL,882 'files': {883 'param-1': os.path.join(884 TEMPDIR, '%s.ext' % ('foo' * 40),885 ),886 },887 },888 },889 {890 'name': (891 'Files by filepath (multipart/form-data)'892 ' with Content-Type'893 ),894 'arguments': {895 'url': TEST_BASE_URL,896 'files': {897 'param-1': (898 os.path.join(TEMPDIR, 'file-1.ext'),899 'text/plain',900 ),901 'param-2': (902 os.path.join(TEMPDIR, 'file-2.ext'),903 'text/csv',904 ),905 },906 },907 },908 {909 'name': (910 'File by filepath (multipart/form-data)'911 ' with Content-Type wrapping'912 ),913 'arguments': {914 'url': TEST_BASE_URL,915 'files': {916 'param-1': (917 os.path.join(TEMPDIR, 'file-1.ext'),918 'text/plain ' * 20,919 ),920 },921 },922 },923 {924 'name': (925 'File by filepath (multipart/form-data),'926 ' Content-Type, header'927 ),928 'arguments': {929 'url': TEST_BASE_URL,930 'files': {931 'param-1': (932 os.path.join(TEMPDIR, 'file-1.ext'),933 'text/plain',934 {'Accept-Language': 'es'},935 ),936 },937 },938 },939 {940 'name': (941 'File by filepath (multipart/form-data),'942 ' Content-Type, headers'943 ),944 'arguments': {945 'url': TEST_BASE_URL,946 'files': {947 'param-1': (948 os.path.join(TEMPDIR, 'file-1.ext'),949 'text/plain',950 {951 'Accept-Language': 'es',952 'Accept-Charset': 'utf-8',953 },954 ),955 },956 },957 },958 {959 'name': 'Files by filepath (multipart/form-data) + parameter',960 'arguments': {961 'url': TEST_BASE_URL,962 'files': {963 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),964 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),965 },966 'parameters': [967 {968 'name': 'param-1',969 'value': 'value-1',970 },971 ],972 },973 },974 {975 'name': 'Files by filepath (multipart/form-data) + parameters',976 'arguments': {977 'url': TEST_BASE_URL,978 'files': {979 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),980 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),981 },982 'parameters': [983 {984 'name': 'param-1',985 'value': 'value-1',986 },987 {988 'name': 'param-2',989 'value': 'value-2',990 },991 ],992 },993 },994 {995 'name': (996 'Files by filepath (multipart/form-data) + parameter'997 ' + header'998 ),999 'arguments': {1000 'url': TEST_BASE_URL,1001 'files': {1002 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1003 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1004 },1005 'parameters': [1006 {1007 'name': 'param-1',1008 'value': 'value-1',1009 },1010 ],1011 'headers': {1012 'Accept-Language': 'fr',1013 },1014 },1015 },1016 {1017 'name': (1018 'Files by filepath (multipart/form-data) + parameter'1019 ' + headers'1020 ),1021 'arguments': {1022 'url': TEST_BASE_URL,1023 'files': {1024 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1025 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1026 },1027 'parameters': [1028 {1029 'name': 'param-1',1030 'value': 'value-1',1031 },1032 ],1033 'headers': {1034 'Accept-Language': 'fr',1035 'Accept-Charset': 'utf-8',1036 },1037 },1038 },1039 {1040 'name': (1041 'Files by filepath (multipart/form-data) + parameters'1042 ' + header'1043 ),1044 'arguments': {1045 'url': TEST_BASE_URL,1046 'files': {1047 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1048 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1049 },1050 'parameters': [1051 {1052 'name': 'param-1',1053 'value': 'value-1',1054 },1055 {1056 'name': 'param-2',1057 'value': 'value-2',1058 },1059 ],1060 'headers': {1061 'Accept-Language': 'es',1062 },1063 },1064 },1065 {1066 'name': (1067 'Files by filepath (multipart/form-data) + parameters'1068 ' + headers'1069 ),1070 'arguments': {1071 'url': TEST_BASE_URL,1072 'files': {1073 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1074 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1075 },1076 'parameters': [1077 {1078 'name': 'param-1',1079 'value': 'value-1',1080 },1081 {1082 'name': 'param-2',1083 'value': 'value-2',1084 },1085 ],1086 'headers': {1087 'Accept-Language': 'fr',1088 'Accept-Charset': 'utf-8',1089 },1090 },1091 },1092 {1093 'name': (1094 'Files by filepath (multipart/form-data) + parameter'1095 ' + header + kwarg'1096 ),1097 'arguments': {1098 'url': TEST_BASE_URL,1099 'files': {1100 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1101 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1102 },1103 'parameters': [1104 {1105 'name': 'param-1',1106 'value': 'value-1',1107 },1108 ],1109 'headers': {1110 'Accept-Language': 'fr',1111 },1112 'kwargs': {1113 'timeout': 10,1114 },1115 },1116 },1117 {1118 'name': (1119 'Files by filepath (multipart/form-data) + parameter'1120 ' + headers + kwarg'1121 ),1122 'arguments': {1123 'url': TEST_BASE_URL,1124 'files': {1125 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1126 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1127 },1128 'parameters': [1129 {1130 'name': 'param-1',1131 'value': 'value-1',1132 },1133 ],1134 'headers': {1135 'Accept-Language': 'fr',1136 'Accept-Charset': 'utf-8',1137 },1138 'kwargs': {1139 'timeout': 10,1140 },1141 },1142 },1143 {1144 'name': (1145 'Files by filepath (multipart/form-data) + parameters'1146 ' + header + kwarg'1147 ),1148 'arguments': {1149 'url': TEST_BASE_URL,1150 'files': {1151 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1152 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1153 },1154 'parameters': [1155 {1156 'name': 'param-1',1157 'value': 'value-1',1158 },1159 {1160 'name': 'param-2',1161 'value': 'value-2',1162 },1163 ],1164 'headers': {1165 'Accept-Language': 'fr',1166 },1167 'kwargs': {1168 'timeout': 10,1169 },1170 },1171 },1172 {1173 'name': (1174 'Files by filepath (multipart/form-data) + parameters'1175 ' + headers + kwarg'1176 ),1177 'arguments': {1178 'url': TEST_BASE_URL,1179 'files': {1180 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1181 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1182 },1183 'parameters': [1184 {1185 'name': 'param-1',1186 'value': 'value-1',1187 },1188 {1189 'name': 'param-2',1190 'value': 'value-2',1191 },1192 ],1193 'headers': {1194 'Accept-Language': 'fr',1195 'Accept-Charset': 'utf-8',1196 },1197 'kwargs': {1198 'timeout': 10,1199 },1200 },1201 },1202 {1203 'name': (1204 'Files by filepath (multipart/form-data) + parameter'1205 ' + header + kwargs'1206 ),1207 'arguments': {1208 'url': TEST_BASE_URL,1209 'files': {1210 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1211 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1212 },1213 'parameters': [1214 {1215 'name': 'param-1',1216 'value': 'value-1',1217 },1218 ],1219 'headers': {1220 'Accept-Language': 'fr',1221 },1222 'kwargs': {1223 'timeout': 10,1224 'cookies': {1225 'hello': 'world',1226 },1227 },1228 },1229 },1230 {1231 'name': (1232 'Files by filepath (multipart/form-data) + parameter'1233 ' + headers + kwargs'1234 ),1235 'arguments': {1236 'url': TEST_BASE_URL,1237 'files': {1238 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1239 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1240 },1241 'parameters': [1242 {1243 'name': 'param-1',1244 'value': 'value-1',1245 },1246 ],1247 'headers': {1248 'Accept-Language': 'fr',1249 'Accept-Charset': 'utf-8',1250 },1251 'kwargs': {1252 'timeout': 10,1253 'stream': False,1254 },1255 },1256 },1257 {1258 'name': (1259 'Files by filepath (multipart/form-data) + parameters'1260 ' + header + kwargs'1261 ),1262 'arguments': {1263 'url': TEST_BASE_URL,1264 'files': {1265 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1266 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1267 },1268 'parameters': [1269 {1270 'name': 'param-1',1271 'value': 'value-1',1272 },1273 {1274 'name': 'param-2',1275 'value': 'value-2',1276 },1277 ],1278 'headers': {1279 'Accept-Language': 'fr',1280 },1281 'kwargs': {1282 'timeout': 10,1283 'stream': False,1284 },1285 },1286 },1287 {1288 'name': (1289 'Files by filepath (multipart/form-data) + parameters'1290 ' + headers + kwargs'1291 ),1292 'arguments': {1293 'url': TEST_BASE_URL,1294 'files': {1295 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1296 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1297 },1298 'parameters': [1299 {1300 'name': 'param-1',1301 'value': 'value-1',1302 },1303 {1304 'name': 'param-2',1305 'value': 'value-2',1306 },1307 ],1308 'headers': {1309 'Accept-Language': 'fr',1310 'Accept-Charset': 'utf-8',1311 },1312 'kwargs': {1313 'timeout': 10,1314 'stream': False,1315 },1316 },1317 },1318 {1319 'name': (1320 'No setup + files by filepath (multipart/form-data)'1321 ' + parameters + headers + kwargs + '1322 ),1323 'arguments': {1324 'url': TEST_BASE_URL,1325 'setup': False,1326 'files': {1327 'param-1': os.path.join(TEMPDIR, 'file-1.ext'),1328 'param-2': os.path.join(TEMPDIR, 'file-2.ext'),1329 },1330 'parameters': [1331 {1332 'name': 'param-1',1333 'value': 'value-1',1334 },1335 {1336 'name': 'param-2',1337 'value': 'value-2',1338 },1339 ],1340 'headers': {1341 'Accept-Language': 'fr',1342 'Accept-Charset': 'utf-8',1343 },1344 'kwargs': {1345 'timeout': 10,1346 'stream': False,1347 },1348 },1349 },1350 ])1351 if include_filenames:1352 for index, args_group in enumerate(response):1353 fname = argument_combination_to_filename(1354 args_group['name'], index,1355 )1356 if dirpath and os.path.exists(dirpath):1357 fname = os.path.join(dirpath, fname)1358 args_group['filename'] = fname...
test_logoscraper.py
Source:test_logoscraper.py
1from bs4 import BeautifulSoup2import scraper.logoscraper as logoscraper3test_base_url = "https://www.testbase.com"4def test_get_logo_should_pass():5 htmls = [6 f"<div class='logo'><img src='https://www.test.com'></img></div>",7 f"<div id='logo'><img src='https://www.test.com'></img></div>",8 f"<a><img src='https://www.test.com'></img></a>",9 f"<div><img src='https://www.test.com'></img></div>",10 f"<a href={test_base_url}><img src='https://www.test.com'></img></div>",11 ]12 for html in htmls:13 assert (14 logoscraper.get_logo(BeautifulSoup(html, "html.parser"), test_base_url)15 == "https://www.test.com"16 )17def test_find_image_tag():18 result = logoscraper.find_img_tag(19 BeautifulSoup(20 f'<a><img src="https://{test_base_url}"></img></a>', "html.parser"21 ),22 test_base_url,23 )24 assert result25def test_find_image_tag_return_itself():26 result = logoscraper.find_img_tag(27 BeautifulSoup(f'<img src="https://{test_base_url}"></img>', "html.parser"),28 test_base_url,29 )30 assert result31def test_find_image_tag_retrun_none():32 result = logoscraper.find_img_tag(33 BeautifulSoup(f'<a href="https://{test_base_url}"></a>', "html.parser"),34 test_base_url,35 )36 assert result is None37def test_format_image_source():38 html = BeautifulSoup(f"<img src='{test_base_url}'></img>", "html.parser").find(39 "img"40 )41 assert logoscraper.format_image_source(html, test_base_url) == test_base_url42def test_format_image_source_no_source():43 html = BeautifulSoup(f"<img></img>", "html.parser").find("img")44 assert logoscraper.format_image_source(html, test_base_url) is None45def test_format_image_source_relative_path():46 html = BeautifulSoup(47 f"<img src='resources/images/image.png'></img>", "html.parser"48 ).find("img")49 assert (50 logoscraper.format_image_source(html, test_base_url)51 == f"{test_base_url}/resources/images/image.png"...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run Selene automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
