How to use update_file method in lisa
Best Python code snippet using lisa_python
update_test_data.py
Source:update_test_data.py 
...62 if not ServiceCheck.service_ok():63 print('Not updating test data as services are unavailable.')64 sys.exit(1)65 pool = LocalSessionThreadPool()66 def update_file(filepath, url, process_fn=None):67 pool.execute(update_file_real, (filepath, url, process_fn))68 # types/boring69 update_file('types/boring/boring.xml',70 build_dov_url('data/boring/2004-103984.xml'))71 update_file(72 'types/boring/wfsgetfeature.xml',73 build_dov_url(74 'geoserver/ows?service=WFS'75 '&version=2.0.0&request=GetFeature&typeName=dov-pub:Boringen'76 '&maxFeatures=1&CQL_Filter=fiche=%27' +77 build_dov_url('data/boring/2004-103984%27')))78 update_file(79 'types/boring/feature.xml',80 build_dov_url(81 'geoserver/ows?service=WFS'82 '&version=2.0.0&request=GetFeature&typeName=dov-pub:Boringen'83 '&maxFeatures=1&CQL_Filter=fiche=%27' +84 build_dov_url('data/boring/2004-103984%27')),85 get_first_featuremember)86 update_file(87 'types/boring/fc_featurecatalogue.xml',88 build_dov_url(89 'geonetwork/srv/dut/csw'90 '?Service=CSW&Request=GetRecordById&Version=2.0.2'91 '&outputSchema=http://www.isotc211.org/2005/gfc'92 '&elementSetName=full&id=c0cbd397-520f-4ee1-aca7'93 '-d70e271eeed6'))94 update_file(95 'types/boring/md_metadata.xml',96 build_dov_url(97 'geonetwork/srv/dut/csw'98 '?Service=CSW&Request=GetRecordById&Version=2.0.2'99 '&outputSchema=http://www.isotc211.org/2005/gmd'100 '&elementSetName=full&id=4e20bf9c-3a5c-42be-b5b6'101 '-bef6214d1fa7'))102 update_file(103 'types/boring/wfsdescribefeaturetype.xml',104 build_dov_url(105 'geoserver/dov-pub/Boringen'106 '/ows?service=wfs&version=2.0.0&request=DescribeFeatureType'))107 for xsd_schema in Boring.get_xsd_schemas():108 update_file(109 'types/boring/xsd_{}.xml'.format(xsd_schema.split('/')[-1]),110 xsd_schema)111 # types/sondering112 update_file('types/sondering/sondering.xml',113 build_dov_url('data/sondering/2002-018435.xml'))114 update_file(115 'types/sondering/wfsgetfeature.xml',116 build_dov_url(117 'geoserver/ows?service=WFS'118 '&version=2.0.0&request=GetFeature&typeName=dov-pub'119 ':Sonderingen&maxFeatures=1&CQL_Filter=fiche=%27' +120 build_dov_url('data/sondering/2002-018435%27')))121 update_file(122 'types/sondering/feature.xml',123 build_dov_url(124 'geoserver/ows?service=WFS'125 '&version=2.0.0&request=GetFeature&typeName=dov-pub'126 ':Sonderingen&maxFeatures=1&CQL_Filter=fiche=%27' +127 build_dov_url('data/sondering/2002-018435%27')),128 get_first_featuremember)129 update_file(130 'types/sondering/fc_featurecatalogue.xml',131 build_dov_url(132 'geonetwork/srv/dut/csw'133 '?Service=CSW&Request=GetRecordById&Version=2.0.2'134 '&outputSchema=http://www.isotc211.org/2005/gfc'135 '&elementSetName=full&id=bd539ba5-5f4d-4c43-9662'136 '-51c16caea351'))137 update_file(138 'types/sondering/md_metadata.xml',139 build_dov_url(140 'geonetwork/srv/dut/csw'141 '?Service=CSW&Request=GetRecordById&Version=2.0.2'142 '&outputSchema=http://www.isotc211.org/2005/gmd'143 '&elementSetName=full&id=b397faec-1b64-4854-8000'144 '-2375edb3b1a8'))145 update_file(146 'types/sondering/wfsdescribefeaturetype.xml',147 build_dov_url(148 'geoserver/dov-pub/Sonderingen'149 '/ows?service=wfs&version=2.0.0&request=DescribeFeatureType'))150 for xsd_schema in Sondering.get_xsd_schemas():151 update_file(152 'types/sondering/xsd_{}.xml'.format(xsd_schema.split('/')[-1]),153 xsd_schema)154 # types/interpretaties/informele_stratigrafie155 update_file('types/interpretaties/informele_stratigrafie'156 '/informele_stratigrafie.xml',157 build_dov_url('data/interpretatie/1962-101692.xml'))158 update_file(159 'types/interpretaties/informele_stratigrafie'160 '/wfsgetfeature.xml',161 build_dov_url(162 'geoserver/ows?service=WFS'163 '&version=2.0.0&request=GetFeature&typeName=interpretaties'164 ':informele_stratigrafie&maxFeatures=1&CQL_Filter'165 '=Interpretatiefiche=%27') +166 build_dov_url(167 'data'168 '/interpretatie/1962-101692%27'))169 update_file(170 'types/interpretaties/informele_stratigrafie/feature.xml',171 build_dov_url(172 'geoserver/ows?service=WFS'173 '&version=2.0.0&request=GetFeature&typeName=interpretaties'174 ':informele_stratigrafie&maxFeatures=1&CQL_Filter'175 '=Interpretatiefiche=%27') +176 build_dov_url(177 'data'178 '/interpretatie/1962-101692%27'),179 get_first_featuremember)180 update_file(181 'types/interpretaties/informele_stratigrafie/fc_featurecatalogue.xml',182 build_dov_url(183 'geonetwork/srv/dut/csw'184 '?Service=CSW&Request=GetRecordById&Version=2.0.2'185 '&outputSchema=http://www.isotc211.org/2005/gfc'186 '&elementSetName=full&id=b6c651f9-5972-4252-ae10-ad69ad08e78d'))187 update_file(188 'types/interpretaties/informele_stratigrafie/md_metadata.xml',189 build_dov_url(190 'geonetwork/srv/dut/csw'191 '?Service=CSW&Request=GetRecordById&Version=2.0.2'192 '&outputSchema=http://www.isotc211.org/2005/gmd'193 '&elementSetName=full&id=bd171ea4-2509-478d-a21c'194 '-c2728d3a9051'))195 update_file(196 'types/interpretaties/informele_stratigrafie/wfsdescribefeaturetype'197 '.xml', build_dov_url(198 'geoserver/interpretaties'199 '/informele_stratigrafie/ows?service=wfs&version=2.0.0&request'200 '=DescribeFeatureType'))201 for xsd_schema in InformeleStratigrafie.get_xsd_schemas():202 update_file(203 'types/interpretaties/informele_stratigrafie/xsd_%s.xml' %204 xsd_schema.split('/')[-1], xsd_schema)205 # types/interpretaties/formele_stratigrafie206 update_file('types/interpretaties/formele_stratigrafie'207 '/formele_stratigrafie.xml',208 build_dov_url('data/interpretatie/2011-249333.xml'))209 update_file(210 'types/interpretaties/formele_stratigrafie'211 '/wfsgetfeature.xml',212 build_dov_url(213 'geoserver/ows?service=WFS'214 '&version=2.0.0&request=GetFeature&typeName=interpretaties'215 ':formele_stratigrafie&maxFeatures=1&CQL_Filter'216 '=Interpretatiefiche=%27') +217 build_dov_url(218 'data'219 '/interpretatie/2011-249333%27'))220 update_file(221 'types/interpretaties/formele_stratigrafie/feature.xml',222 build_dov_url(223 'geoserver/ows?service=WFS'224 '&version=2.0.0&request=GetFeature&typeName=interpretaties'225 ':formele_stratigrafie&maxFeatures=1&CQL_Filter'226 '=Interpretatiefiche=%27') +227 build_dov_url(228 'data'229 '/interpretatie/2011-249333%27'),230 get_first_featuremember)231 update_file(232 'types/interpretaties/formele_stratigrafie/fc_featurecatalogue.xml',233 build_dov_url(234 'geonetwork/srv/dut/csw'235 '?Service=CSW&Request=GetRecordById&Version=2.0.2'236 '&outputSchema=http://www.isotc211.org/2005/gfc'237 '&elementSetName=full&id=68405b5d-51e6-44d0-b634-b580bc2f9eb6'))238 update_file(239 'types/interpretaties/formele_stratigrafie/md_metadata.xml',240 build_dov_url(241 'geonetwork/srv/dut/csw'242 '?Service=CSW&Request=GetRecordById&Version=2.0.2'243 '&outputSchema=http://www.isotc211.org/2005/gmd'244 '&elementSetName=full&id=212af8cd-bffd-423c-9d2b'245 '-69c544ab3b04'))246 update_file(247 'types/interpretaties/formele_stratigrafie/wfsdescribefeaturetype'248 '.xml', build_dov_url(249 'geoserver/interpretaties'250 '/formele_stratigrafie/ows?service=wfs&version=2.0.0&request'251 '=DescribeFeatureType'))252 for xsd_schema in FormeleStratigrafie.get_xsd_schemas():253 update_file(254 'types/interpretaties/formele_stratigrafie/xsd_%s.xml' %255 xsd_schema.split('/')[-1], xsd_schema)256 # types/interpretaties/hydrogeologische_stratigrafie257 update_file('types/interpretaties/hydrogeologische_stratigrafie'258 '/hydrogeologische_stratigrafie.xml',259 build_dov_url('data/interpretatie/2001-186543.xml'))260 update_file(261 'types/interpretaties/hydrogeologische_stratigrafie'262 '/wfsgetfeature.xml',263 build_dov_url(264 'geoserver/ows?service=WFS'265 '&version=2.0.0&request=GetFeature&typeName=interpretaties'266 ':hydrogeologische_stratigrafie&maxFeatures=1&CQL_Filter'267 '=Interpretatiefiche=%27') +268 build_dov_url(269 'data'270 '/interpretatie/2001-186543%27'))271 update_file(272 'types/interpretaties/hydrogeologische_stratigrafie'273 '/feature.xml',274 build_dov_url(275 'geoserver/ows?service=WFS'276 '&version=2.0.0&request=GetFeature&typeName=interpretaties'277 ':hydrogeologische_stratigrafie&maxFeatures=1&CQL_Filter'278 '=Interpretatiefiche=%27') +279 build_dov_url(280 'data/'281 'interpretatie/2001-186543%27'),282 get_first_featuremember)283 update_file(284 'types/interpretaties/hydrogeologische_stratigrafie/'285 'fc_featurecatalogue.xml', build_dov_url(286 'geonetwork/srv/dut/csw'287 '?Service=CSW&Request=GetRecordById&Version=2.0.2'288 '&outputSchema=http://www.isotc211.org/2005/gfc'289 '&elementSetName=full&id=b89e72de-35a9-4bca-8d0b-712d1e881ea6'))290 update_file(291 'types/interpretaties/hydrogeologische_stratigrafie/'292 'md_metadata.xml', build_dov_url(293 'geonetwork/srv/dut/csw'294 '?Service=CSW&Request=GetRecordById&Version=2.0.2'295 '&outputSchema=http://www.isotc211.org/2005/gmd'296 '&elementSetName=full&id=25c5d9fa-c2ba-4184-b796'297 '-fde790e73d39'))298 update_file(299 'types/interpretaties/hydrogeologische_stratigrafie/'300 'wfsdescribefeaturetype.xml', build_dov_url(301 'geoserver/interpretaties'302 '/hydrogeologische_stratigrafie/ows?service=wfs&version=2.0.0'303 '&request=DescribeFeatureType'))304 for xsd_schema in HydrogeologischeStratigrafie.get_xsd_schemas():305 update_file(306 'types/interpretaties/hydrogeologische_stratigrafie/xsd_%s.xml' %307 xsd_schema.split('/')[-1], xsd_schema)308 # types/interpretaties/lithologische_beschrijvingen309 update_file('types/interpretaties/lithologische_beschrijvingen'310 '/lithologische_beschrijvingen.xml',311 build_dov_url('data/interpretatie/1958-003925.xml'))312 update_file(313 'types/interpretaties/lithologische_beschrijvingen'314 '/wfsgetfeature.xml',315 build_dov_url(316 'geoserver/ows?service=WFS'317 '&version=2.0.0&request=GetFeature&typeName=interpretaties'318 ':lithologische_beschrijvingen&maxFeatures=1&CQL_Filter'319 '=Interpretatiefiche=%27') +320 build_dov_url(321 'data'322 '/interpretatie/1958-003925%27'))323 update_file(324 'types/interpretaties/lithologische_beschrijvingen/feature.xml',325 build_dov_url(326 'geoserver/ows?service=WFS'327 '&version=2.0.0&request=GetFeature&typeName=interpretaties'328 ':lithologische_beschrijvingen&maxFeatures=1&CQL_Filter'329 '=Interpretatiefiche=%27') +330 build_dov_url(331 'data'332 '/interpretatie/1958-003925%27'),333 get_first_featuremember)334 update_file(335 'types/interpretaties/lithologische_beschrijvingen/'336 'fc_featurecatalogue.xml',337 build_dov_url(338 'geonetwork/srv/dut/csw'339 '?Service=CSW&Request=GetRecordById&Version=2.0.2'340 '&outputSchema=http://www.isotc211.org/2005/gfc'341 '&elementSetName=full&id=2450d592-29bc-4970-a89f-a7b14bd38dc2'))342 update_file(343 'types/interpretaties/lithologische_beschrijvingen/md_metadata.xml',344 build_dov_url(345 'geonetwork/srv/dut/csw'346 '?Service=CSW&Request=GetRecordById&Version=2.0.2'347 '&outputSchema=http://www.isotc211.org/2005/gmd'348 '&elementSetName=full&id=45b5610e-9a66-42bd-b920'349 '-af099e399f3b'))350 update_file(351 'types/interpretaties/lithologische_beschrijvingen/'352 'wfsdescribefeaturetype.xml', build_dov_url(353 'geoserver/interpretaties'354 '/lithologische_beschrijvingen/ows?service=wfs&version=2.0.0'355 '&request=DescribeFeatureType'))356 for xsd_schema in LithologischeBeschrijvingen.get_xsd_schemas():357 update_file(358 'types/interpretaties/lithologische_beschrijvingen/xsd_%s.xml' %359 xsd_schema.split('/')[-1], xsd_schema)360 # types/interpretaties/gecodeerde_lithologie361 update_file('types/interpretaties/gecodeerde_lithologie'362 '/gecodeerde_lithologie.xml',363 build_dov_url('data/interpretatie/2001-046845.xml'))364 update_file(365 'types/interpretaties/gecodeerde_lithologie'366 '/wfsgetfeature.xml',367 build_dov_url(368 'geoserver/ows?service=WFS'369 '&version=2.0.0&request=GetFeature&typeName=interpretaties'370 ':gecodeerde_lithologie&maxFeatures=1&CQL_Filter'371 '=Interpretatiefiche=%27') +372 build_dov_url(373 'data'374 '/interpretatie/2001-046845%27'))375 update_file(376 'types/interpretaties/gecodeerde_lithologie/feature.xml',377 build_dov_url(378 'geoserver/ows?service=WFS'379 '&version=2.0.0&request=GetFeature&typeName=interpretaties'380 ':gecodeerde_lithologie&maxFeatures=1&CQL_Filter'381 '=Interpretatiefiche=%27') +382 build_dov_url(383 'data'384 '/interpretatie/2001-046845%27'),385 get_first_featuremember)386 update_file(387 'types/interpretaties/gecodeerde_lithologie/fc_featurecatalogue.xml',388 build_dov_url(389 'geonetwork/srv/dut/csw'390 '?Service=CSW&Request=GetRecordById&Version=2.0.2'391 '&outputSchema=http://www.isotc211.org/2005/gfc'392 '&elementSetName=full&id=0032241d-8920-415e-b1d8-fa0a48154904'))393 update_file(394 'types/interpretaties/gecodeerde_lithologie/md_metadata.xml',395 build_dov_url(396 'geonetwork/srv/dut/csw'397 '?Service=CSW&Request=GetRecordById&Version=2.0.2'398 '&outputSchema=http://www.isotc211.org/2005/gmd'399 '&elementSetName=full&id=35d630e4-9145-46f9-b7dc'400 '-da290a0adc55'))401 update_file(402 'types/interpretaties/gecodeerde_lithologie/wfsdescribefeaturetype'403 '.xml', build_dov_url(404 'geoserver/interpretaties'405 '/gecodeerde_lithologie/ows?service=wfs&version=2.0.0&request'406 '=DescribeFeatureType'))407 for xsd_schema in GecodeerdeLithologie.get_xsd_schemas():408 update_file(409 'types/interpretaties/gecodeerde_lithologie/xsd_%s.xml' %410 xsd_schema.split('/')[-1], xsd_schema)411 # types/interpretaties/geotechnische_codering412 update_file('types/interpretaties/geotechnische_codering'413 '/geotechnische_codering.xml',414 build_dov_url('data/interpretatie/2016-298511.xml'))415 update_file(416 'types/interpretaties/geotechnische_codering'417 '/wfsgetfeature.xml',418 build_dov_url(419 'geoserver/ows?service=WFS'420 '&version=2.0.0&request=GetFeature&typeName=interpretaties'421 ':geotechnische_coderingen&maxFeatures=1&CQL_Filter'422 '=Interpretatiefiche=%27') +423 build_dov_url(424 'data'425 '/interpretatie/2016-298511%27'))426 update_file(427 'types/interpretaties/geotechnische_codering/feature.xml',428 build_dov_url(429 'geoserver/ows?service=WFS'430 '&version=2.0.0&request=GetFeature&typeName=interpretaties'431 ':geotechnische_coderingen&maxFeatures=1&CQL_Filter'432 '=Interpretatiefiche=%27') +433 build_dov_url(434 'data'435 '/interpretatie/2016-298511%27'),436 get_first_featuremember)437 update_file(438 'types/interpretaties/geotechnische_codering/fc_featurecatalogue.xml',439 build_dov_url(440 'geonetwork/srv/dut/csw'441 '?Service=CSW&Request=GetRecordById&Version=2.0.2'442 '&outputSchema=http://www.isotc211.org/2005/gfc'443 '&elementSetName=full&id=85404aa6-2d88-46f6-ae5a-575aece71efd'))444 update_file(445 'types/interpretaties/geotechnische_codering/md_metadata.xml',446 build_dov_url(447 'geonetwork/srv/dut/csw'448 '?Service=CSW&Request=GetRecordById&Version=2.0.2'449 '&outputSchema=http://www.isotc211.org/2005/gmd'450 '&elementSetName=full&id=6a3dc5d4-0744-4d9c-85ce'451 '-da50913851cc'))452 update_file(453 'types/interpretaties/geotechnische_codering/wfsdescribefeaturetype'454 '.xml', build_dov_url(455 'geoserver/interpretaties'456 '/geotechnische_coderingen/ows?service=wfs&version=2.0.0&request'457 '=DescribeFeatureType'))458 for xsd_schema in GeotechnischeCodering.get_xsd_schemas():459 update_file(460 'types/interpretaties/geotechnische_codering/xsd_%s.xml' %461 xsd_schema.split('/')[-1], xsd_schema)462 # types/interpretaties/informele_hydrogeologische_stratigrafie463 update_file('types/interpretaties/informele_hydrogeologische_stratigrafie'464 '/informele_hydrogeologische_stratigrafie.xml',465 build_dov_url('data/interpretatie/2003-297774.xml'))466 update_file(467 'types/interpretaties/informele_hydrogeologische_stratigrafie'468 '/wfsgetfeature.xml',469 build_dov_url(470 'geoserver/ows?service=WFS'471 '&version=2.0.0&request=GetFeature&typeName=interpretaties'472 ':informele_hydrogeologische_stratigrafie&maxFeatures=1'473 '&CQL_Filter=Interpretatiefiche=%27') +474 build_dov_url(475 'data'476 '/interpretatie/2003-297774%27'))477 update_file(478 'types/interpretaties/informele_hydrogeologische_stratigrafie'479 '/feature.xml',480 build_dov_url(481 'geoserver/ows?service=WFS'482 '&version=2.0.0&request=GetFeature&typeName=interpretaties'483 ':informele_hydrogeologische_stratigrafie&maxFeatures=1'484 '&CQL_Filter=Interpretatiefiche=%27') +485 build_dov_url(486 'data'487 '/interpretatie/2003-297774%27'),488 get_first_featuremember)489 update_file(490 'types/interpretaties/informele_hydrogeologische_stratigrafie'491 '/fc_featurecatalogue.xml',492 build_dov_url(493 'geonetwork/srv/dut/csw'494 '?Service=CSW&Request=GetRecordById&Version=2.0.2'495 '&outputSchema=http://www.isotc211.org/2005/gfc'496 '&elementSetName=full&id=69f71840-bd29-4b59-9b02-4e36aafaa041'))497 update_file(498 'types/interpretaties/informele_hydrogeologische_stratigrafie'499 '/md_metadata.xml',500 build_dov_url(501 'geonetwork/srv/dut/csw'502 '?Service=CSW&Request=GetRecordById&Version=2.0.2'503 '&outputSchema=http://www.isotc211.org/2005/gmd'504 '&elementSetName=full'505 '&id=ca1d704a-cdee-4968-aa65-9c353863e4b1'))506 update_file(507 'types/interpretaties/informele_hydrogeologische_stratigrafie/'508 'wfsdescribefeaturetype.xml', build_dov_url(509 'geoserver/interpretaties'510 '/informele_hydrogeologische_stratigrafie/'511 'ows?service=wfs&version=2.0.0&request=DescribeFeatureType'))512 for xsd_schema in InformeleHydrogeologischeStratigrafie.get_xsd_schemas():513 update_file(514 'types/interpretaties/informele_hydrogeologische_stratigrafie/'515 'xsd_%s.xml' % xsd_schema.split('/')[-1], xsd_schema)516 # types/grondwaterfilter517 update_file('types/grondwaterfilter/grondwaterfilter.xml',518 build_dov_url('data/filter/2003-004471.xml'))519 update_file('types/grondwaterfilter/wfsgetfeature.xml',520 build_dov_url('geoserver/ows?service=WFS'521 '&version=2.0.0&request=GetFeature&typeName='522 'gw_meetnetten:meetnetten&maxFeatures=1&'523 'CQL_Filter=filterfiche=%27' + build_dov_url(524 'data/filter/2003-004471%27')))525 update_file('types/grondwaterfilter/feature.xml',526 build_dov_url('geoserver/ows?service=WFS'527 '&version=2.0.0&request=GetFeature&typeName='528 'gw_meetnetten:meetnetten&maxFeatures=1&'529 'CQL_Filter=filterfiche=%27' + build_dov_url(530 'data/filter/2003-004471%27')),531 get_first_featuremember)532 update_file(533 'types/grondwaterfilter/fc_featurecatalogue.xml',534 build_dov_url(535 'geonetwork/srv/dut/csw'536 '?Service=CSW&Request=GetRecordById&Version=2.0.2'537 '&outputSchema=http://www.isotc211.org/2005/gfc'538 '&elementSetName=full&id=b142965f-b2aa-429e-86ff'539 '-a7cb0e065d48'))540 update_file(541 'types/grondwaterfilter/md_metadata.xml',542 build_dov_url(543 'geonetwork/srv/dut/csw'544 '?Service=CSW&Request=GetRecordById&Version=2.0.2'545 '&outputSchema=http://www.isotc211.org/2005/gmd'546 '&elementSetName=full&id=6c39d716-aecc-4fbc-bac8'547 '-4f05a49a78d5'))548 update_file('types/grondwaterfilter/wfsdescribefeaturetype.xml',549 build_dov_url('geoserver/gw_meetnetten/'550 'meetnetten/ows?service=wfs&version=2.0.0&'551 'request=DescribeFeatureType'))552 for xsd_schema in GrondwaterFilter.get_xsd_schemas():553 update_file(554 'types/grondwaterfilter/xsd_%s.xml' % xsd_schema.split('/')[-1],555 xsd_schema)556 update_file('types/grondwaterfilter/grondwaterfilter_geenpeilmeting.xml',557 build_dov_url('data/filter/1976-101132.xml'))558 update_file('types/grondwaterfilter/wfsgetfeature_geenpeilmeting.xml',559 build_dov_url('geoserver/ows?service=WFS'560 '&version=2.0.0&request=GetFeature&typeName='561 'gw_meetnetten:meetnetten&maxFeatures=1&'562 'CQL_Filter=filterfiche=%27' + build_dov_url(563 'data/filter/1976-101132%27')))564 update_file('types/grondwaterfilter/feature_geenpeilmeting.xml',565 build_dov_url('geoserver/ows?service=WFS'566 '&version=2.0.0&request=GetFeature&typeName='567 'gw_meetnetten:meetnetten&maxFeatures=1&'568 'CQL_Filter=filterfiche=%27' + build_dov_url(569 'data/filter/1976-101132%27')),570 get_first_featuremember)571 # types/grondwatermonster572 update_file('types/grondwatermonster/grondwatermonster.xml',573 build_dov_url('data/watermonster/2006-115684.xml'))574 update_file(575 'types/grondwatermonster/wfsgetfeature.xml',576 build_dov_url(577 'geoserver/ows?service=WFS'578 '&version=2.0.0&request=GetFeature&typeName='579 'gw_meetnetten:grondwatermonsters&maxFeatures=1&'580 'CQL_Filter=grondwatermonsterfiche=%27' +581 build_dov_url('data/watermonster/2006-115684') +582 '%27'))583 update_file(584 'types/grondwatermonster/feature.xml',585 build_dov_url(586 'geoserver/ows?service=WFS'587 '&version=2.0.0&request=GetFeature&typeName='588 'gw_meetnetten:grondwatermonsters&maxFeatures=1&'589 'CQL_Filter=grondwatermonsterfiche=%27' +590 build_dov_url('data/watermonster/2006-115684') +591 '%27'),592 get_first_featuremember)593 update_file(594 'types/grondwatermonster/fc_featurecatalogue.xml',595 build_dov_url(596 'geonetwork/srv/dut/csw'597 '?Service=CSW&Request=GetRecordById&Version=2.0.2'598 '&outputSchema=http://www.isotc211.org/2005/gfc'599 '&elementSetName=full&'600 'id=639c9612-4bbb-4826-86fd-fec9afd49bf7'))601 update_file(602 'types/grondwatermonster/md_metadata.xml',603 build_dov_url(604 'geonetwork/srv/dut/csw'605 '?Service=CSW&Request=GetRecordById&Version=2.0.2'606 '&outputSchema=http://www.isotc211.org/2005/gmd'607 '&elementSetName=full&'608 'id=0b378716-39fb-4151-96c5-2021672f4762'))609 update_file(610 'types/grondwatermonster/wfsdescribefeaturetype.xml',611 build_dov_url(612 'geoserver/gw_meetnetten/'613 'grondwatermonsters/ows?service=wfs&version=2.0.0&'614 'request=DescribeFeatureType'))615 for xsd_schema in GrondwaterMonster.get_xsd_schemas():616 update_file(617 'types/grondwatermonster/xsd_%s.xml' % xsd_schema.split('/')[-1],618 xsd_schema)619 # util/owsutil620 update_file(621 'util/owsutil/fc_featurecatalogue_notfound.xml',622 build_dov_url(623 'geonetwork/srv/dut/csw'624 '?Service=CSW&Request=GetRecordById&Version=2.0.2'625 '&outputSchema=http://www.isotc211.org/2005/gfc'626 '&elementSetName=full&id=badfc000-0000-0000-0000'627 '-badfc00badfc'))628 update_file('util/owsutil/wfscapabilities.xml',629 build_dov_url('geoserver/wfs?request'630 '=getcapabilities&service=wfs&version=2.0.0'))631 # types/interpretaties/quartaire_stratigrafie632 update_file('types/interpretaties/quartaire_stratigrafie'633 '/quartaire_stratigrafie.xml',634 build_dov_url('data/interpretatie/1999-057087.xml'))635 update_file(636 'types/interpretaties/quartaire_stratigrafie'637 '/wfsgetfeature.xml',638 build_dov_url(639 'geoserver/ows?service=WFS'640 '&version=2.0.0&request=GetFeature&typeName=interpretaties'641 ':quartaire_stratigrafie&maxFeatures=1&CQL_Filter'642 '=Interpretatiefiche=%27') +643 build_dov_url(644 'data'645 '/interpretatie/1999-057087%27'))646 update_file(647 'types/interpretaties/quartaire_stratigrafie/feature.xml',648 build_dov_url(649 'geoserver/ows?service=WFS'650 '&version=2.0.0&request=GetFeature&typeName=interpretaties'651 ':quartaire_stratigrafie&maxFeatures=1&CQL_Filter'652 '=Interpretatiefiche=%27') +653 build_dov_url(654 'data'655 '/interpretatie/1999-057087%27'),656 get_first_featuremember)657 update_file(658 'types/interpretaties/quartaire_stratigrafie/fc_featurecatalogue.xml',659 build_dov_url(660 'geonetwork/srv/dut/csw'661 '?Service=CSW&Request=GetRecordById&Version=2.0.2'662 '&outputSchema=http://www.isotc211.org/2005/gfc'663 '&elementSetName=full&id=d40ef884-3278-45db-ad69-2c2a8c3981c3'))664 update_file(665 'types/interpretaties/quartaire_stratigrafie/md_metadata.xml',666 build_dov_url(667 'geonetwork/srv/dut/csw'668 '?Service=CSW&Request=GetRecordById&Version=2.0.2'669 '&outputSchema=http://www.isotc211.org/2005/gmd'670 '&elementSetName=full&id=8b204ed6-e44c-4567-bbe8'671 '-bd427eba082c'))672 update_file(673 'types/interpretaties/quartaire_stratigrafie/wfsdescribefeaturetype'674 '.xml', build_dov_url(675 'geoserver/interpretaties'676 '/quartaire_stratigrafie/ows?service=wfs&version=2.0.0&request'677 '=DescribeFeatureType'))678 for xsd_schema in QuartairStratigrafie.get_xsd_schemas():679 update_file(680 'types/interpretaties/quartaire_stratigrafie/xsd_%s.xml' %681 xsd_schema.split('/')[-1], xsd_schema)682 # types/grondmonster683 update_file('types/grondmonster/grondmonster.xml',684 build_dov_url('data/grondmonster/2018-211728.xml'))685 update_file(686 'types/grondmonster/wfsgetfeature.xml',687 build_dov_url(688 'geoserver/ows?service=WFS'689 '&version=2.0.0&request=GetFeature&typeName='690 'boringen:grondmonsters&maxFeatures=1&CQL_Filter'691 '=grondmonsterfiche=%27' +692 build_dov_url(693 'data'694 '/grondmonster/2018-211728') +695 '%27'))696 update_file(697 'types/grondmonster/feature.xml',698 build_dov_url(699 'geoserver/ows?service=WFS'700 '&version=2.0.0&request=GetFeature&typeName='701 'boringen:grondmonsters&maxFeatures=1&CQL_Filter'702 '=grondmonsterfiche=%27' +703 build_dov_url(704 'data'705 '/grondmonster/2018-211728') +706 '%27'),707 get_first_featuremember)708 update_file(709 'types/grondmonster/fc_featurecatalogue.xml',710 build_dov_url(711 'geonetwork/srv/dut/csw'712 '?Service=CSW&Request=GetRecordById&Version=2.0.2'713 '&outputSchema=http://www.isotc211.org/2005/gfc'714 '&elementSetName=full&id=b9338fb5-fc9c-4229-858b-06a5fa3ee49d'))715 update_file(716 'types/grondmonster/md_metadata.xml',717 build_dov_url(718 'geonetwork/srv/dut/csw'719 '?Service=CSW&Request=GetRecordById&Version=2.0.2'720 '&outputSchema=http://www.isotc211.org/2005/gmd'721 '&elementSetName=full&'722 'id=6edeab46-2cfc-4aa2-ae03-307d772f34ae'))723 update_file(724 'types/grondmonster/wfsdescribefeaturetype'725 '.xml',726 build_dov_url('geoserver/boringen'727 '/grondmonsters/ows?service=wfs&version=2.0.0&request'728 '=DescribeFeatureType'))729 for xsd_schema in Grondmonster.get_xsd_schemas():730 update_file(731 'types/grondmonster/xsd_%s.xml' %732 xsd_schema.split('/')[-1], xsd_schema)733 # types/bodemlocatie734 update_file('types/bodemlocatie/bodemlocatie.xml',735 build_dov_url('data/bodemlocatie/2011-000002.xml'))736 update_file(737 'types/bodemlocatie/wfsgetfeature.xml',738 build_dov_url(739 'geoserver/ows?service=WFS'740 '&version=2.0.0&request=GetFeature&typeName=bodem:bodemlocaties'741 '&maxFeatures=1&CQL_Filter=Bodemlocatiefiche=%27' +742 build_dov_url('data/bodemlocatie/2011-000002%27')))743 update_file(744 'types/bodemlocatie/feature.xml',745 build_dov_url(746 'geoserver/ows?service=WFS'747 '&version=2.0.0&request=GetFeature&typeName=bodem:bodemlocaties'748 '&maxFeatures=1&CQL_Filter=Bodemlocatiefiche=%27' +749 build_dov_url('data/bodemlocatie/2011-000002%27')),750 get_first_featuremember)751 update_file(752 'types/bodemlocatie/fc_featurecatalogue.xml',753 build_dov_url(754 'geonetwork/srv/dut/csw'755 '?Service=CSW&Request=GetRecordById&Version=2.0.2'756 '&outputSchema=http://www.isotc211.org/2005/gfc'757 '&elementSetName=full&id=89d4f9a1-0474-4ade-b30f-442c31d17dc6'))758 update_file(759 'types/bodemlocatie/md_metadata.xml',760 build_dov_url(761 'geonetwork/srv/dut/csw'762 '?Service=CSW&Request=GetRecordById&Version=2.0.2'763 '&outputSchema=http://www.isotc211.org/2005/gmd'764 '&elementSetName=full&id=3f507fd9-24c0-40ab-9328-29f0dff571fe'))765 update_file(766 'types/bodemlocatie/wfsdescribefeaturetype.xml',767 build_dov_url(768 'geoserver/bodem/bodemlocaties'769 '/ows?service=wfs&version=2.0.0&request=DescribeFeatureType'))770 for xsd_schema in Bodemlocatie.get_xsd_schemas():771 update_file(772 'types/bodemlocatie/xsd_{}.xml'.format(xsd_schema.split('/')[-1]),773 xsd_schema)774 # types/bodemdiepteinterval775 update_file(776 'types/bodemdiepteinterval/wfsgetfeature.xml',777 build_dov_url(778 'geoserver/ows?service=WFS'779 '&version=2.0.0&request=GetFeature&typeName='780 'bodem:bodemdiepteintervallen&maxFeatures=1&'781 'CQL_Filter=Diepteintervalfiche=%27' +782 build_dov_url('data/bodemdiepteinterval/2018-000004%27')))783 update_file(784 'types/bodemdiepteinterval/feature.xml',785 build_dov_url(786 'geoserver/ows?service=WFS'787 '&version=2.0.0&request=GetFeature&typeName='788 'bodem:bodemdiepteintervallen&maxFeatures=1&'789 'CQL_Filter=Diepteintervalfiche=%27' +790 build_dov_url('data/bodemdiepteinterval/2018-000004%27')),791 get_first_featuremember)792 update_file(793 'types/bodemdiepteinterval/fc_featurecatalogue.xml',794 build_dov_url(795 'geonetwork/srv/dut/csw'796 '?Service=CSW&Request=GetRecordById&Version=2.0.2'797 '&outputSchema=http://www.isotc211.org/2005/gfc'798 '&elementSetName=full&id=859ad05d-d6fc-4850-b040-1bdac3641ff5'))799 update_file(800 'types/bodemdiepteinterval/md_metadata.xml',801 build_dov_url(802 'geonetwork/srv/dut/csw'803 '?Service=CSW&Request=GetRecordById&Version=2.0.2'804 '&outputSchema=http://www.isotc211.org/2005/gmd'805 '&elementSetName=full&id=96a1127f-d7d0-4a33-b24c-04d982b63dce'))806 update_file(807 'types/bodemdiepteinterval/wfsdescribefeaturetype.xml',808 build_dov_url(809 'geoserver/bodem/bodemdiepteintervallen'810 '/ows?service=wfs&version=2.0.0&request=DescribeFeatureType'))811 for xsd_schema in Bodemdiepteinterval.get_xsd_schemas():812 update_file(813 'types/bodemdiepteinterval/xsd_{}.xml'.format(814 xsd_schema.split('/')[-1]),815 xsd_schema)816 # types/bodemobservatie817 update_file('types/bodemobservatie/bodemobservatie.xml',818 build_dov_url('data/bodemobservatie/2019-001221.xml'))819 update_file(820 'types/bodemobservatie/wfsgetfeature.xml',821 build_dov_url(822 'geoserver/ows?service=WFS'823 '&version=2.0.0&request=GetFeature&typeName=bodem:bodemobservaties'824 '&maxFeatures=1&CQL_Filter=Bodemobservatiefiche=%27' +825 build_dov_url('data/bodemobservatie/2019-001221%27')))826 update_file(827 'types/bodemobservatie/feature.xml',828 build_dov_url(829 'geoserver/ows?service=WFS'830 '&version=2.0.0&request=GetFeature&typeName=bodem:bodemobservaties'831 '&maxFeatures=1&CQL_Filter=Bodemobservatiefiche=%27' +832 build_dov_url('data/bodemobservatie/2019-001221%27')),833 get_first_featuremember)834 update_file(835 'types/bodemobservatie/fc_featurecatalogue.xml',836 build_dov_url(837 'geonetwork/srv/dut/csw'838 '?Service=CSW&Request=GetRecordById&Version=2.0.2'839 '&outputSchema=http://www.isotc211.org/2005/gfc'840 '&elementSetName=full&id=44df1272-6b57-471b-9f7a-2dc82f760137'))841 update_file(842 'types/bodemobservatie/md_metadata.xml',843 build_dov_url(844 'geonetwork/srv/dut/csw'845 '?Service=CSW&Request=GetRecordById&Version=2.0.2'846 '&outputSchema=http://www.isotc211.org/2005/gmd'847 '&elementSetName=full&id=dd327fef-62c7-4980-9788-9fac047a1553'))848 update_file(849 'types/bodemobservatie/wfsdescribefeaturetype.xml',850 build_dov_url(851 'geoserver/bodem/bodemobservaties'852 '/ows?service=wfs&version=2.0.0&request=DescribeFeatureType'))853 for xsd_schema in Bodemobservatie.get_xsd_schemas():854 update_file(855 'types/bodemobservatie/xsd_{}.xml'.format(856 xsd_schema.split('/')[-1]),857 xsd_schema)858 # types/bodemmonster859 update_file('types/bodemmonster/bodemmonster.xml',860 build_dov_url('data/bodemmonster/2015-211807.xml'))861 update_file(862 'types/bodemmonster/wfsgetfeature.xml',863 build_dov_url(864 'geoserver/ows?service=WFS'865 '&version=2.0.0&request=GetFeature&typeName=bodem:bodemmonsters'866 '&maxFeatures=1&CQL_Filter=Bodemmonsterfiche=%27' +867 build_dov_url('data/bodemmonster/2015-211807%27')))868 update_file(869 'types/bodemmonster/feature.xml',870 build_dov_url(871 'geoserver/ows?service=WFS'872 '&version=2.0.0&request=GetFeature&typeName=bodem:bodemmonsters'873 '&maxFeatures=1&CQL_Filter=Bodemmonsterfiche=%27' +874 build_dov_url('data/bodemmonster/2015-211807%27')),875 get_first_featuremember)876 update_file(877 'types/bodemmonster/fc_featurecatalogue.xml',878 build_dov_url(879 'geonetwork/srv/dut/csw'880 '?Service=CSW&Request=GetRecordById&Version=2.0.2'881 '&outputSchema=http://www.isotc211.org/2005/gfc'882 '&elementSetName=full&id=7d69c092-fa5a-4399-86ed-003877f5899e'))883 update_file(884 'types/bodemmonster/md_metadata.xml',885 build_dov_url(886 'geonetwork/srv/dut/csw'887 '?Service=CSW&Request=GetRecordById&Version=2.0.2'888 '&outputSchema=http://www.isotc211.org/2005/gmd'889 '&elementSetName=full&id=ff1902b2-7ba2-46be-ba8c-bfcf893444c2'))890 update_file(891 'types/bodemmonster/wfsdescribefeaturetype.xml',892 build_dov_url(893 'geoserver/bodem/bodemmonsters'894 '/ows?service=wfs&version=2.0.0&request=DescribeFeatureType'))895 for xsd_schema in Bodemmonster.get_xsd_schemas():896 update_file(897 'types/bodemmonster/xsd_{}.xml'.format(xsd_schema.split('/')[-1]),898 xsd_schema)899 # types/bodemsite900 update_file('types/bodemsite/bodemsite.xml',901 build_dov_url('data/bodemsite/2013-000180.xml'))902 update_file(903 'types/bodemsite/wfsgetfeature.xml',904 build_dov_url(905 'geoserver/ows?service=WFS'906 '&version=2.0.0&request=GetFeature&typeName=bodem:bodemsites'907 '&maxFeatures=1&CQL_Filter=Bodemsitefiche=%27' +908 build_dov_url('data/bodemsite/2013-000180%27')))909 update_file(910 'types/bodemsite/feature.xml',911 build_dov_url(912 'geoserver/ows?service=WFS'913 '&version=2.0.0&request=GetFeature&typeName=bodem:bodemsites'914 '&maxFeatures=1&CQL_Filter=Bodemsitefiche=%27' +915 build_dov_url('data/bodemsite/2013-000180%27')),916 get_first_featuremember)917 update_file(918 'types/bodemsite/fc_featurecatalogue.xml',919 build_dov_url(920 'geonetwork/srv/dut/csw'921 '?Service=CSW&Request=GetRecordById&Version=2.0.2'922 '&outputSchema=http://www.isotc211.org/2005/gfc'923 '&elementSetName=full&id=955d4bc8-9d78-4af6-9782-85698caae0aa'))924 update_file(925 'types/bodemsite/md_metadata.xml',926 build_dov_url(927 'geonetwork/srv/dut/csw'928 '?Service=CSW&Request=GetRecordById&Version=2.0.2'929 '&outputSchema=http://www.isotc211.org/2005/gmd'930 '&elementSetName=full&id=27142078-f0d0-46e3-b97e-2ffc2c6bdd41'))931 update_file(932 'types/bodemsite/wfsdescribefeaturetype.xml',933 build_dov_url(934 'geoserver/bodem/bodemsites'935 '/ows?service=wfs&version=2.0.0&request=DescribeFeatureType'))936 for xsd_schema in Bodemsite.get_xsd_schemas():937 update_file(938 'types/bodemsite/xsd_{}.xml'.format(xsd_schema.split('/')[-1]),939 xsd_schema)940 # types/bodemclassificatie941 update_file(942 'types/bodemclassificatie/wfsgetfeature.xml',943 build_dov_url(944 'geoserver/ows?service=WFS'945 '&version=2.0.0&request=GetFeature&typeName='946 'bodem:bodemclassificaties&maxFeatures=1&'947 'CQL_Filter=Bodemclassificatiefiche=%27' +948 build_dov_url('data/belgischebodemclassificatie/2018-000146%27')))949 update_file(950 'types/bodemclassificatie/feature.xml',951 build_dov_url(952 'geoserver/ows?service=WFS'953 '&version=2.0.0&request=GetFeature&typeName='954 'bodem:bodemclassificaties&maxFeatures=1&'955 'CQL_Filter=Bodemclassificatiefiche=%27' +956 build_dov_url('data/belgischebodemclassificatie/2018-000146%27')),957 get_first_featuremember)958 update_file(959 'types/bodemclassificatie/fc_featurecatalogue.xml',960 build_dov_url('geonetwork/srv/dut/csw'961 '?Service=CSW&Request=GetRecordById&Version=2.0.2'962 '&outputSchema=http://www.isotc211.org/2005/gfc'963 '&elementSetName=full&'964 'id=9ace8d74-8daa-461f-86bb-273573bc7fa9'))965 update_file('types/bodemclassificatie/md_metadata.xml',966 build_dov_url(967 'geonetwork/srv/dut/csw'968 '?Service=CSW&Request=GetRecordById&Version=2.0.2'969 '&outputSchema=http://www.isotc211.org/2005/gmd'970 '&elementSetName=full&'971 'id=7f668812-edc3-464b-870c-08e964f884b6'))972 update_file('types/bodemclassificatie/wfsdescribefeaturetype.xml',973 build_dov_url(974 'geoserver/bodem/bodemclassificaties'975 '/ows?service=wfs&version=2.0.0'976 '&request=DescribeFeatureType'))977 for xsd_schema in Bodemclassificatie.get_xsd_schemas():978 update_file(979 'types/bodemclassificatie/xsd_%s.xml' %980 xsd_schema.split('/')[-1], xsd_schema)981 # types/gw_vergunningen982 update_file(983 'types/grondwatervergunning/wfsgetfeature.xml',984 build_dov_url(985 'geoserver/ows?service=WFS'986 '&version=2.0.0&request=GetFeature&typeName='987 'gw_vergunningen:alle_verg&maxFeatures=1&CQL_Filter'988 '=id=38598'))989 update_file(990 'types/grondwatervergunning/feature.xml',991 build_dov_url(992 'geoserver/ows?service=WFS'993 '&version=2.0.0&request=GetFeature&typeName='994 'gw_vergunningen:alle_verg&maxFeatures=1&CQL_Filter'995 '=id=38598'), get_first_featuremember)996 update_file(997 'types/grondwatervergunning/fc_featurecatalogue.xml',998 build_dov_url('geonetwork/srv/dut/csw'999 '?Service=CSW&Request=GetRecordById&Version=2.0.2'1000 '&outputSchema=http://www.isotc211.org/2005/gfc'1001 '&elementSetName=full&id=5e605651-'1002 '6b2e-406f-863f-d2eda4d3e534'))1003 update_file('types/grondwatervergunning/md_metadata.xml',1004 build_dov_url(1005 'geonetwork/srv/dut/csw'1006 '?Service=CSW&Request=GetRecordById&Version=2.0.2'1007 '&outputSchema=http://www.isotc211.org/2005/gmd'1008 '&elementSetName=full&id=1b8d9dce-40b2-450c-81ed-'1009 'decb453b3143'))1010 update_file('types/grondwatervergunning/wfsdescribefeaturetype.xml',1011 build_dov_url(1012 'geoserver/gw_vergunningen/alle_verg'1013 '/ows?service=wfs&version=2.0.0'1014 '&request=DescribeFeatureType'))1015 for xsd_schema in GrondwaterVergunning.get_xsd_schemas():1016 update_file(1017 'types/grondwatervergunning/xsd_%s.xml' %1018 xsd_schema.split('/')[-1], xsd_schema)1019 for r in pool.join():1020 if r.get_error() is not None:1021 sys.stdout.write('{}: {}\n'.format(1022 type(r.get_error()).__name__, r.get_error()))1023 sys.exit(1)1024 else:1025 output, failed = r.get_result()1026 sys.stdout.write(output)1027 if failed:...
pubmed_updater.py
Source:pubmed_updater.py
1from data_load.base.data_source_processor import DataSourceProcessor2from data_load.base.data_source_xml import XMLDataSource3from data_load.pubmed.pubmed_relationship_processor import PubmedRelationshipProcessor4from data_load.base.constants import API_URL, ID_PUBMED5from data_load.base.utils.export_doc_ids import get_doc_ids6import data_load.base.utils.file_utils as file_utils7import os8import json9import time10import requests11# from config import *12from email_client import EmailClient13from prospective_citations import FindProspectiveCitations14ALL_PUBMED_IDS_FILE = 'all_pubmed_ids.json'15DIR_DOCS_WITH_NEW_CITATION = 'docs_with_new_citations'16DIR_DOCS_CITATION_HISTORY = 'docs_citation_history'17DIR_PROSPECTS = 'prospects'18DIR_UPDATE_SUMMARY = 'update_summary'19class PubmedUpdater(object):20 def __init__(self, load_manager):21 self.load_manager = load_manager22 # self.file_summaries = {}23 self.existing_pubmed_ids = {}24 # self.docs_with_new_citations = {}25 def get_existing_pmids(self):26 load_config = self.load_manager.get_load_config()27 self.existing_pubmed_ids = get_doc_ids(server=load_config.server,28 src_index=load_config.index,29 src_type=load_config.type,30 dest_dir=load_config.other_files_directory(),31 dest_file_name=ALL_PUBMED_IDS_FILE)32 load_config.logger().info('Existing pmid count: ' + str(len(self.existing_pubmed_ids)))33 def clean_up_data_source_summary(self, data_source_summary):34 updated_ids = data_source_summary['updated_ids']35 indexed_ids = data_source_summary['indexed_ids']36 filtered_indexed_ids = {}37 for pmid in indexed_ids:38 if pmid in self.existing_pubmed_ids:39 updated_ids[pmid] = 040 else:41 filtered_indexed_ids[pmid] = 042 data_source_summary['updated_ids'] = updated_ids43 data_source_summary['indexed_ids'] = filtered_indexed_ids44 return data_source_summary45 def process_file(self, update_file):46 file_name = os.path.basename(update_file)47 load_config = self.load_manager.get_load_config()48 load_config.data_source_name = file_name.split('.')[0]49 load_config.logger().info('Processing file: ' + str(file_name))50 load_config.process_count = 451 data_processor = DataSourceProcessor(52 load_config, XMLDataSource(update_file, 2))53 data_processor.run()54 # Get data load summary55 data_source_summary = data_processor.get_combined_data_source_summary()56 57 # Clean up updated vs indexed ids58 data_source_summary = self.clean_up_data_source_summary(data_source_summary)59 # Save update summary60 self.save_update_summary(data_source_summary, update_file)61 return data_source_summary62 # Process relationships63 # self.process_relationships(update_file, data_source_summary)64 def process_relationships(self, update_file):65 file_name = os.path.basename(update_file)66 data_source_summary = self.load_update_summary(update_file)67 68 load_config = self.load_manager.get_load_config()69 load_config.data_source_name = file_name.split('.')[0] + '_relations'70 load_config.process_count = 471 load_config.append_relations = True72 load_config.source = ''73 74 # print 'Processing relationships......'75 load_config.logger().info('Processing relationships: ' + str(file_name))76 data_processor = PubmedRelationshipProcessor(load_config, XMLDataSource(update_file, 2), data_source_summary)77 data_processor.run()78 79 docs_with_new_citations = data_processor.get_docs_with_new_citations()80 # Save docs with new citations81 self.save_docs_with_new_citations(docs_with_new_citations, update_file)82 # Save docs citation history83 docs_citations_history = data_processor.get_citations_history()84 self.save_docs_citations_history(docs_citations_history, update_file)85 return docs_with_new_citations86 def save_new_pmids(self, update_files):87 load_config = self.load_manager.get_load_config()88 self.existing_pubmed_ids = file_utils.load_file(load_config.other_files_directory(), ALL_PUBMED_IDS_FILE)89 update_summary = self.generate_update_summary(update_files)90 for update_file in update_summary:91 update_summary_for_file = update_summary[update_file]92 articles_processed = update_summary_for_file['articles_processed']93 for _id in articles_processed:94 self.existing_pubmed_ids[_id] = None95 file_utils.save_file(load_config.other_files_directory(), ALL_PUBMED_IDS_FILE, self.existing_pubmed_ids)96 def get_update_records_directory(self, sub_directory=None):97 load_config = self.load_manager.get_load_config()98 other_files_directory = load_config.other_files_directory()99 100 update_records_directory = other_files_directory + '/' + 'update_records'101 if sub_directory is not None:102 update_records_directory += '/' + sub_directory103 file_utils.make_directory(update_records_directory)104 return update_records_directory105 def save_update_summary(self, update_summary, update_file):106 update_record_file_name = self.get_update_summary_file_name(update_file)107 file_utils.save_file(self.get_update_records_directory(DIR_UPDATE_SUMMARY), update_record_file_name, update_summary) 108 def load_update_summary(self, update_file):109 update_record_file_name = self.get_update_summary_file_name(update_file)110 # print 'update_record_file_name', update_record_file_name111 return file_utils.load_file(self.get_update_records_directory(DIR_UPDATE_SUMMARY), update_record_file_name)112 def save_docs_with_new_citations(self, docs_with_new_citations, update_file):113 update_record_file_name = self.get_docs_with_new_citations_file_name(update_file)114 file_utils.save_file(self.get_update_records_directory(DIR_DOCS_WITH_NEW_CITATION), update_record_file_name, docs_with_new_citations) 115 def load_docs_with_new_citations(self, update_file):116 update_record_file_name = self.get_docs_with_new_citations_file_name(update_file)117 return file_utils.load_file(self.get_update_records_directory(DIR_DOCS_WITH_NEW_CITATION), update_record_file_name) 118 def save_docs_citations_history(self, doc_citations_history, update_file):119 update_record_file_name = self.get_docs_citations_history_file_name(update_file)120 file_utils.save_file(self.get_update_records_directory(DIR_DOCS_CITATION_HISTORY), update_record_file_name, doc_citations_history) 121 def get_update_file_name(self, update_file):122 file_name = os.path.basename(update_file)123 name = file_name.split('.')[0]124 return name125 def get_update_summary_file_name(self, update_file):126 name = self.get_update_file_name(update_file)127 file_name = 'update_summary_' + name + '.json'128 return file_name129 def get_docs_with_new_citations_file_name(self, update_file):130 name = self.get_update_file_name(update_file)131 file_name = 'docs_with_new_citations_' + name + '.json'132 return file_name133 def get_docs_citations_history_file_name(self, update_file):134 name = self.get_update_file_name(update_file)135 file_name = 'docs_citations_history_' + name + '.json'136 return file_name137 def get_new_pmids_per_file(self, update_files):138 new_pmids_per_file = {}139 for update_file in update_files:140 indexed_ids = {}141 data_source_summary = self.load_update_summary(update_file)142 batch_indexed_ids = data_source_summary['indexed_ids']143 for _id in batch_indexed_ids:144 if _id not in self.existing_pubmed_ids:145 indexed_ids[_id] = 0146 new_pmids_per_file[update_file] = indexed_ids147 # print 'New pmids', len(indexed_ids)148 return new_pmids_per_file149 def get_total_updated_ids(self, update_files):150 total_updated_ids_per_file = {}151 for update_file in update_files:152 # indexed_ids = {}153 updated_ids = {}154 data_source_summary = self.load_update_summary(update_file)155 batch_indexed_ids = data_source_summary['indexed_ids']156 batch_updated_ids = data_source_summary['updated_ids']157 for _id in batch_indexed_ids:158 updated_ids[_id] = _id159 for _id in batch_updated_ids:160 updated_ids[_id] = _id161 total_updated_ids_per_file[update_file] = updated_ids162 # print 'Total updated pmids', total_updated_ids163 return total_updated_ids_per_file164 def generate_update_summary(self, update_files):165 # Update stats166 total_updated_ids_per_file = self.get_total_updated_ids(update_files)167 new_pmids_per_file = self.get_new_pmids_per_file(update_files)168 update_data = {}169 for update_file in update_files:170 total_updated_ids = {}171 new_pmids = {}172 if update_file in total_updated_ids_per_file:173 total_updated_ids = total_updated_ids_per_file[update_file]174 if update_file in new_pmids_per_file:175 new_pmids = new_pmids_per_file[update_file]176 update_data[update_file] = {177 'articles_processed': total_updated_ids,178 'new_articles': new_pmids179 }180 return update_data181 def get_docs_with_new_citations(self, update_files):182 # Merge docs with new citations from different update files183 docs_with_new_citations = {}184 for update_file in update_files:185 docs_with_new_citations_for_update_file = self.load_docs_with_new_citations(update_file)186 for pmid in docs_with_new_citations_for_update_file:187 if pmid not in docs_with_new_citations:188 docs_with_new_citations[pmid] = []189 new_citations = docs_with_new_citations_for_update_file[pmid]190 docs_with_new_citations[pmid].extend(new_citations)191 192 return docs_with_new_citations193 ...
list.py
Source:list.py
1# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other2# Spack Project Developers. See the top-level COPYRIGHT file for details.3#4# SPDX-License-Identifier: (Apache-2.0 OR MIT)5import pytest6from spack.main import SpackCommand7list = SpackCommand('list')8def test_list():9 output = list()10 assert 'cloverleaf3d' in output11 assert 'hdf5' in output12def test_list_filter():13 output = list('py-*')14 assert 'py-numpy' in output15 assert 'perl-file-copy-recursive' not in output16 output = list('py-')17 assert 'py-numpy' in output18 assert 'perl-file-copy-recursive' in output19@pytest.mark.maybeslow20def test_list_search_description():21 output = list('--search-description', 'xml')22 assert 'expat' in output23def test_list_tags():24 output = list('--tags', 'proxy-app')25 assert 'cloverleaf3d' in output26 assert 'hdf5' not in output27def test_list_format_name_only():28 output = list('--format', 'name_only')29 assert 'cloverleaf3d' in output30 assert 'hdf5' in output31@pytest.mark.maybeslow32def test_list_format_version_json():33 output = list('--format', 'version_json')34 assert ' {"name": "cloverleaf3d",' in output35 assert ' {"name": "hdf5",' in output36 import json37 json.loads(output)38@pytest.mark.maybeslow39def test_list_format_html():40 output = list('--format', 'html')41 assert '<div class="section" id="cloverleaf3d">' in output42 assert '<h1>cloverleaf3d' in output43 assert '<div class="section" id="hdf5">' in output44 assert '<h1>hdf5' in output45def test_list_update(tmpdir):46 update_file = tmpdir.join('output')47 # not yet created when list is run48 list('--update', str(update_file))49 assert update_file.exists()50 with update_file.open() as f:51 assert f.read()52 # created but older than any package53 with update_file.open('w') as f:54 f.write('empty\n')55 update_file.setmtime(0)56 list('--update', str(update_file))57 assert update_file.exists()58 with update_file.open() as f:59 assert f.read() != 'empty\n'60 # newer than any packages61 with update_file.open('w') as f:62 f.write('empty\n')63 list('--update', str(update_file))64 assert update_file.exists()65 with update_file.open() as f:...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run lisa automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
