How to use _build_and_install_from_package method in autotest

Best Python code snippet using autotest_python Github


Full Screen

...160 """Subclasses MUST provide their own implementation."""161 raise NotImplementedError162 def _build_and_install_current_dir(self, install_dir):163 """164 Subclasses that use _build_and_install_from_package() MUST provide165 their own implementation of this method.166 """167 raise NotImplementedError168 def build_and_install(self, install_dir):169 """170 Builds and installs the package. It must have been fetched already.171 @param install_dir - The package installation directory. If it does172 not exist it will be created.173 """174 if not self.verified_package:175 raise Error('Must call fetch() first. - %s' % self._check_os_requirements()177 return self._build_and_install(install_dir)178 def _check_os_requirements(self):179 if not self.os_requirements:180 return181 failed = False182 for file_names, package_name in self.os_requirements.iteritems():183 if not any(os.path.exists(file_name) for file_name in file_names):184 failed = True185 logging.error('Can\'t find %s, %s probably needs it.',186 ' or '.join(file_names), logging.error('Perhaps you need to install something similar '188 'to the %s package for OS first.', package_name)189 if failed:190 raise Error('Missing OS requirements for %s. (see above)' %191 def _build_and_install_current_dir_setup_py(self, install_dir):193 """For use as a _build_and_install_current_dir implementation."""194 egg_path = self._build_egg_using_setup_py(setup_py='')195 if not egg_path:196 return False197 return self._install_from_egg(install_dir, egg_path)198 def _build_and_install_current_dir_setupegg_py(self, install_dir):199 """For use as a _build_and_install_current_dir implementation."""200 egg_path = self._build_egg_using_setup_py(setup_py='')201 if not egg_path:202 return False203 return self._install_from_egg(install_dir, egg_path)204 def _build_and_install_current_dir_noegg(self, install_dir):205 if not self._build_using_setup_py():206 return False207 return self._install_using_setup_py_and_rsync(install_dir)208 def _get_extension(self, package):209 """Get extension of package."""210 valid_package_extensions = ['.tar.gz', '.tar.bz2', '.zip']211 extension = None212 for ext in valid_package_extensions:213 if package.endswith(ext):214 extension = ext215 break216 if not extension:217 raise Error('Unexpected package file extension on %s' % package)218 return extension219 def _build_and_install_from_package(self, install_dir):220 """221 This method may be used as a _build_and_install() implementation222 for subclasses if they implement _build_and_install_current_dir().223 Extracts the .tar.gz file, chdirs into the extracted directory224 (which is assumed to match the tar filename) and calls225 _build_and_isntall_current_dir from there.226 Afterwards the build (regardless of failure) extracted .tar.gz227 directory is cleaned up.228 @returns True on success, False otherwise.229 @raises OSError If the expected extraction directory does not exist.230 """231 self._extract_compressed_package()232 extension = self._get_extension(self.verified_package)233 os.chdir(os.path.dirname(self.verified_package))234 os.chdir(self.extracted_package_path)235 extracted_dir = os.getcwd()236 try:237 return self._build_and_install_current_dir(install_dir)238 finally:239 os.chdir(os.path.join(extracted_dir, '..'))240 shutil.rmtree(extracted_dir)241 def _extract_compressed_package(self):242 """Extract the fetched compressed .tar or .zip within its directory."""243 if not self.verified_package:244 raise Error('Package must have been fetched first.')245 os.chdir(os.path.dirname(self.verified_package))246 if self.verified_package.endswith('gz'):247 status = system("tar -xzf '%s'" % self.verified_package)248 elif self.verified_package.endswith('bz2'):249 status = system("tar -xjf '%s'" % self.verified_package)250 elif self.verified_package.endswith('zip'):251 status = system("unzip '%s'" % self.verified_package)252 else:253 raise Error('Unknown compression suffix on %s.' %254 self.verified_package)255 if status:256 raise Error('tar failed with %s' % (status,))257 def _build_using_setup_py(self, setup_py=''):258 """259 Assuming the cwd is the extracted python package, execute a simple260 python build.261 @param setup_py - The name of the file to execute.262 @returns True on success, False otherwise.263 """264 if not os.path.exists(setup_py):265 raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))266 status = system("'%s' %s build" % (sys.executable, setup_py))267 if status:268 logging.error('%s build failed.', return False270 return True271 def _build_egg_using_setup_py(self, setup_py=''):272 """273 Assuming the cwd is the extracted python package, execute a simple274 python bdist_egg.275 @param setup_py - The name of the file to execute.276 @returns The relative path to the resulting egg file or '' on failure.277 """278 if not os.path.exists(setup_py):279 raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))280 egg_subdir = 'dist'281 if os.path.isdir(egg_subdir):282 shutil.rmtree(egg_subdir)283 status = system("'%s' %s bdist_egg" % (sys.executable, setup_py))284 if status:285 logging.error('bdist_egg of setuptools failed.')286 return ''287 # I've never seen a bdist_egg lay multiple .egg files.288 for filename in os.listdir(egg_subdir):289 if filename.endswith('.egg'):290 return os.path.join(egg_subdir, filename)291 def _install_from_egg(self, install_dir, egg_path):292 """293 Install a module from an egg file by unzipping the necessary parts294 into install_dir.295 @param install_dir - The installation directory.296 @param egg_path - The pathname of the egg file.297 """298 status = system("unzip -q -o -d '%s' '%s'" % (install_dir, egg_path))299 if status:300 logging.error('unzip of %s failed', egg_path)301 return False302 egg_info_dir = os.path.join(install_dir, 'EGG-INFO')303 if os.path.isdir(egg_info_dir):304 egg_info_new_path = self._get_egg_info_path(install_dir)305 if egg_info_new_path:306 if os.path.exists(egg_info_new_path):307 shutil.rmtree(egg_info_new_path)308 os.rename(egg_info_dir, egg_info_new_path)309 else:310 shutil.rmtree(egg_info_dir)311 return True312 def _get_egg_info_path(self, install_dir):313 """Get egg-info path for this package.314 Example path: install_dir/MySQL_python-1.2.3.egg-info315 """316 if self.dist_name:317 egg_info_name_part = self.dist_name.replace('-', '_')318 if self.version:319 egg_info_filename = '%s-%s.egg-info' % (egg_info_name_part,320 self.version)321 else:322 egg_info_filename = '%s.egg-info' % (egg_info_name_part,)323 return os.path.join(install_dir, egg_info_filename)324 else:325 return None326 def _get_temp_dir(self):327 return tempfile.mkdtemp(dir='/var/tmp')328 def _site_packages_path(self, temp_dir):329 # This makes assumptions about what python install330 # does when given a prefix. Is this always correct?331 python_xy = 'python%s' % sys.version[:3]332 return os.path.join(temp_dir, 'lib', python_xy, 'site-packages')333 def _rsync (self, temp_site_dir, install_dir):334 """Rsync contents. """335 status = system("rsync -r '%s/' '%s/'" %336 (os.path.normpath(temp_site_dir),337 os.path.normpath(install_dir)))338 if status:339 logging.error('%s rsync to install_dir failed.', return False341 return True342 def _install_using_setup_py_and_rsync(self, install_dir,343 setup_py='',344 temp_dir=None):345 """346 Assuming the cwd is the extracted python package, execute a simple:347 python install --prefix=BLA348 BLA will be a temporary directory that everything installed will349 be picked out of and rsynced to the appropriate place under350 install_dir afterwards.351 Afterwards, it deconstructs the extra lib/pythonX.Y/site-packages/352 directory tree that setuptools created and moves all installed353 site-packages directly up into install_dir itself.354 @param install_dir the directory for the install to happen under.355 @param setup_py - The name of the file to execute.356 @returns True on success, False otherwise.357 """358 if not os.path.exists(setup_py):359 raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))360 if temp_dir is None:361 temp_dir = self._get_temp_dir()362 try:363 status = system("'%s' %s install --no-compile --prefix='%s'"364 % (sys.executable, setup_py, temp_dir))365 if status:366 logging.error('%s install failed.', return False368 if os.path.isdir(os.path.join(temp_dir, 'lib')):369 # NOTE: This ignores anything outside of the lib/ dir that370 # was installed.371 temp_site_dir = self._site_packages_path(temp_dir)372 else:373 temp_site_dir = temp_dir374 return self._rsync(temp_site_dir, install_dir)375 finally:376 shutil.rmtree(temp_dir)377 def _build_using_make(self, install_dir):378 """Build the current package using configure/make.379 @returns True on success, False otherwise.380 """381 install_prefix = os.path.join(install_dir, 'usr', 'local')382 status = system('./configure --prefix=%s' % install_prefix)383 if status:384 logging.error('./configure failed for %s', return False386 status = system('make')387 if status:388 logging.error('make failed for %s', return False390 status = system('make check')391 if status:392 logging.error('make check failed for %s', return False394 return True395 def _install_using_make(self):396 """Install the current package using make install.397 Assumes the install path was set up while running ./configure (in398 _build_using_make()).399 @returns True on success, False otherwise.400 """401 status = system('make install')402 return status == 0403 def fetch(self, dest_dir):404 """405 Fetch the package from one its URLs and save it in dest_dir.406 If the the package already exists in dest_dir and the checksum407 matches this code will not fetch it again.408 Sets the 'verified_package' attribute with the destination pathname.409 @param dest_dir - The destination directory to save the local file.410 If it does not exist it will be created.411 @returns A boolean indicating if we the package is now in dest_dir.412 @raises FetchError - When something unexpected happens.413 """414 if not os.path.exists(dest_dir):415 os.makedirs(dest_dir)416 local_path = os.path.join(dest_dir, self.local_filename)417 # If the package exists, verify its checksum and be happy if it is good.418 if os.path.exists(local_path):419 actual_hex_sum = _checksum_file(local_path)420 if self.hex_sum == actual_hex_sum:421'Good checksum for existing %s package.',422 self.verified_package = local_path424 return True425 logging.warning('Bad checksum for existing %s package. '426 'Re-downloading', os.rename(local_path, local_path + '.wrong-checksum')428 # Download the package from one of its urls, rejecting any if the429 # checksum does not match.430 for url in self.urls:431'Fetching %s', url)432 try:433 url_file = urllib2.urlopen(url)434 except (urllib2.URLError, EnvironmentError):435 logging.warning('Could not fetch %s package from %s.',436, url)437 continue438 data_length = int('Content-Length',439 _MAX_PACKAGE_SIZE))440 if data_length <= 0 or data_length > _MAX_PACKAGE_SIZE:441 raise FetchError('%s from %s fails Content-Length %d '442 'sanity check.' % (, url,443 data_length))444 checksum = utils.hash('sha1')445 total_read = 0446 output = open(local_path, 'wb')447 try:448 while total_read < data_length:449 data = if not data:451 break452 output.write(data)453 checksum.update(data)454 total_read += len(data)455 finally:456 output.close()457 if self.hex_sum != checksum.hexdigest():458 logging.warning('Bad checksum for %s fetched from %s.',459, url)460 logging.warning('Got %s', checksum.hexdigest())461 logging.warning('Expected %s', self.hex_sum)462 os.unlink(local_path)463 continue464'Good checksum.')465 self.verified_package = local_path466 return True467 else:468 return False469# NOTE: This class definition must come -before- all other ExternalPackage470# classes that need to use this version of setuptools so that is is inserted471# into the ExternalPackage.subclasses list before them.472class SetuptoolsPackage(ExternalPackage):473 """setuptools package"""474 # For all known setuptools releases a string compare works for the475 # version string. Hopefully they never release a 0.10. (Their own476 # version comparison code would break if they did.)477 # Any system with setuptools > 18.0.1 is fine. If none installed, then478 # try to install the latest found on the upstream.479 minimum_version = '18.0.1'480 version = '18.0.1'481 urls = (_CHROMEOS_MIRROR + 'setuptools-%s.tar.gz' % (version,),)482 local_filename = 'setuptools-%s.tar.gz' % version483 hex_sum = 'ebc4fe81b7f6d61d923d9519f589903824044f52'484 SUDO_SLEEP_DELAY = 15485 def _build_and_install(self, install_dir):486 """Install setuptools on the system."""487'NOTE: setuptools install does not use install_dir.')488 return self._build_and_install_from_package(install_dir)489 def _build_and_install_current_dir(self, install_dir):490 egg_path = self._build_egg_using_setup_py()491 if not egg_path:492 return False493 print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n'494 print 'About to run sudo to install setuptools', self.version495 print 'on your system for use by', sys.executable, '\n'496 print '!! ^C within', self.SUDO_SLEEP_DELAY, 'seconds to abort.\n'497 time.sleep(self.SUDO_SLEEP_DELAY)498 # Copy the egg to the local filesystem /var/tmp so that root can499 # access it properly (avoid NFS squashroot issues).500 temp_dir = self._get_temp_dir()501 try:502 shutil.copy(egg_path, temp_dir)503 egg_name = os.path.split(egg_path)[1]504 temp_egg = os.path.join(temp_dir, egg_name)505 p = subprocess.Popen(['sudo', '/bin/sh', temp_egg],506 stdout=subprocess.PIPE)507 regex = re.compile('Copying (.*?) to (.*?)\n')508 match =[0])509 status = p.wait()510 if match:511 compiled = os.path.join(, os.system("sudo chmod a+r '%s'" % compiled)513 finally:514 shutil.rmtree(temp_dir)515 if status:516 logging.error('install of setuptools from egg failed.')517 return False518 return True519class MySQLdbPackage(ExternalPackage):520 """mysql package, used in scheduler."""521 module_name = 'MySQLdb'522 version = '1.2.3'523 local_filename = 'MySQL-python-%s.tar.gz' % version524 urls = (''525 'distfiles/%s' % local_filename,)526 hex_sum = '3511bb8c57c6016eeafa531d5c3ea4b548915e3c'527 _build_and_install_current_dir = (528 ExternalPackage._build_and_install_current_dir_setup_py)529 def _build_and_install(self, install_dir):530 if not os.path.exists('/usr/bin/mysql_config'):531 error_msg = '''\532You need to install /usr/bin/mysql_config.533On recent Debian based distros, run: \534sudo apt-get install libmariadbclient-dev-compat535On older Debian based distros, run: sudo apt-get install libmysqlclient15-dev536'''537 logging.error(error_msg)538 return False, error_msg539 return self._build_and_install_from_package(install_dir)540class DjangoPackage(ExternalPackage):541 """django package."""542 version = '1.5.1'543 local_filename = 'Django-%s.tar.gz' % version544 urls = (_CHROMEOS_MIRROR + local_filename,)545 hex_sum = '0ab97b90c4c79636e56337f426f1e875faccbba1'546 _build_and_install = ExternalPackage._build_and_install_from_package547 _build_and_install_current_dir = (548 ExternalPackage._build_and_install_current_dir_noegg)549 def _get_installed_version_from_module(self, module):550 try:551 return module.get_version().split()[0]552 except AttributeError:553 return '0.9.6'554class NumpyPackage(ExternalPackage):555 """numpy package, required by matploglib."""556 version = '1.7.0'557 local_filename = 'numpy-%s.tar.gz' % version558 urls = (_CHROMEOS_MIRROR + local_filename,)559 hex_sum = 'ba328985f20390b0f969a5be2a6e1141d5752cf9'560 _build_and_install = ExternalPackage._build_and_install_from_package561 _build_and_install_current_dir = (562 ExternalPackage._build_and_install_current_dir_setupegg_py)563class JsonRPCLib(ExternalPackage):564 """jsonrpclib package"""565 version = '0.1.3'566 module_name = 'jsonrpclib'567 local_filename = '%s-%s.tar.gz' % (module_name, version)568 urls = (_CHROMEOS_MIRROR + local_filename,)569 hex_sum = '431714ed19ab677f641ce5d678a6a95016f5c452'570 def _get_installed_version_from_module(self, module):571 # jsonrpclib doesn't contain a proper version572 return self.version573 _build_and_install = ExternalPackage._build_and_install_from_package574 _build_and_install_current_dir = (575 ExternalPackage._build_and_install_current_dir_noegg)576class GwtPackage(ExternalPackage):577 """Fetch and extract a local copy of GWT used to build the frontend."""578 version = '2.3.0'579 local_filename = '' % version580 urls = (_CHROMEOS_MIRROR + local_filename,)581 hex_sum = 'd51fce9166e6b31349659ffca89baf93e39bc84b'582 name = 'gwt'583 about_filename = 'about.txt'584 module_name = None # Not a Python module.585 def is_needed(self, install_dir):586 gwt_dir = os.path.join(install_dir, about_file = os.path.join(install_dir,, self.about_filename)588 if not os.path.exists(gwt_dir) or not os.path.exists(about_file):589'gwt not installed for autotest')590 return True591 f = open(about_file, 'r')592 version_line = f.readline()593 f.close()594 match = re.match(r'Google Web Toolkit (.*)', version_line)595 if not match:596'did not find gwt version')597 return True598'found gwt version %s', return != self.version600 def _build_and_install(self, install_dir):601 os.chdir(install_dir)602 self._extract_compressed_package()603 extracted_dir = self.local_filename[:-len('.zip')]604 target_dir = os.path.join(install_dir, if os.path.exists(target_dir):606 shutil.rmtree(target_dir)607 os.rename(extracted_dir, target_dir)608 return True609class PyudevPackage(ExternalPackage):610 """611 pyudev module612 Used in unittests.613 """614 version = '0.16.1'615 url_filename = 'pyudev-%s.tar.gz' % version616 local_filename = url_filename617 urls = (_CHROMEOS_MIRROR + local_filename,)618 hex_sum = 'b36bc5c553ce9b56d32a5e45063a2c88156771c0'619 _build_and_install = ExternalPackage._build_and_install_from_package620 _build_and_install_current_dir = (621 ExternalPackage._build_and_install_current_dir_setup_py)622class PyMoxPackage(ExternalPackage):623 """624 mox module625 Used in unittests.626 """627 module_name = 'mox'628 version = '0.5.3'629 # Note: url_filename does not match local_filename, because of630 # an uncontrolled fork at some point in time of mox versions.631 url_filename = 'mox-%s-autotest.tar.gz' % version632 local_filename = 'mox-%s.tar.gz' % version633 urls = (_CHROMEOS_MIRROR + url_filename,)634 hex_sum = '1c502d2c0a8aefbba2c7f385a83d33e7d822452a'635 _build_and_install = ExternalPackage._build_and_install_from_package636 _build_and_install_current_dir = (637 ExternalPackage._build_and_install_current_dir_noegg)638 def _get_installed_version_from_module(self, module):639 # mox doesn't contain a proper version640 return self.version641class PySeleniumPackage(ExternalPackage):642 """643 selenium module644 Used in wifi_interop suite.645 """646 module_name = 'selenium'647 version = '2.37.2'648 url_filename = 'selenium-%s.tar.gz' % version649 local_filename = url_filename650 urls = (_CHROMEOS_MIRROR + local_filename,)651 hex_sum = '66946d5349e36d946daaad625c83c30c11609e36'652 _build_and_install = ExternalPackage._build_and_install_from_package653 _build_and_install_current_dir = (654 ExternalPackage._build_and_install_current_dir_setup_py)655class FaultHandlerPackage(ExternalPackage):656 """657 faulthandler module658 """659 module_name = 'faulthandler'660 version = '2.3'661 url_filename = '%s-%s.tar.gz' % (module_name, version)662 local_filename = url_filename663 urls = (_CHROMEOS_MIRROR + local_filename,)664 hex_sum = 'efb30c068414fba9df892e48fcf86170cbf53589'665 _build_and_install = ExternalPackage._build_and_install_from_package666 _build_and_install_current_dir = (667 ExternalPackage._build_and_install_current_dir_noegg)668class PsutilPackage(ExternalPackage):669 """670 psutil module671 """672 module_name = 'psutil'673 version = '2.1.1'674 url_filename = '%s-%s.tar.gz' % (module_name, version)675 local_filename = url_filename676 urls = (_CHROMEOS_MIRROR + local_filename,)677 hex_sum = '0c20a20ed316e69f2b0881530439213988229916'678 _build_and_install = ExternalPackage._build_and_install_from_package679 _build_and_install_current_dir = (680 ExternalPackage._build_and_install_current_dir_setup_py)681class ElasticSearchPackage(ExternalPackage):682 """elasticsearch-py package."""683 version = '1.6.0'684 url_filename = 'elasticsearch-%s.tar.gz' % version685 local_filename = url_filename686 urls = ('' %687 (url_filename),)688 hex_sum = '3e676c96f47935b1f52df82df3969564bd356b1c'689 _build_and_install = ExternalPackage._build_and_install_from_package690 _build_and_install_current_dir = (691 ExternalPackage._build_and_install_current_dir_setup_py)692 def _get_installed_version_from_module(self, module):693 # Elastic's version format is like tuple (1, 6, 0), which needs to be694 # transferred to try:696 return '.'.join(str(i) for i in module.__version__)697 except:698 return self.version699class Urllib3Package(ExternalPackage):700 """elasticsearch-py package."""701 version = '1.9'702 url_filename = 'urllib3-%s.tar.gz' % version703 local_filename = url_filename704 urls = (_CHROMEOS_MIRROR + local_filename,)705 hex_sum = '9522197efb2a2b49ce804de3a515f06d97b6602f'706 _build_and_install = ExternalPackage._build_and_install_from_package707 _build_and_install_current_dir = (708 ExternalPackage._build_and_install_current_dir_setup_py)709class ImagingLibraryPackage(ExternalPackage):710 """Python Imaging Library (PIL)."""711 version = '1.1.7'712 url_filename = 'Imaging-%s.tar.gz' % version713 local_filename = url_filename714 urls = (''715 'distfiles/%s' % url_filename,)716 hex_sum = '76c37504251171fda8da8e63ecb8bc42a69a5c81'717 def _build_and_install(self, install_dir):718 #The path of zlib library might be different from what PIL is719 #expected. Following change does the best attempt to link the library720 #to a path PIL will try.721 libz_possible_path = '/usr/lib/x86_64-linux-gnu/'722 libz_expected_path = '/usr/lib/'723 if (os.path.exists(libz_possible_path) and724 not os.path.exists(libz_expected_path)):725'sudo ln -s %s %s' %726 (libz_possible_path, libz_expected_path))727 return self._build_and_install_from_package(install_dir)728 _build_and_install_current_dir = (729 ExternalPackage._build_and_install_current_dir_noegg)730class AstroidPackage(ExternalPackage):731 """astroid package."""732 version = '1.5.3'733 url_filename = 'astroid-%s.tar.gz' % version734 local_filename = url_filename735 urls = (_CHROMEOS_MIRROR + local_filename,)736 hex_sum = 'e654225ab5bd2788e5e246b156910990bf33cde6'737 _build_and_install = ExternalPackage._build_and_install_from_package738 _build_and_install_current_dir = (739 ExternalPackage._build_and_install_current_dir_setup_py)740class LazyObjectProxyPackage(ExternalPackage):741 """lazy-object-proxy package (dependency for astroid)."""...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:


You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run autotest automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?