How to use _build_and_install method in lisa

Best Python code snippet using lisa_python
external_packages.py
Source:external_packages.py
...155            return module.__version__156        except AttributeError:157            logging.error('could not get version from %s', module)158            return ''159    def _build_and_install(self, install_dir):160        """Subclasses MUST provide their own implementation."""161        raise NotImplementedError162    def _build_and_install_current_dir(self, install_dir):163        """164        Subclasses that use _build_and_install_from_package() MUST provide165        their own implementation of this method.166        """167        raise NotImplementedError168    def build_and_install(self, install_dir):169        """170        Builds and installs the package.  It must have been fetched already.171        @param install_dir - The package installation directory.  If it does172            not exist it will be created.173        """174        if not self.verified_package:175            raise Error('Must call fetch() first.  - %s' % self.name)176        self._check_os_requirements()177        return self._build_and_install(install_dir)178    def _check_os_requirements(self):179        if not self.os_requirements:180            return181        failed = False182        for file_names, package_name in self.os_requirements.iteritems():183            if not any(os.path.exists(file_name) for file_name in file_names):184                failed = True185                logging.error('Can\'t find %s, %s probably needs it.',186                              ' or '.join(file_names), self.name)187                logging.error('Perhaps you need to install something similar '188                              'to the %s package for OS first.', package_name)189        if failed:190            raise Error('Missing OS requirements for %s.  (see above)' %191                        self.name)192    def _build_and_install_current_dir_setup_py(self, install_dir):193        """For use as a _build_and_install_current_dir implementation."""194        egg_path = self._build_egg_using_setup_py(setup_py='setup.py')195        if not egg_path:196            return False197        return self._install_from_egg(install_dir, egg_path)198    def _build_and_install_current_dir_setupegg_py(self, install_dir):199        """For use as a _build_and_install_current_dir implementation."""200        egg_path = self._build_egg_using_setup_py(setup_py='setupegg.py')201        if not egg_path:202            return False203        return self._install_from_egg(install_dir, egg_path)204    def _build_and_install_current_dir_noegg(self, install_dir):205        if not self._build_using_setup_py():206            return False207        return self._install_using_setup_py_and_rsync(install_dir)208    def _get_extension(self, package):209        """Get extension of package."""210        valid_package_extensions = ['.tar.gz', '.tar.bz2', '.zip']211        extension = None212        for ext in valid_package_extensions:213            if package.endswith(ext):214                extension = ext215                break216        if not extension:217            raise Error('Unexpected package file extension on %s' % package)218        return extension219    def _build_and_install_from_package(self, install_dir):220        """221        This method may be used as a _build_and_install() implementation222        for subclasses if they implement _build_and_install_current_dir().223        Extracts the .tar.gz file, chdirs into the extracted directory224        (which is assumed to match the tar filename) and calls225        _build_and_isntall_current_dir from there.226        Afterwards the build (regardless of failure) extracted .tar.gz227        directory is cleaned up.228        @returns True on success, False otherwise.229        @raises OSError If the expected extraction directory does not exist.230        """231        self._extract_compressed_package()232        extension = self._get_extension(self.verified_package)233        os.chdir(os.path.dirname(self.verified_package))234        os.chdir(self.extracted_package_path)235        extracted_dir = os.getcwd()236        try:237            return self._build_and_install_current_dir(install_dir)238        finally:239            os.chdir(os.path.join(extracted_dir, '..'))240            shutil.rmtree(extracted_dir)241    def _extract_compressed_package(self):242        """Extract the fetched compressed .tar or .zip within its directory."""243        if not self.verified_package:244            raise Error('Package must have been fetched first.')245        os.chdir(os.path.dirname(self.verified_package))246        if self.verified_package.endswith('gz'):247            status = system("tar -xzf '%s'" % self.verified_package)248        elif self.verified_package.endswith('bz2'):249            status = system("tar -xjf '%s'" % self.verified_package)250        elif self.verified_package.endswith('zip'):251            status = system("unzip '%s'" % self.verified_package)252        else:253            raise Error('Unknown compression suffix on %s.' %254                        self.verified_package)255        if status:256            raise Error('tar failed with %s' % (status,))257    def _build_using_setup_py(self, setup_py='setup.py'):258        """259        Assuming the cwd is the extracted python package, execute a simple260        python setup.py build.261        @param setup_py - The name of the setup.py file to execute.262        @returns True on success, False otherwise.263        """264        if not os.path.exists(setup_py):265            raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))266        status = system("'%s' %s build" % (sys.executable, setup_py))267        if status:268            logging.error('%s build failed.', self.name)269            return False270        return True271    def _build_egg_using_setup_py(self, setup_py='setup.py'):272        """273        Assuming the cwd is the extracted python package, execute a simple274        python setup.py bdist_egg.275        @param setup_py - The name of the setup.py file to execute.276        @returns The relative path to the resulting egg file or '' on failure.277        """278        if not os.path.exists(setup_py):279            raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))280        egg_subdir = 'dist'281        if os.path.isdir(egg_subdir):282            shutil.rmtree(egg_subdir)283        status = system("'%s' %s bdist_egg" % (sys.executable, setup_py))284        if status:285            logging.error('bdist_egg of setuptools failed.')286            return ''287        # I've never seen a bdist_egg lay multiple .egg files.288        for filename in os.listdir(egg_subdir):289            if filename.endswith('.egg'):290                return os.path.join(egg_subdir, filename)291    def _install_from_egg(self, install_dir, egg_path):292        """293        Install a module from an egg file by unzipping the necessary parts294        into install_dir.295        @param install_dir - The installation directory.296        @param egg_path - The pathname of the egg file.297        """298        status = system("unzip -q -o -d '%s' '%s'" % (install_dir, egg_path))299        if status:300            logging.error('unzip of %s failed', egg_path)301            return False302        egg_info_dir = os.path.join(install_dir, 'EGG-INFO')303        if os.path.isdir(egg_info_dir):304            egg_info_new_path = self._get_egg_info_path(install_dir)305            if egg_info_new_path:306                if os.path.exists(egg_info_new_path):307                    shutil.rmtree(egg_info_new_path)308                os.rename(egg_info_dir, egg_info_new_path)309            else:310                shutil.rmtree(egg_info_dir)311        return True312    def _get_egg_info_path(self, install_dir):313        """Get egg-info path for this package.314        Example path: install_dir/MySQL_python-1.2.3.egg-info315        """316        if self.dist_name:317            egg_info_name_part = self.dist_name.replace('-', '_')318            if self.version:319                egg_info_filename = '%s-%s.egg-info' % (egg_info_name_part,320                                                        self.version)321            else:322                egg_info_filename = '%s.egg-info' % (egg_info_name_part,)323            return os.path.join(install_dir, egg_info_filename)324        else:325            return None326    def _get_temp_dir(self):327        return tempfile.mkdtemp(dir='/var/tmp')328    def _site_packages_path(self, temp_dir):329        # This makes assumptions about what python setup.py install330        # does when given a prefix.  Is this always correct?331        python_xy = 'python%s' % sys.version[:3]332        return os.path.join(temp_dir, 'lib', python_xy, 'site-packages')333    def _rsync (self, temp_site_dir, install_dir):334        """Rsync contents. """335        status = system("rsync -r '%s/' '%s/'" %336                        (os.path.normpath(temp_site_dir),337                         os.path.normpath(install_dir)))338        if status:339            logging.error('%s rsync to install_dir failed.', self.name)340            return False341        return True342    def _install_using_setup_py_and_rsync(self, install_dir,343                                          setup_py='setup.py',344                                          temp_dir=None):345        """346        Assuming the cwd is the extracted python package, execute a simple:347          python setup.py install --prefix=BLA348        BLA will be a temporary directory that everything installed will349        be picked out of and rsynced to the appropriate place under350        install_dir afterwards.351        Afterwards, it deconstructs the extra lib/pythonX.Y/site-packages/352        directory tree that setuptools created and moves all installed353        site-packages directly up into install_dir itself.354        @param install_dir the directory for the install to happen under.355        @param setup_py - The name of the setup.py file to execute.356        @returns True on success, False otherwise.357        """358        if not os.path.exists(setup_py):359            raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))360        if temp_dir is None:361            temp_dir = self._get_temp_dir()362        try:363            status = system("'%s' %s install --no-compile --prefix='%s'"364                            % (sys.executable, setup_py, temp_dir))365            if status:366                logging.error('%s install failed.', self.name)367                return False368            if os.path.isdir(os.path.join(temp_dir, 'lib')):369                # NOTE: This ignores anything outside of the lib/ dir that370                # was installed.371                temp_site_dir = self._site_packages_path(temp_dir)372            else:373                temp_site_dir = temp_dir374            return self._rsync(temp_site_dir, install_dir)375        finally:376            shutil.rmtree(temp_dir)377    def _build_using_make(self, install_dir):378        """Build the current package using configure/make.379        @returns True on success, False otherwise.380        """381        install_prefix = os.path.join(install_dir, 'usr', 'local')382        status = system('./configure --prefix=%s' % install_prefix)383        if status:384            logging.error('./configure failed for %s', self.name)385            return False386        status = system('make')387        if status:388            logging.error('make failed for %s', self.name)389            return False390        status = system('make check')391        if status:392            logging.error('make check failed for %s', self.name)393            return False394        return True395    def _install_using_make(self):396        """Install the current package using make install.397        Assumes the install path was set up while running ./configure (in398        _build_using_make()).399        @returns True on success, False otherwise.400        """401        status = system('make install')402        return status == 0403    def fetch(self, dest_dir):404        """405        Fetch the package from one its URLs and save it in dest_dir.406        If the the package already exists in dest_dir and the checksum407        matches this code will not fetch it again.408        Sets the 'verified_package' attribute with the destination pathname.409        @param dest_dir - The destination directory to save the local file.410            If it does not exist it will be created.411        @returns A boolean indicating if we the package is now in dest_dir.412        @raises FetchError - When something unexpected happens.413        """414        if not os.path.exists(dest_dir):415            os.makedirs(dest_dir)416        local_path = os.path.join(dest_dir, self.local_filename)417        # If the package exists, verify its checksum and be happy if it is good.418        if os.path.exists(local_path):419            actual_hex_sum = _checksum_file(local_path)420            if self.hex_sum == actual_hex_sum:421                logging.info('Good checksum for existing %s package.',422                             self.name)423                self.verified_package = local_path424                return True425            logging.warning('Bad checksum for existing %s package.  '426                            'Re-downloading', self.name)427            os.rename(local_path, local_path + '.wrong-checksum')428        # Download the package from one of its urls, rejecting any if the429        # checksum does not match.430        for url in self.urls:431            logging.info('Fetching %s', url)432            try:433                url_file = urllib2.urlopen(url)434            except (urllib2.URLError, EnvironmentError):435                logging.warning('Could not fetch %s package from %s.',436                                self.name, url)437                continue438            data_length = int(url_file.info().get('Content-Length',439                                                  _MAX_PACKAGE_SIZE))440            if data_length <= 0 or data_length > _MAX_PACKAGE_SIZE:441                raise FetchError('%s from %s fails Content-Length %d '442                                 'sanity check.' % (self.name, url,443                                                    data_length))444            checksum = utils.hash('sha1')445            total_read = 0446            output = open(local_path, 'wb')447            try:448                while total_read < data_length:449                    data = url_file.read(_READ_SIZE)450                    if not data:451                        break452                    output.write(data)453                    checksum.update(data)454                    total_read += len(data)455            finally:456                output.close()457            if self.hex_sum != checksum.hexdigest():458                logging.warning('Bad checksum for %s fetched from %s.',459                                self.name, url)460                logging.warning('Got %s', checksum.hexdigest())461                logging.warning('Expected %s', self.hex_sum)462                os.unlink(local_path)463                continue464            logging.info('Good checksum.')465            self.verified_package = local_path466            return True467        else:468            return False469# NOTE: This class definition must come -before- all other ExternalPackage470# classes that need to use this version of setuptools so that is is inserted471# into the ExternalPackage.subclasses list before them.472class SetuptoolsPackage(ExternalPackage):473    """setuptools package"""474    # For all known setuptools releases a string compare works for the475    # version string.  Hopefully they never release a 0.10.  (Their own476    # version comparison code would break if they did.)477    # Any system with setuptools > 18.0.1 is fine. If none installed, then478    # try to install the latest found on the upstream.479    minimum_version = '18.0.1'480    version = '18.0.1'481    urls = (_CHROMEOS_MIRROR + 'setuptools-%s.tar.gz' % (version,),)482    local_filename = 'setuptools-%s.tar.gz' % version483    hex_sum = 'ebc4fe81b7f6d61d923d9519f589903824044f52'484    SUDO_SLEEP_DELAY = 15485    def _build_and_install(self, install_dir):486        """Install setuptools on the system."""487        logging.info('NOTE: setuptools install does not use install_dir.')488        return self._build_and_install_from_package(install_dir)489    def _build_and_install_current_dir(self, install_dir):490        egg_path = self._build_egg_using_setup_py()491        if not egg_path:492            return False493        print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n'494        print 'About to run sudo to install setuptools', self.version495        print 'on your system for use by', sys.executable, '\n'496        print '!! ^C within', self.SUDO_SLEEP_DELAY, 'seconds to abort.\n'497        time.sleep(self.SUDO_SLEEP_DELAY)498        # Copy the egg to the local filesystem /var/tmp so that root can499        # access it properly (avoid NFS squashroot issues).500        temp_dir = self._get_temp_dir()501        try:502            shutil.copy(egg_path, temp_dir)503            egg_name = os.path.split(egg_path)[1]504            temp_egg = os.path.join(temp_dir, egg_name)505            p = subprocess.Popen(['sudo', '/bin/sh', temp_egg],506                                 stdout=subprocess.PIPE)507            regex = re.compile('Copying (.*?) to (.*?)\n')508            match = regex.search(p.communicate()[0])509            status = p.wait()510            if match:511                compiled = os.path.join(match.group(2), match.group(1))512                os.system("sudo chmod a+r '%s'" % compiled)513        finally:514            shutil.rmtree(temp_dir)515        if status:516            logging.error('install of setuptools from egg failed.')517            return False518        return True519class MySQLdbPackage(ExternalPackage):520    """mysql package, used in scheduler."""521    module_name = 'MySQLdb'522    version = '1.2.3'523    local_filename = 'MySQL-python-%s.tar.gz' % version524    urls = ('http://commondatastorage.googleapis.com/chromeos-mirror/gentoo/'525            'distfiles/%s' % local_filename,)526    hex_sum = '3511bb8c57c6016eeafa531d5c3ea4b548915e3c'527    _build_and_install_current_dir = (528            ExternalPackage._build_and_install_current_dir_setup_py)529    def _build_and_install(self, install_dir):530        if not os.path.exists('/usr/bin/mysql_config'):531            error_msg = '''\532You need to install /usr/bin/mysql_config.533On recent Debian based distros, run: \534sudo apt-get install libmariadbclient-dev-compat535On older Debian based distros, run: sudo apt-get install libmysqlclient15-dev536'''537            logging.error(error_msg)538            return False, error_msg539        return self._build_and_install_from_package(install_dir)540class DjangoPackage(ExternalPackage):541    """django package."""542    version = '1.5.1'543    local_filename = 'Django-%s.tar.gz' % version544    urls = (_CHROMEOS_MIRROR + local_filename,)545    hex_sum = '0ab97b90c4c79636e56337f426f1e875faccbba1'546    _build_and_install = ExternalPackage._build_and_install_from_package547    _build_and_install_current_dir = (548            ExternalPackage._build_and_install_current_dir_noegg)549    def _get_installed_version_from_module(self, module):550        try:551            return module.get_version().split()[0]552        except AttributeError:553            return '0.9.6'554class NumpyPackage(ExternalPackage):555    """numpy package, required by matploglib."""556    version = '1.7.0'557    local_filename = 'numpy-%s.tar.gz' % version558    urls = (_CHROMEOS_MIRROR + local_filename,)559    hex_sum = 'ba328985f20390b0f969a5be2a6e1141d5752cf9'560    _build_and_install = ExternalPackage._build_and_install_from_package561    _build_and_install_current_dir = (562            ExternalPackage._build_and_install_current_dir_setupegg_py)563class JsonRPCLib(ExternalPackage):564    """jsonrpclib package"""565    version = '0.1.3'566    module_name = 'jsonrpclib'567    local_filename = '%s-%s.tar.gz' % (module_name, version)568    urls = (_CHROMEOS_MIRROR + local_filename,)569    hex_sum = '431714ed19ab677f641ce5d678a6a95016f5c452'570    def _get_installed_version_from_module(self, module):571        # jsonrpclib doesn't contain a proper version572        return self.version573    _build_and_install = ExternalPackage._build_and_install_from_package574    _build_and_install_current_dir = (575                        ExternalPackage._build_and_install_current_dir_noegg)576class GwtPackage(ExternalPackage):577    """Fetch and extract a local copy of GWT used to build the frontend."""578    version = '2.3.0'579    local_filename = 'gwt-%s.zip' % version580    urls = (_CHROMEOS_MIRROR + local_filename,)581    hex_sum = 'd51fce9166e6b31349659ffca89baf93e39bc84b'582    name = 'gwt'583    about_filename = 'about.txt'584    module_name = None  # Not a Python module.585    def is_needed(self, install_dir):586        gwt_dir = os.path.join(install_dir, self.name)587        about_file = os.path.join(install_dir, self.name, self.about_filename)588        if not os.path.exists(gwt_dir) or not os.path.exists(about_file):589            logging.info('gwt not installed for autotest')590            return True591        f = open(about_file, 'r')592        version_line = f.readline()593        f.close()594        match = re.match(r'Google Web Toolkit (.*)', version_line)595        if not match:596            logging.info('did not find gwt version')597            return True598        logging.info('found gwt version %s', match.group(1))599        return match.group(1) != self.version600    def _build_and_install(self, install_dir):601        os.chdir(install_dir)602        self._extract_compressed_package()603        extracted_dir = self.local_filename[:-len('.zip')]604        target_dir = os.path.join(install_dir, self.name)605        if os.path.exists(target_dir):606            shutil.rmtree(target_dir)607        os.rename(extracted_dir, target_dir)608        return True609class PyudevPackage(ExternalPackage):610    """611    pyudev module612    Used in unittests.613    """614    version = '0.16.1'615    url_filename = 'pyudev-%s.tar.gz' % version616    local_filename = url_filename617    urls = (_CHROMEOS_MIRROR + local_filename,)618    hex_sum = 'b36bc5c553ce9b56d32a5e45063a2c88156771c0'619    _build_and_install = ExternalPackage._build_and_install_from_package620    _build_and_install_current_dir = (621                        ExternalPackage._build_and_install_current_dir_setup_py)622class PyMoxPackage(ExternalPackage):623    """624    mox module625    Used in unittests.626    """627    module_name = 'mox'628    version = '0.5.3'629    # Note: url_filename does not match local_filename, because of630    # an uncontrolled fork at some point in time of mox versions.631    url_filename = 'mox-%s-autotest.tar.gz' % version632    local_filename = 'mox-%s.tar.gz' % version633    urls = (_CHROMEOS_MIRROR + url_filename,)634    hex_sum = '1c502d2c0a8aefbba2c7f385a83d33e7d822452a'635    _build_and_install = ExternalPackage._build_and_install_from_package636    _build_and_install_current_dir = (637                        ExternalPackage._build_and_install_current_dir_noegg)638    def _get_installed_version_from_module(self, module):639        # mox doesn't contain a proper version640        return self.version641class PySeleniumPackage(ExternalPackage):642    """643    selenium module644    Used in wifi_interop suite.645    """646    module_name = 'selenium'647    version = '2.37.2'648    url_filename = 'selenium-%s.tar.gz' % version649    local_filename = url_filename650    urls = (_CHROMEOS_MIRROR + local_filename,)651    hex_sum = '66946d5349e36d946daaad625c83c30c11609e36'652    _build_and_install = ExternalPackage._build_and_install_from_package653    _build_and_install_current_dir = (654                        ExternalPackage._build_and_install_current_dir_setup_py)655class FaultHandlerPackage(ExternalPackage):656    """657    faulthandler module658    """659    module_name = 'faulthandler'660    version = '2.3'661    url_filename = '%s-%s.tar.gz' % (module_name, version)662    local_filename = url_filename663    urls = (_CHROMEOS_MIRROR + local_filename,)664    hex_sum = 'efb30c068414fba9df892e48fcf86170cbf53589'665    _build_and_install = ExternalPackage._build_and_install_from_package666    _build_and_install_current_dir = (667            ExternalPackage._build_and_install_current_dir_noegg)668class PsutilPackage(ExternalPackage):669    """670    psutil module671    """672    module_name = 'psutil'673    version = '2.1.1'674    url_filename = '%s-%s.tar.gz' % (module_name, version)675    local_filename = url_filename676    urls = (_CHROMEOS_MIRROR + local_filename,)677    hex_sum = '0c20a20ed316e69f2b0881530439213988229916'678    _build_and_install = ExternalPackage._build_and_install_from_package679    _build_and_install_current_dir = (680                        ExternalPackage._build_and_install_current_dir_setup_py)681class ElasticSearchPackage(ExternalPackage):682    """elasticsearch-py package."""683    version = '1.6.0'684    url_filename = 'elasticsearch-%s.tar.gz' % version685    local_filename = url_filename686    urls = ('https://pypi.python.org/packages/source/e/elasticsearch/%s' %687            (url_filename),)688    hex_sum = '3e676c96f47935b1f52df82df3969564bd356b1c'689    _build_and_install = ExternalPackage._build_and_install_from_package690    _build_and_install_current_dir = (691            ExternalPackage._build_and_install_current_dir_setup_py)692    def _get_installed_version_from_module(self, module):693        # Elastic's version format is like tuple (1, 6, 0), which needs to be694        # transferred to 1.6.0.695        try:696            return '.'.join(str(i) for i in module.__version__)697        except:698            return self.version699class Urllib3Package(ExternalPackage):700    """elasticsearch-py package."""701    version = '1.9'702    url_filename = 'urllib3-%s.tar.gz' % version703    local_filename = url_filename704    urls = (_CHROMEOS_MIRROR + local_filename,)705    hex_sum = '9522197efb2a2b49ce804de3a515f06d97b6602f'706    _build_and_install = ExternalPackage._build_and_install_from_package707    _build_and_install_current_dir = (708            ExternalPackage._build_and_install_current_dir_setup_py)709class ImagingLibraryPackage(ExternalPackage):710     """Python Imaging Library (PIL)."""711     version = '1.1.7'712     url_filename = 'Imaging-%s.tar.gz' % version713     local_filename = url_filename714     urls = ('http://commondatastorage.googleapis.com/chromeos-mirror/gentoo/'715             'distfiles/%s' % url_filename,)716     hex_sum = '76c37504251171fda8da8e63ecb8bc42a69a5c81'717     def _build_and_install(self, install_dir):718         #The path of zlib library might be different from what PIL setup.py is719         #expected. Following change does the best attempt to link the library720         #to a path PIL setup.py will try.721         libz_possible_path = '/usr/lib/x86_64-linux-gnu/libz.so'722         libz_expected_path = '/usr/lib/libz.so'723         if (os.path.exists(libz_possible_path) and724             not os.path.exists(libz_expected_path)):725             utils.run('sudo ln -s %s %s' %726                       (libz_possible_path, libz_expected_path))727         return self._build_and_install_from_package(install_dir)728     _build_and_install_current_dir = (729             ExternalPackage._build_and_install_current_dir_noegg)730class AstroidPackage(ExternalPackage):731    """astroid package."""...
Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.