How to use collect_process method in tox

Best Python code snippet using tox_python

discovery.py

Source:discovery.py

...259    if s[7] == 'r': mode |= stat.S_IROTH260    if s[8] == 'w': mode |= stat.S_IWOTH261    if s[9] == 'x': mode |= stat.S_IXOTH262    return mode263def collect_process(client, process_info):264    result = client.run(['readlink', '/proc/%d/cwd' % process_info.pid])265    cwd = result.output.strip()266    process = ProcessResource(267        pid=process_info.pid,268        cmdline=process_info.command,269        cwd=cwd)270    process.listen_sockets = get_process_listen_sockets(client, process.pid)271    return process272def collect_file(client, path):273    ls = client.run(['ls', '-ld', '--time-style=full-iso', path])274    if ls.return_code != 0:275        return None276    line = ls.output.split("\n")[0]277    perm, links, owner, group, size, date, time, timezone, name = \278        line.split()279    permissions = permissions_string_to_mode(perm)280    with client.open(path) as f:281        contents = f.read()282    return FileResource(path, contents, owner, group, permissions)283def collect_directory(client, path):284    ls = client.run(['ls', '-ld', '--time-style=full-iso', path])285    if ls.return_code != 0:286        return None287    line = ls.output.split("\n")[0]288    perm, links, owner, group, size, date, time, timezone, name = \289        line.split()290    permissions = permissions_string_to_mode(perm)291    return DirectoryResource(path, owner, group, permissions)292def collect_component_configs(client, component,293                              command, default_config=None):294    config_files = []295    args = shlex.split(command)[1:]296    p = index(args, lambda s: s == '--config-file')297    if p != -1 and p + 1 < len(args):298        config_path = args[p + 1]299    else:300        config_path = default_config301    if config_path:302        r = collect_file(client, config_path)303        if r:304            config_files.append(r)305    p = index(args, lambda s: s == '--config-dir')306    if p != -1 and p + 1 < len(args):307        result = client.run(['ls', '%s/*.conf' % args[p + 1]])308        if result.return_code == 0:309            for config_path in result.output.split("\n"):310                config_files.extend(311                    collect_file(client, config_path))312    component.config_files = config_files313    for i, arg in enumerate(args):314        if arg.startswith('--'):315            name = arg[2:]316            if '=' in name:317                name, value = name.split('=', 1)318            elif i + 1 < len(args):319                value = args[i + 1]320                i += 1321            else:322                continue323            component.config.set_cli(name, value)324# Marker class325class BaseDiscovery(object):326    def __init__(self):327        self._seen_items = []328    def seen(self, driver, host, **data):329        return False330class HostDiscovery(BaseDiscovery):331    item_type = 'host'332    def discover(self, driver, host, **data):333        client = driver.client(host)334        hostname = client.run(['hostname']).output.strip()335        item = HostResource(name=hostname)336        item.id = get_host_id(client)337        item.network_addresses = get_host_network_addresses(client)338        process_sockets = get_listen_sockets(client)339        # Service detection part340        process = find_python_process(client, 'keystone-all')341        if process:342            driver.enqueue(343                'keystone', host=host, pid=process.pid,344                sockets=process_sockets.get(process.pid, []))345        for service in [346            'nova-api', 'nova-volume', 'nova-scheduler',347            'glance-api', 'glance-registry',348            'cinder-api', 'cinder-volume', 'cinder-scheduler',349            'neutron-server', 'neutron-dhcp-agent', 'neutron-l3-agent',350            'neutron-metadata-agent', 'neutron-openvswitch-agent',351            'swift-proxy-server', 'swift-container-server',352            'swift-account-server', 'swift-object-server'353        ]:354            process = find_python_process(client, service)355            if not process:356                continue357            driver.enqueue(358                service, host=host, pid=process.pid,359                sockets=process_sockets.get(process.pid, []))360        for service in ['mysql', 'rabbitmq']:361            process = find_process(client, name=service)362            if not process:363                continue364            driver.enqueue(365                service, host=host, pid=process.pid,366                sockets=process_sockets.get(process.pid, []))367        self._seen_items.append(item)368        return item369    def seen(self, driver, host, **data):370        item = find(self._seen_items, lambda h: host in h.network_addresses)371        return item is not None372class FileSystemDiscovery(BaseDiscovery):373    def seen(self, driver, host, path=None, **data):374        if not path:375            return True376        client = driver.client(host)377        host_id = get_host_id(client)378        item = find(self._seen_items,379                    lambda f: f.path == path and f.host_id == host_id)380        return item is not None381class FileDiscovery(FileSystemDiscovery):382    item_type = 'file'383    def discover(self, driver, host, path=None, **data):384        client = driver.client(host)385        if not path:386            return None387        item = collect_file(client, path)388        if not item:389            return None390        item.host_id = get_host_id(client)391        self._seen_items.append(item)392        return item393class DirectoryDiscovery(FileSystemDiscovery):394    item_type = 'directory'395    def discover(self, driver, host, path=None, **data):396        client = driver.client(host)397        if not path:398            return None399        item = collect_directory(client, path)400        if not item:401            return None402        item.host_id = get_host_id(client)403        self._seen_items.append(item)404        return item405class ServiceDiscovery(BaseDiscovery):406    def seen(self, driver, host, **data):407        if 'sockets' in data:408            item = find(self._seen_items,409                        lambda s: data['sockets'] == s.process.listen_sockets)410        elif 'port' in data:411            item = find(self._seen_items,412                        lambda s: (host, data['port']) in s.process.listen_sockets)413        else:414            client = driver.client(host)415            host_id = client.get_host_id()416            item = find(self._seen_items, lambda s: host_id == s.host_id)417        return item is not None418class KeystoneDiscovery(ServiceDiscovery):419    item_type = 'keystone'420    def discover(self, driver, host, **data):421        client = driver.client(host)422        process = find_python_process(client, 'keystone-all')423        if not process:424            return None425        keystone = KeystoneComponent()426        keystone.host_id = get_host_id(client)427        keystone.process = collect_process(client, process)428        keystone.version = find_python_package_version(client, 'keystone')429        keystone.config_files = []430        collect_component_configs(431            client, keystone, process.command,432            default_config='/etc/keystone/keystone.conf')433        token = keystone.config['admin_token']434        host = keystone.config['bind_host']435        if host == '0.0.0.0':436            host = '127.0.0.1'437        port = int(keystone.config['admin_port'])438        keystone_env = {439            'OS_SERVICE_TOKEN': token,440            'OS_SERVICE_ENDPOINT': 'http://%s:%d/v2.0' % (host, port)441        }442        def db(command):443            return self._get_keystone_db_data(client, command,444                                              env=keystone_env)445        keystone.db = dict()446        keystone.db['tenants'] = db('tenant-list')447        keystone.db['users'] = db('user-list')448        keystone.db['services'] = db('service-list')449        keystone.db['endpoints'] = db('endpoint-list')450        self._seen_items.append(keystone)451        return keystone452    def _get_keystone_db_data(self, client, command, env={}):453        result = client.run(['keystone', command], update_env=env)454        if result.return_code != 0:455            return []456        lines = result.output.strip().split("\n")457        columns = []458        last_pos = 0459        l = lines[0]460        while True:461            pos = l.find('+', last_pos + 1)462            if pos == -1:463                break464            columns.append({'start': last_pos + 1, 'end': pos - 1})465            last_pos = pos466        l = lines[1]467        for c in columns:468            c['name'] = l[c['start']:c['end']].strip()469        data = []470        for l in lines[3:-1]:471            d = dict()472            for c in columns:473                d[c['name']] = l[c['start']:c['end']].strip()474            data.append(d)475        return data476class NovaApiDiscovery(ServiceDiscovery):477    item_type = 'nova-api'478    def discover(self, driver, host, **data):479        client = driver.client(host)480        process = find_python_process(client, 'nova-api')481        if not process:482            return None483        nova_api = NovaApiComponent()484        nova_api.host_id = get_host_id(client)485        nova_api.process = collect_process(client, process)486        nova_api.version = find_python_package_version(client, 'nova')487        collect_component_configs(488            client, nova_api, process.command,489            default_config='/etc/nova/nova.conf')490        config_dir = '/etc/nova'491        if len(nova_api.config_files) > 0:492            config_dir = os.path.dirname(nova_api.config_files[0].path)493        for param in nova_api.config.schema:494            if param.type == 'file':495                path = nova_api.config[param.name]496                driver.enqueue('file', host=host, path=path)497            elif param.type == 'directory':498                path = nova_api.config[param.name]499                driver.enqueue('directory', host=host, path=path)500        paste_config_path = path_relative_to(501            nova_api.config['api_paste_config'], config_dir)502        nova_api.paste_config_file = collect_file(503            client, paste_config_path)504        self._seen_items.append(nova_api)505        return nova_api506class NovaComputeDiscovery(ServiceDiscovery):507    item_type = 'nova-compute'508    def discover(self, driver, host, **data):509        client = driver.client(host)510        process = find_python_process(client, 'nova-compute')511        if not process:512            return None513        nova_compute = NovaComputeComponent()514        nova_compute.host_id = get_host_id(client)515        nova_compute.process = collect_process(client, process)516        nova_compute.version = find_python_package_version(client, 'nova')517        collect_component_configs(518            client, nova_compute, process.command,519            default_config='/etc/nova/nova.conf')520        self._seen_items.append(nova_compute)521        return nova_compute522class NovaSchedulerDiscovery(ServiceDiscovery):523    item_type = 'nova-scheduler'524    def discover(self, driver, host, **data):525        client = driver.client(host)526        process = find_python_process(client, 'nova-scheduler')527        if not process:528            return None529        nova_scheduler = NovaSchedulerComponent()530        nova_scheduler.host_id = get_host_id(client)531        nova_scheduler.process = collect_process(client, process)532        nova_scheduler.version = find_python_package_version(client, 'nova')533        collect_component_configs(534            client, nova_scheduler, process.command,535            default_config='/etc/nova/nova.conf')536        self._seen_items.append(nova_scheduler)537        return nova_scheduler538class GlanceApiDiscovery(ServiceDiscovery):539    item_type = 'glance-api'540    def discover(self, driver, host, **data):541        client = driver.client(host)542        process = find_python_process(client, 'glance-api')543        if not process:544            return None545        glance_api = GlanceApiComponent()546        glance_api.host_id = get_host_id(client)547        glance_api.process = collect_process(client, process)548        glance_api.version = find_python_package_version(client, 'glance')549        collect_component_configs(550            client, glance_api, process.command,551            default_config='/etc/glance/glance.conf')552        self._seen_items.append(glance_api)553        return glance_api554class GlanceRegistryDiscovery(ServiceDiscovery):555    item_type = 'glance-registry'556    def discover(self, driver, host, **data):557        client = driver.client(host)558        process = find_python_process(client, 'glance-registry')559        if not process:560            return None561        glance_registry = GlanceRegistryComponent()562        glance_registry.host_id = get_host_id(client)563        glance_registry.process = collect_process(client, process)564        glance_registry.version = find_python_package_version(client, 'glance')565        collect_component_configs(566            client, glance_registry, process.command,567            default_config='/etc/glance/glance.conf')568        self._seen_items.append(glance_registry)569        return glance_registry570class CinderApiDiscovery(ServiceDiscovery):571    item_type = 'cinder-api'572    def discover(self, driver, host, **data):573        client = driver.client(host)574        process = find_python_process(client, 'cinder-api')575        if not process:576            return None577        cinder_api = CinderApiComponent()578        cinder_api.host_id = get_host_id(client)579        cinder_api.process = collect_process(client, process)580        cinder_api.version = find_python_package_version(client, 'cinder')581        collect_component_configs(582            client, cinder_api, process.command,583            default_config='/etc/cinder/cinder.conf')584        config_dir = '/etc/cinder'585        if len(cinder_api.config_files) > 0:586            config_dir = os.path.dirname(cinder_api.config_files[0].path)587        paste_config_path = path_relative_to(588            cinder_api.config['api_paste_config'], config_dir)589        cinder_api.paste_config_file = collect_file(590            client, paste_config_path)591        self._seen_items.append(cinder_api)592        return cinder_api593class CinderVolumeDiscovery(ServiceDiscovery):594    item_type = 'cinder-volume'595    def discover(self, driver, host, **data):596        client = driver.client(host)597        process = find_python_process(client, 'cinder-volume')598        if not process:599            return None600        cinder_volume = CinderVolumeComponent()601        cinder_volume.host_id = get_host_id(client)602        cinder_volume.process = collect_process(client, process)603        cinder_volume.version = find_python_package_version(client, 'cinder')604        collect_component_configs(605            client, cinder_volume, process.command,606            default_config='/etc/cinder/cinder.conf')607        config_dir = '/etc/cinder'608        if len(cinder_volume.config_files) > 0:609            config_dir = os.path.dirname(cinder_volume.config_files[0].path)610        rootwrap_config_path = path_relative_to(611            cinder_volume.config['rootwrap_config'], config_dir)612        cinder_volume.rootwrap_config = collect_file(613            client, rootwrap_config_path)614        self._seen_items.append(cinder_volume)615        return cinder_volume616class CinderSchedulerDiscovery(ServiceDiscovery):617    item_type = 'cinder-scheduler'618    def discover(self, driver, host, **data):619        client = driver.client(host)620        process = find_python_process(client, 'cinder-scheduler')621        if not process:622            return None623        cinder_scheduler = CinderSchedulerComponent()624        cinder_scheduler.host_id = get_host_id(client)625        cinder_scheduler.process = collect_process(client, process)626        cinder_scheduler.version = find_python_package_version(client,627                                                               'cinder')628        collect_component_configs(629            client, cinder_scheduler, process.command,630            default_config='/etc/cinder/cinder.conf')631        self._seen_items.append(cinder_scheduler)632        return cinder_scheduler633class MysqlDiscovery(ServiceDiscovery):634    item_type = 'mysql'635    def discover(self, driver, host, **data):636        client = driver.client(host)637        process = find_process(client, name='mysqld')638        if not process:639            return None640        mysqld_version_re = re.compile('mysqld\s+Ver\s(\S+)\s')641        mysql = MysqlComponent()642        mysql.host_id = get_host_id(client)643        mysql.process = collect_process(client, process)644        version_result = client.run(['mysqld', '--version'])645        m = mysqld_version_re.match(version_result.output)646        mysql.version = m.group(1) if m else 'unknown'647        mysql.config_files = []648        config_locations_result = client.run(649            ['bash', '-c',650             'mysqld --help --verbose '651             '| grep "Default options are read from" -A 1'])652        config_locations = config_locations_result.output.strip().split(653            "\n")[-1].split()654        for path in config_locations:655            f = collect_file(client, path)656            if f:657                mysql.config_files.append(f)658        self._seen_items.append(mysql)659        return mysql660class RabbitmqDiscovery(ServiceDiscovery):661    item_type = 'rabbitmq'662    def discover(self, driver, host, **data):663        client = driver.client(host)664        process = find_process(client, name='beam.smp')665        if not process:666            process = find_process(client, name='beam')667            if not process:668                return None669        if process.command.find('rabbit') == -1:670            return None671        rabbitmq = RabbitMqComponent()672        rabbitmq.host_id = get_host_id(client)673        rabbitmq.process = collect_process(client, process)674        rabbitmq.version = 'unknown'675        env_file = '/etc/rabbitmq/rabbitmq-env.conf'676        env_vars = {}677        result = client.run(['bash', '-c', 'source %s && set' % env_file])678        if result.return_code == 0:679            lines = result.output.split("\n")680            env_vars = dict((k, v) for k, v in lines.split('=', 1))681        rabbitmq_env_vars = \682            dict((key.replace('RABBITMQ_', ''), value)683                 for key, value in env_vars if key.startswith('RABBITMQ_'))684        for key, value in rabbitmq_env_vars:685            rabbitmq.config.set_env(key, value)686        args = shlex.split(process.command)687        for i, s in enumerate(args):688            if s == '-rabbit' and i + 2 <= len(args):689                rabbitmq.config.set_cli(args[i + 1], args[i + 2])690        self._seen_items.append(rabbitmq)691        return rabbitmq692class SwiftProxyServerDiscovery(ServiceDiscovery):693    item_type = 'swift-proxy-server'694    def discover(self, driver, host, **data):695        client = driver.client(host)696        process = find_python_process(client, 'swift-proxy-server')697        if not process:698            return None699        swift_proxy_server = SwiftProxyServerComponent()700        swift_proxy_server.host_id = get_host_id(client)701        swift_proxy_server.process = collect_process(client, process)702        swift_proxy_server.version = find_python_package_version(client,703                                                                 'swift')704        collect_component_configs(705            client, swift_proxy_server, process.command,706            default_config='/etc/swift/proxy-server.conf')707        self._seen_items.append(swift_proxy_server)708        return swift_proxy_server709class SwiftContainerServerDiscovery(ServiceDiscovery):710    item_type = 'swift-container-server'711    def discover(self, driver, host, **data):712        client = driver.client(host)713        process = find_python_process(client, 'swift-container-server')714        if not process:715            return None716        swift_container_server = SwiftContainerServerComponent()717        swift_container_server.host_id = get_host_id(client)718        swift_container_server.process = collect_process(client, process)719        swift_container_server.version = find_python_package_version(client,720                                                                     'swift')721        collect_component_configs(722            client, swift_container_server, process.command,723            default_config='/etc/swift/container-server/1.conf')724        self._seen_items.append(swift_container_server)725        return swift_container_server726class SwiftAccountServerDiscovery(ServiceDiscovery):727    item_type = 'swift-account-server'728    def discover(self, driver, host, **data):729        client = driver.client(host)730        process = find_python_process(client, 'swift-account-server')731        if not process:732            return None733        swift_account_server = SwiftAccountServerComponent()734        swift_account_server.host_id = get_host_id(client)735        swift_account_server.process = collect_process(client, process)736        swift_account_server.version = find_python_package_version(client,737                                                                   'swift')738        collect_component_configs(739            client, swift_account_server, process.command,740            default_config='/etc/swift/account-server/1.conf')741        self._seen_items.append(swift_account_server)742        return swift_account_server743class SwiftObjectServerDiscovery(ServiceDiscovery):744    item_type = 'swift-object-server'745    def discover(self, driver, host, **data):746        client = driver.client(host)747        process = find_python_process(client, 'swift-object-server')748        if not process:749            return None750        swift_object_server = SwiftObjectServerComponent()751        swift_object_server.host_id = get_host_id(client)752        swift_object_server.process = collect_process(client, process)753        swift_object_server.version = find_python_package_version(client,754                                                                  'swift')755        collect_component_configs(756            client, swift_object_server, process.command,757            default_config='/etc/swift/object-server/1.conf')758        self._seen_items.append(swift_object_server)759        return swift_object_server760class NeutronServerDiscovery(ServiceDiscovery):761    item_type = 'neutron-server'762    def discover(self, driver, host, **data):763        client = driver.client(host)764        process = find_python_process(client, 'neutron-server')765        if not process:766            return None767        neutron_server = NeutronServerComponent()768        neutron_server.host_id = get_host_id(client)769        neutron_server.process = collect_process(client, process)770        neutron_server.version = find_python_package_version(client, 'neutron')771        collect_component_configs(772            client, neutron_server, process.command,773            default_config='/etc/neutron/neutron.conf')774        self._seen_items.append(neutron_server)775        return neutron_server776class NeutronDhcpAgentDiscovery(ServiceDiscovery):777    item_type = 'neutron-dhcp-agent'778    def discover(self, driver, host, **data):779        client = driver.client(host)780        process = find_python_process(client, 'neutron-dhcp-agent')781        if not process:782            return None783        neutron_dhcp_agent = NeutronDhcpAgentComponent()784        neutron_dhcp_agent.host_id = get_host_id(client)785        neutron_dhcp_agent.process = collect_process(client, process)786        neutron_dhcp_agent.version = find_python_package_version(client,787                                                                 'neutron')788        collect_component_configs(789            client, neutron_dhcp_agent, process.command,790            default_config='/etc/neutron/dhcp_agent.ini')791        self._seen_items.append(neutron_dhcp_agent)792        return neutron_dhcp_agent793class NeutronL3AgentDiscovery(ServiceDiscovery):794    item_type = 'neutron-l3-agent'795    def discover(self, driver, host, **data):796        client = driver.client(host)797        process = find_python_process(client, 'neutron-l3-agent')798        if not process:799            return None800        neutron_l3_agent = NeutronL3AgentComponent()801        neutron_l3_agent.host_id = get_host_id(client)802        neutron_l3_agent.process = collect_process(client, process)803        neutron_l3_agent.version = find_python_package_version(client,804                                                               'neutron')805        collect_component_configs(806            client, neutron_l3_agent, process.command,807            default_config='/etc/neutron/l3_agent.ini')808        self._seen_items.append(neutron_l3_agent)809        return neutron_l3_agent810class NeutronMetadataAgentDiscovery(ServiceDiscovery):811    item_type = 'neutron-metadata-agent'812    def discover(self, driver, host, **data):813        client = driver.client(host)814        process = find_python_process(client, 'neutron-metadata-agent')815        if not process:816            return None817        neutron_metadata_agent = NeutronMetadataAgentComponent()818        neutron_metadata_agent.host_id = get_host_id(client)819        neutron_metadata_agent.process = collect_process(client, process)820        neutron_metadata_agent.version = find_python_package_version(client,821                                                                     'neutron')822        collect_component_configs(823            client, neutron_metadata_agent, process.command,824            default_config='/etc/neutron/metadata_agent.ini')825        self._seen_items.append(neutron_metadata_agent)826        return neutron_metadata_agent827class NeutronOpenvswitchAgentDiscovery(ServiceDiscovery):828    item_type = 'neutron-openvswitch-agent'829    def discover(self, driver, host, **data):830        client = driver.client(host)831        process = find_python_process(client, 'neutron-openvswitch-agent')832        if not process:833            return None834        neutron_openvswitch_agent = NeutronOpenvswitchAgentComponent()835        neutron_openvswitch_agent.host_id = get_host_id(client)836        neutron_openvswitch_agent.process = collect_process(client, process)837        neutron_openvswitch_agent.version = find_python_package_version(838            client, 'neutron')839        collect_component_configs(840            client, neutron_openvswitch_agent, process.command,841            default_config='/etc/neutron/plugins/ml2/ml2_conf.ini')842        self._seen_items.append(neutron_openvswitch_agent)843        return neutron_openvswitch_agent844DiscoveryTask = recordtype('DiscoveryTask', ['type', 'host', 'data'])845class DiscoveryDriver(object):846    def __init__(self, defaultPrivateKey):847        self.queue = deque()848        self.defaultPrivateKey = defaultPrivateKey849        self._hosts = {}850        self._clients = {}...

win_service.py

Source:win_service.py

1import socket2import configparser3import os4import logging5import subprocess6from subprocess import Popen7import win32serviceutil8import win32service9import win32event10import servicemanager11from namsd import init12from config import config13import logger14class AppServerSvc(win32serviceutil.ServiceFramework):15    _svc_name_ = "NAMS Service"16    _svc_display_name_ = "NAMS Service"17    _config = configparser.ConfigParser()18    def __init__(self, args):19        win32serviceutil.ServiceFramework.__init__(self, args)20        self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)21        self.hWaitCollect = win32event.CreateEvent(None, 0, 0, "Global\\CollectEvent")22        socket.setdefaulttimeout(60)23        self._logger = logger.create_rotate_logger("NAMSD", logging.DEBUG, "service.log")24    def SvcStop(self):25        try:26            self.stop_process(self.api_process.pid)27            self.stop_process(self.collect_process.pid)28        except Exception:29            pass30        self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)31        win32event.SetEvent(self.hWaitStop)32    def SvcDoRun(self):33        self.ReportServiceStatus(win32service.SERVICE_RUNNING)34        servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,35                              servicemanager.PYS_SERVICE_STARTED,36                              (self._svc_name_, ''))37        self._logger.info("Service is starting")38        self.main()39    def start_process(self, args):40        return Popen(args)41    def stop_process(self, pid):42        subprocess.call(43            ['taskkill', '/F', '/T', '/PID', str(pid)])44    def main(self):45        try:46            init()47        except Exception as e:48            self._logger.info("init exception", e)49        self.api_process = self.start_process(50            [os.path.join(config['path'], 'api.exe')])51        self.collect_process = self.start_process(52            [os.path.join(config['path'], 'collect.exe')])53        while True:54            handles = [self.hWaitStop, self.hWaitCollect]55            rc = win32event.WaitForMultipleObjects(handles, 0, win32event.INFINITE)56            self._logger.info('event received, %d', rc)57            if rc == win32event.WAIT_OBJECT_0:58                break59            elif rc == win32event.WAIT_OBJECT_0+1:60                try:61                    self.stop_process(self.collect_process.pid)62                except:63                    pass64                self.collect_process = self.start_process(65                    [os.path.join(config['path'], 'collect.exe')])...

nested.py

Source:nested.py

1from multiprocessing import Process, Queue2from threading import Thread, activeCount, enumerate3import traceback4from libs.getbsoup import get_b_soup5from libs.logger import Logger6import config7def crawl_thread(url, module):8    Logger.debug('Thread start')9    encoding = getattr(module, 'ENCODING', None)10    try:11        soup = get_b_soup(url, encoding=encoding)12        module.crawl(soup)13    except:14        Logger.error('Crawl error url: ' + url)15        Logger.error(traceback.format_exc())16        return17    Logger.debug('Thread done')18def crawl(module, url_queue):19    #Execute the crawl process20    Logger.info('Start Crawler')21    while True:22        try:23            url = url_queue.get(timeout=config.NESTED_CRAWL_TIMEOUT)24        except:25            #If all threads are done then break the loop, Otherwise continue.26            #Why 2 ? because its need to deduct by the main thread and queue thread,27            # You can comment out the enumerate() line to see what is going on28            #Logger.debug(str(enumerate()))29            if activeCount() <= 2:30                Logger.info('Break crawl')31                break32            else:33                Logger.debug('There are ' + str(activeCount() - 2) + ' threads left')34                continue35        #Spawn a new threads immediate after get the url36        thread = Thread(target=crawl_thread, args=(url, module), name='CrawlThread')37        thread.start()38    Logger.info('Crawl done')39def collect(module, url_queue):40    #Excute the collect process41    Logger.info('Start Collector')42    #Send the url queue to bot via args43    module.collect(url_queue)44    Logger.info('Collect done')45def execute(module, name):46    #Execute the nested crawl method47    Logger.info('Start Execute')48    if not hasattr(module, 'collect'):49        raise NotImplementedError('You must implement collect()')50    #Create the url queue and send to collect and crawl process51    url_queue = Queue()52    collect_process = Process(target=collect, args=(module, url_queue),53                              name=name + '_CollectProcess')54    collect_process.start()55    crawl_process = Process(target=crawl, args=(module, url_queue),56                            name=name + '_CrawlProcess')57    crawl_process.start()58    #Both collect and crawl are safe59    collect_process.join()60    crawl_process.join()...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.