How to use wait_for_all_workers method in lisa

Best Python code snippet using lisa_python

test_wsgi_latest.py

Source:test_wsgi_latest.py

...57        self.podf = os.path.join(self.revdir, "1491", "_pod.json")58        self.hdlr = None59        self.resp = []60    def tearDown(self):61        self.svc.wait_for_all_workers(300)62        self.tf.clean()63    def gethandler(self, path, env):64        return wsgi.LatestHandler(path, self.svc, env, self.start, "secret")65    def test_do_POST(self):66        req = {67            'REQUEST_METHOD': "POST",68            'CONTENT_TYPE': 'application/json',69            'PATH_INFO': '/pdr/latest',70            'HTTP_AUTHORIZATION': 'Bearer secret'71        }72        self.hdlr = self.gethandler('', req)73        with open(self.podf) as fd:74            req['wsgi.input'] = fd75            body = self.hdlr.handle()76        self.assertIn("201", self.resp[0])77        self.assertEquals(body, [])78        self.assertTrue(os.path.isdir(os.path.join(self.bagparent,"mdbags",self.midasid)))79        self.svc.wait_for_all_workers(300)80        self.assertTrue(os.path.isfile(os.path.join(self.bagparent,"nrdserv",81                                                    self.midasid+".json")))82    def test_do_private_POST(self):83        req = {84            'REQUEST_METHOD': "POST",85            'CONTENT_TYPE': 'application/json',86            'PATH_INFO': '/pdr/latest',87            'HTTP_AUTHORIZATION': 'Bearer secret'88        }89        self.hdlr = self.gethandler('', req)90        pod = None91        with open(self.podf) as fd:92            pod = json.load(fd)93        req['wsgi.input'] = StringIO(json.dumps(pod))94        body = self.hdlr.handle()95        self.assertIn("201", self.resp[0])96        self.assertEquals(body, [])97        bagdir = os.path.join(self.bagparent,"mdbags",self.midasid)98        self.assertTrue(os.path.isdir(bagdir))99        self.svc.wait_for_all_workers(300)100        self.assertTrue(os.path.isfile(os.path.join(self.bagparent,"nrdserv",101                                                    self.midasid+".json")))102        # now turn it private103        pod['accessLevel'] = "non-public"104        self.resp = []105        req['wsgi.input'] = StringIO(json.dumps(pod))106        body = self.hdlr.handle()107        self.assertTrue(not os.path.isdir(bagdir))108        self.svc.wait_for_all_workers(300)109        self.assertTrue(not os.path.isfile(os.path.join(self.bagparent,"nrdserv",110                                                        self.midasid+".json")))111    def test_block_private_clobber(self):112        # first do a normal POST to create a submission113        req = {114            'REQUEST_METHOD': "POST",115            'CONTENT_TYPE': 'application/json',116            'PATH_INFO': '/pdr/latest',117            'HTTP_AUTHORIZATION': 'Bearer secret'118        }119        self.hdlr = self.gethandler('', req)120        pod = None121        with open(self.podf) as fd:122            pod = json.load(fd)123        req['wsgi.input'] = StringIO(json.dumps(pod))124        body = self.hdlr.handle()125        self.assertIn("201", self.resp[0])126        self.assertEquals(body, [])127        bagdir = os.path.join(self.bagparent,"mdbags",self.midasid)128        self.assertTrue(os.path.isdir(bagdir))129        self.svc.wait_for_all_workers(300)130        # now try to clobber it131        self.resp = []132        req = {133            'REQUEST_METHOD': "POST",134            'CONTENT_TYPE': 'application/json',135            'PATH_INFO': '/pdr/latest',136            'HTTP_AUTHORIZATION': 'Bearer secret'137        }138        self.hdlr = self.gethandler('', req)139        pod['identifier'] = ""140        pod['accessLevel'] = "non-public"141        req['wsgi.input'] = StringIO(json.dumps(pod))142        body = self.hdlr.handle()143        self.assertIn("400", self.resp[0])...

test_wsgi_compdelbug.py

Source:test_wsgi_compdelbug.py

...77        self.sipdir = os.path.join(self.upldir, "7213")78        self.hdlr = None79        self.resp = []80    def tearDown(self):81        self.svc.wait_for_all_workers(300)82        self.tf.clean()83    def gethandler(self, path, env):84        return wsgi.LatestHandler(path, self.svc, env, self.start, "secret")85    def test_comp_delete_bug(self):86        req = {87            'REQUEST_METHOD': "POST",88            'CONTENT_TYPE': 'application/json',89            'PATH_INFO': '/pdr/latest',90            'HTTP_AUTHORIZATION': 'Bearer secret'91        }92        self.hdlr = self.gethandler('', req)93        94        podf = os.path.join(self.sipdir, "pod1.json")95        with open(podf) as fd:96            req['wsgi.input'] = fd97            body = self.hdlr.handle()98        self.assertIn("201", self.resp[0])99        self.assertEquals(body, [])100        self.assertTrue(os.path.isdir(self.bagdir))101        self.assertTrue(os.path.isdir(self.mddir))102        self.svc.wait_for_all_workers(300)103        self.assertTrue(os.path.isfile(os.path.join(self.workdir,"nrdserv",104                                                    self.midasid+".json")))105        self.assertTrue(os.path.isdir(os.path.join(self.mddir,106                                                   "RegistryFederationFigure.pptx.sha256")))107        self.assertTrue(os.path.isdir(os.path.join(self.mddir,108                                                   "RegistryFederationFigure.pptx")))109        self.assertTrue(not os.path.exists(os.path.join(self.mddir, "res-md.xsd.sha256")))110        self.assertTrue(not os.path.exists(os.path.join(self.mddir, "res-md.xsd")))111        self.assertTrue(not os.path.exists(os.path.join(self.mddir, "k+_data.txt.sha256")))112        self.assertTrue(not os.path.exists(os.path.join(self.mddir, "k+_data.txt")))113        podf = os.path.join(self.sipdir, "pod2.json")114        with open(podf) as fd:115            req['wsgi.input'] = fd116            body = self.hdlr.handle()117        self.assertIn("201", self.resp[0])118        self.assertEquals(body, [])119        self.svc.wait_for_all_workers(300)120        # new file added but old file was not deleted121        self.assertTrue(os.path.isdir(os.path.join(self.mddir, "res-md.xsd.sha256")))122        self.assertTrue(os.path.isdir(os.path.join(self.mddir, "res-md.xsd")))123        self.assertTrue(os.path.isdir(os.path.join(self.mddir,124                                                   "RegistryFederationFigure.pptx.sha256")))125        self.assertTrue(os.path.isdir(os.path.join(self.mddir,126                                                   "RegistryFederationFigure.pptx")))127        self.assertTrue(not os.path.exists(os.path.join(self.mddir, "k+_data.txt.sha256")))128        self.assertTrue(not os.path.exists(os.path.join(self.mddir, "k+_data.txt")))129        podf = os.path.join(self.sipdir, "pod3.json")130        with open(podf) as fd:131            req['wsgi.input'] = fd132            body = self.hdlr.handle()133        self.assertIn("201", self.resp[0])134        self.assertEquals(body, [])135        self.svc.wait_for_all_workers(300)136        # new file added but old files were not deleted137        self.assertTrue(os.path.isdir(os.path.join(self.mddir, "k+_data.txt.sha256")))138        self.assertTrue(os.path.isdir(os.path.join(self.mddir, "k+_data.txt")))139        self.assertTrue(os.path.isdir(os.path.join(self.mddir, "res-md.xsd.sha256")))140        self.assertTrue(os.path.isdir(os.path.join(self.mddir, "res-md.xsd")))141        self.assertTrue(os.path.isdir(os.path.join(self.mddir,142                                                   "RegistryFederationFigure.pptx.sha256")))143        self.assertTrue(os.path.isdir(os.path.join(self.mddir,144                                                   "RegistryFederationFigure.pptx")))145        146        147        148if __name__ == '__main__':149    test.main()

lsf_cluster_example.py

Source:lsf_cluster_example.py

1"""2Example code to launch a dask cluster, when you aren't using flyemflows.3This example is not related to flyemflows per se.4It's just a simple example for new dask users, showing how to spin5up a dask distributed cluster on LSF, using LSFCluster() from dask_jobqueue.6(Normally when you run a Workflow using flyemflows,7the cluster is automatically started for you, so there is8no need to use this code in that case.)9For an example dask configuration to use with LSFCluster,10see example-dask-config.yaml11To run this example, start ipython like this:12    $ export DASK_CONFIG=example-dask-config.yaml13    $ ipython14And then paste this code into the terminal.15"""16import time17import getpass18import dask19import dask.config20import dask.bag as db21from distributed import Client22def init_cluster(num_workers, wait_for_all_workers=True):23    """24    Start up a dask cluster, optionally wait until all workers have been launched,25    and then return the resulting distributed.Client object.26    27    Args:28        num_workers:29            How many workers to launch.30        wait_for_all_workers:31            If True, pause until all workers have been launched before returning.32            Otherwise, just wait for a single worker to launch.33    34    Returns:35        distributed.Client36    """37    # Local import: LSFCluster probably isn't importable on your local machine,38    # so it's nice to avoid importing it when you're just running local tests without a cluster.39    from dask_jobqueue import LSFCluster40    cluster = LSFCluster(ip='0.0.0.0')41    cluster.scale(num_workers)42    43    required_workers = 144    if wait_for_all_workers:45        required_workers = num_workers46        47    client = Client(cluster)48    while (wait_for_all_workers and49           client.status == "running" and50           len(cluster.scheduler.workers) < required_workers):51        print(f"Waiting for {required_workers - len(cluster.scheduler.workers)} workers...")52        time.sleep(1.0)53    54    print(f'Dashboard running at {client.cluster.dashboard_link}')55    return client56def main():57    USER = getpass.getuser()58    # See example-dask-config.yaml for explanations.59    dask.config.update(dask.config.config,60                       {'jobqueue':61                          {'lsf':62                            {'cores': 1,63                             'memory': '15GB',64                             'walltime': '01:00',65                             'log-directory': 'dask-logs',66                             'local-directory': f'/scratch/{USER}',67                             'use-stdin': True    # Implementation detail regarding how bsub is called by dask-jobqueue.68                                                  # Under Janelia's LSF configuration, this must be set to 'True'.69                       }}})70    71    client = init_cluster(2)72    73    try:74        def double(x):75            return 2*x76    77        bag = db.from_sequence(range(100))78        doubled = bag.map(double).compute()79        print(doubled)80    81    finally:82        client.close()83        client.cluster.close()84if __name__ == "__main__":...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.