How to use run_config method in avocado

Best Python code snippet using avocado_python

trainer.py

Source:trainer.py

...93            2. a model_fn takes following args:94                1. features95                2. param96                3. mode97                4. run_config(optional)98               and returns a `propeller.ModelSpec`99        params: any python object, will pass to your `model_fn` or `propeller.train.Model`100        run_config (propeller.RunConfig): run_config.max_steps should not be None.101        warm_start_setting (propeller.WarmStartSetting): Optional. warm start variable will overwrite model variable.102        """103        if run_config.model_dir is None:104            raise ValueError('model_dir should specified in run_config')105        if inspect.isfunction(model_class_or_model_fn):106            _model_fn = model_class_or_model_fn107        elif issubclass(model_class_or_model_fn, Model):108            _model_fn = _build_model_fn(model_class_or_model_fn)109        else:110            raise ValueError('unknown model %s' % model_class_or_model_fn)111        self.model_fn = _model_fn112        self.params = params113        self.run_config = run_config114        self.warm_start_setting = warm_start_setting115    def _build_for_train(self, train_dataset):116        train_dataset.name = 'train'117        train_program = F.Program()118        startup_prog = F.Program()119        with F.program_guard(train_program, startup_prog):120            with collection.Collections() as collections:121                log.info('Building Train Graph...')122                fea = train_dataset.features()123                model_spec = _build_net(self.model_fn, fea, RunMode.TRAIN, self.params, self.run_config)124                log.info('Building Train Graph: Done')125            scalars = collections.get(collection.Key.SUMMARY_SCALAR)126            histograms = collections.get(collection.Key.SUMMARY_HISTOGRAM)127            skip_optimize_ops = collections.get(collection.Key.SKIP_OPTIMIZE)128            skip_opt = set()129            if skip_optimize_ops is not None:130                skip_opt |= set(skip_optimize_ops)131            if scalars is not None:132                skip_opt |= {t for _, t in scalars}133            if histograms is not None:134                skip_opt |= {t for _, t in histograms}135            skip_opt = list(skip_opt)136        log.info('Train with: \n> Run_config: %s\n> Params: %s\n> Train_model_spec: %s\n' % (repr(137            self.run_config), repr(self.params), repr(model_spec)))138        summary_record = SummaryRecord(139            scalar=collections.get(collection.Key.SUMMARY_SCALAR),140            histogram=collections.get(collection.Key.SUMMARY_HISTOGRAM),141        )142        return ProgramPair(train_program=train_program, startup_program=startup_prog), model_spec, summary_record143    def _build_for_eval(self, ds):144        ds.name = 'eval'145        program = F.Program()146        startup_prog = F.Program()147        with F.program_guard(program, startup_prog):148            #share var with Train net149            log.info('Building Eval Graph')150            fea = ds.features()151            model_spec = _build_net(self.model_fn, fea, RunMode.EVAL, self.params, self.run_config)152            log.info('Done')153        #program = program.clone(for_test=True)154        log.info('Eval with: \n> Run_config: %s\n> Params: %s\n> Train_model_spec: %s\n' % (repr(155            self.run_config), repr(self.params), repr(model_spec)))156        return ProgramPair(train_program=program, startup_program=startup_prog), model_spec157    def _build_for_predict(self, ds):158        ds.name = 'predict'159        program = F.Program()160        startup_prog = F.Program()161        with F.program_guard(program, startup_prog):162            #share var with Train net163            log.info('Building Predict Graph')164            fea = ds.features()165            model_spec = _build_net(self.model_fn, fea, RunMode.PREDICT, self.params, self.run_config)166            log.info('Done')167        #program = program.clone(for_test=True)168        log.info('Predict with: \n> Run_config: %s\n> Params: %s\n> Train_model_spec: %s\n' % (repr(169            self.run_config), repr(self.params), repr(model_spec)))170        return ProgramPair(train_program=program, startup_program=startup_prog), model_spec171    def train(self, train_ds, train_hooks=[]):172        """train on a `Dataset`"""173        if not isinstance(train_ds, Dataset):174            raise ValueError('expect dataset to be instance of Dataset, got %s' % repr(train_ds))175        train_program, model_spec, summary_record = self._build_for_train(train_ds)176        train_run_hooks = [177            hooks.StopAtStepHook(self.run_config.max_steps, self.run_config.run_steps),178            hooks.LoggingHook(179                model_spec.loss,180                summary_record=summary_record,181                summary_writer=_get_summary_writer(os.path.join(self.run_config.model_dir, 'train_history')),182                per_step=self.run_config.log_steps,183                skip_step=self.run_config.skip_steps),184        ]185        if model_spec.train_hooks is not None:186            train_run_hooks.extend(model_spec.train_hooks)187        train_run_hooks.extend(train_hooks)188        train_executor = F.Executor(_get_one_place())189        mon_exe = MonitoredExecutor(190            train_executor,191            train_program,192            loss=model_spec.loss,193            run_config=self.run_config,194            run_hooks=train_run_hooks,195            warm_start_setting=self.warm_start_setting)196        distribution.init_distribuition_env(train_program)  #only initialize distribute training with197        mon_exe.init_or_restore_variables()198        if distribution.status.is_master:199            mon_exe._hooks.append(200                hooks.CheckpointSaverHook(mon_exe._saver, per_step=mon_exe._save_steps, skip_step=mon_exe._skip_steps))201        try:202            with mon_exe:203                for data in train_ds.start():204                    mon_exe.run(feed=data)205        except (StopException, F.core.EOFException) as e:206            pass207        return mon_exe.result208    def evaluate(self, eval_dataset, eval_hooks=[]):209        """eval on a `Dataset`"""210        if not isinstance(eval_dataset, Dataset):211            raise ValueError('expect dataset to be instance of Dataset, got %s' % repr(eval_dataset))212        program, model_spec = self._build_for_eval(eval_dataset)213        single_card_place = _get_one_place()214        eval_executor = F.Executor(single_card_place)215        eval_run_hooks = [216            hooks.StopAtStepHook(self.run_config.eval_max_steps, self.run_config.eval_max_steps),217            hooks.EvalHook(model_spec.metrics, )218        ]219        if model_spec.eval_hooks is not None:220            eval_run_hooks.extend(model_spec.eval_hooks)221        eval_run_hooks.extend(eval_hooks)222        mon_exe = MonitoredExecutor(eval_executor, program, run_config=self.run_config, run_hooks=eval_run_hooks)223        mon_exe.init_or_restore_variables()224        try:225            with mon_exe:226                for data in eval_dataset.start(places=[single_card_place]):227                    mon_exe.run(feed=data)228        except (StopException, F.core.EOFException) as e:229            pass230        _, eval_result = mon_exe.result231        summary_writer = _get_summary_writer(os.path.join(self.run_config.model_dir, 'eval_history'))232        _log_eval_result('eval', eval_result, summary_writer, mon_exe.state)233        return mon_exe.result234    def predict(self, predict_dataset, ckpt=-1, ckpt_path=None, steps=-1, split_batch=True):235        """236        Perform predictoin237        will call `model_fn` and initiate user-specifed model in `propeller.RunMode.PREDICT` mode238        Args:239            infer_dataset (propeller.data.Dataset): should not `shuffle` or `repeat`240            steps (int): steps to predict, if None is specifed,241                will stop when `StopException` is raised in `infer_dataset`242            ckpt_path (None|str): Path of a specific checkpoint to predict.243                If None, the latest checkpoint in model_dir is used.244                If there are no checkpoints in model_dir,245                prediction is run with newly initialized Variables instead of ones restored from checkpoint.246            ckpt (int): deprecated args247            split_batch (bool): if True, prediction of each example in a batch is returned.248        Yields:249            Evaluated values of predictions tensors.250        """251        if not isinstance(predict_dataset, Dataset):252            raise ValueError('expect dataset to be instance of Dataset, got %s' % repr(predict_dataset))253        program, model_spec = self._build_for_predict(predict_dataset)254        single_card_place = _get_one_place()255        executor = F.Executor(single_card_place)256        pred_run_config = RunConfig(run_steps=steps if steps == -1 else None, model_dir=self.run_config.model_dir)257        mon_exe = MonitoredExecutor(258            executor,259            program,260            run_config=pred_run_config,261            warm_start_setting=self.warm_start_setting,262        )263        mon_exe.init_or_restore_variables(ckpt if ckpt_path is None else ckpt_path)264        try:265            with mon_exe:266                log.info('Runining predict from dir: %s' % repr(mon_exe.state))267                single_card_place = _get_one_place()268                for data in predict_dataset.start(places=[single_card_place]):269                    res = mon_exe.run(fetch_list=model_spec.predictions, feed=data)270                    if split_batch:271                        res = map(lambda i: i.tolist(), res)272                        res = zip(*res)  # transpose273                        for r in res:274                            yield r275                    else:276                        yield list(map(lambda i: i.tolist(), res))277        except (StopException, F.core.EOFException) as e:278            pass279def train_and_eval(_placeholder=None,280                   model_class_or_model_fn=None,281                   params=None,282                   run_config=None,283                   train_dataset=None,284                   eval_dataset=None,285                   warm_start_setting=None,286                   train_hooks=[],287                   eval_hooks=[],288                   exporters=[]):289    """290    Perform train and evaluate procesure.291    will call `model_fn` and initiate user-specifed model in `propeller.RunMode.PREDICT` mode292    Args:293        model_class_or_model_fn(callable|propeller.train.Model): `model_class_or_model_fn` be specified in 2 ways:294            1. subclass of propeller.train.Model295            2. a model_fn takes following args: 1. features; 2. param; 3. mode; 4. run_config(optional)296               and returns a `propeller.ModelSpec`297        params: any python object, will pass to your `model_fn` or `propeller.train.Model`298        run_config (propeller.RunConfig): run_config.max_steps should not be None.299        train_dataset (propeller.paddle.data.Dataset): training will stop if global_step > run_config.max_steps.300        eval_dataset (propeller.paddle.data.Dataset|dict): Optional, if Dict of propeller.data.Dataset were specified,301            will perform evluatation on every evaluation sets and report results.302        warm_start_setting (propeller.WarmStartSetting): Optional. warm start variable will overwrite model variable.303        train_hooks (list of propeller.paddle.train.RunHook): Optional.304        eval_hooks (list of propeller.paddle.train.RunHook): Optional.305        exporters (list of propeller.paddle.train.Exporter): Optional.306    """307    if _placeholder is not None:308        raise ValueError('specify keyword args to this function')309    if model_class_or_model_fn is None or params is None or run_config is None or train_dataset is None:...

hcsearchSegRunner.py

Source:hcsearchSegRunner.py

1#!/usr/bin/env python2import argparse3import os4import subprocess5import platform6import time7import traceback8import shutil9# paths to MATLAB10MAC_MATLAB_PATH = "/Applications/MATLAB_R2015b.app/bin/matlab"11WIN_MATLAB_PATH = "C:\\Program Files\\MATLAB\\R2015b\\bin\\matlab.exe"12# training mode13# 0 = use shipped models, no training14# 1 = use shipped training images for training (images not yet uploaded)15# 2 = use provided training images for training (not yet implemented)16TRAIN_OPTION = 017# time bound for HC-Search18TIME_BOUND = 1019# whether to log MATLAB runs (for debugging)20LOG_MATLAB_RUNS = True21def run_matlab_function(func_string, func_name, logs_dir):22    '''Wraps function with try-catch to exit MATLAB on errors'''23    wrapped_func_string = "try;{0};catch exception;disp(getReport(exception));exit(1);end;exit".format(func_string)24    print "executing: {0}".format(wrapped_func_string)25    print26    # check OS and use appropriate command/arguments27    logfile = os.path.join(logs_dir,28        'matlab_run_{0}_{1}.txt'.format(29            int(time.time()),30            func_name))31    application = []32    if platform.system() == 'Darwin': # Mac33        application = [34        MAC_MATLAB_PATH,35        "-nodisplay",36        '-r "{0}"'.format(wrapped_func_string)]37        # for Mac, have to separate arguments like this38        if LOG_MATLAB_RUNS:39            application.append('-logfile "{0}"'.format(logfile))40    elif platform.system() == 'Windows': # Windows41        application = [42        WIN_MATLAB_PATH,43        "-nosplash",44        "-wait",45        "-nodesktop",46        "-minimize",47        "-r",48        '"{0}"'.format(wrapped_func_string)]49        # for Windows, have to separate arguments like this50        if LOG_MATLAB_RUNS:51            application.append("-logfile")52            application.append('"{0}"'.format(logfile))53    elif platform.system() == 'Linux': # Linux54        print "Linux unsupported at this time."55        print56        print "running step Error"57        exit(1)58    else:59        print "Unrecognized OS/Platform: {0}".format(platform.system())60        print61        print "running step Error"62        exit(1)63    # execute MATLAB64    try:65        exit_status = subprocess.call(application)66    except:67        print "Could not successfully open MATLAB. Is this the correct path: "68        if platform.system() == 'Windows':69            print WIN_MATLAB_PATH70        elif platform.system() == 'Darwin':71            print MAC_MATLAB_PATH72        else:73            print "(unknown os)"74        traceback.print_exc()75        exit(1)76    if exit_status != 0:77        print "MATLAB exited with error. ({0})".format(func_name)78        print79        print "running step Error"80        exit(1)81    print "MATLAB exited successfully. ({0})".format(func_name)82def run_hc_search(input_dir, output_dir, time_bound, logs_dir, base_dir, infer_only):83    '''Calls HC-Search command line.'''84    # check OS and use appropriate command/arguments85    stdout_logfile = os.path.join(logs_dir,86        'hcsearch_run_stdout_{0}.txt'.format(int(time.time())))87    stderr_logfile = os.path.join(logs_dir,88        'hcsearch_run_stderr_{0}.txt'.format(int(time.time())))89    application = []90    HC_SEARCH_BASE_PATH = 'nematocyst/'91    if platform.system() == 'Darwin': # Mac92        application = [93        os.path.join(HC_SEARCH_BASE_PATH, 'HCSearch'),94        '{}'.format(input_dir),95        '{}'.format(output_dir),96        str(time_bound)]97        if not infer_only:98            application.append("--learn")99        application.extend(["--infer",100        "--prune",101        "none",102        "--ranker",103        "vw",104        "--successor",105        "flipbit-neighbors",106        "--base-path",107        '{}'.format(base_dir)])108        # for Mac, have to separate arguments like this109    elif platform.system() == 'Windows': # Windows110        application = [111        os.path.join(HC_SEARCH_BASE_PATH, 'HCSearch.exe'),112        '{}'.format(input_dir),113        '{}'.format(output_dir),114        str(time_bound)]115        if not infer_only:116            application.append("--learn")117        application.extend(["--infer",118        "--prune",119        "none",120        "--ranker",121        "vw",122        "--successor",123        "flipbit-neighbors",124        "--base-path",125        '{}'.format(base_dir)])126        # for Windows, have to separate arguments like this127    elif platform.system() == 'Linux': # Linux128        print "Linux unsupported at this time."129        print130        print "running step Error"131        exit(1)132    else:133        print "Unrecognized OS/Platform: {0}".format(platform.system())134        print135        print "running step Error"136        exit(1)137    # application = [HC_SEARCH_PATH, '--help'] # DEBUG TODO138    print "Calling HC-Search command line: {}".format(application)139    print140    # execute HC-Search141    try:142        with open(stdout_logfile, 'w') as stdout:143            with open(stderr_logfile, 'w') as stderr:144                exit_status = subprocess.call(application, stdout=stdout, stderr=stderr)145    except:146        print "Could not successfully call HC-Search command."147        traceback.print_exc()148        exit(1)149    if exit_status != 0:150        print "HC-Search exited with error."151        print152        print "running step Error"153        exit(1)154    print "HC-Search ran without error."155def main():156    '''Main loop'''157    # parse arguments158    parser = argparse.ArgumentParser(description="Launch HC-Search segmentation algorithm")159    parser.add_argument("runConfigFileName", help="path to the runConfig_segmentation.txt file that is generated from the avatol_cv program.")160    args = parser.parse_args()161    run_config_file_name = args.runConfigFileName162    # constants: paths163    THIS_DIR = os.path.dirname(os.path.realpath(__file__))164    THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '3rdParty')165    # constants: keys in run_config_file_name166    TEST_IMAGES_FILE = "testImagesFile"167    SEGMENTATION_OUTPUT_DIR = "segmentationOutputDir"168    TRAIN_IMAGES_FILE = "userProvidedGroundTruthImagesFile"169    GT_IMAGES_FILE = "userProvidedTrainImagesFile"170    #171    # parse run config file172    #173    print "run_config_file_name is {0}".format(run_config_file_name)174    print175    # loop to parse run config file for key-value pairs176    run_config = {}177    with open(run_config_file_name, "r") as f:178        for line in f:179            key, value = line.partition("=")[::2]180            run_config[key.strip()] = value.strip()181    # check that run config file has all the expected key-value pairs182    # expecting: testImagesFile, segmentationOutputDir183    if TEST_IMAGES_FILE not in run_config:184        print "segmentationRunConfig file missing entry for {0}".format(TEST_IMAGES_FILE)185        exit(1)186    if SEGMENTATION_OUTPUT_DIR not in run_config:187        print "segmentationRunConfig file missing entry for {0}".format(SEGMENTATION_OUTPUT_DIR)188        exit(1)189    if TRAIN_IMAGES_FILE not in run_config:190        print "segmentationRunConfig file missing optional entry for {0}".format(TRAIN_IMAGES_FILE)191    if GT_IMAGES_FILE not in run_config:192        print "segmentationRunConfig file missing optional entry for {0}".format(GT_IMAGES_FILE)193    print "run_config['{1}'] is {0}".format(194        run_config[TEST_IMAGES_FILE],195        TEST_IMAGES_FILE)196    print "run_config['{1}'] is {0}".format(197        run_config[SEGMENTATION_OUTPUT_DIR],198        SEGMENTATION_OUTPUT_DIR)199    if TRAIN_IMAGES_FILE in run_config:200        print "run_config['{1}'] is {0}".format(201            run_config[TRAIN_IMAGES_FILE],202            TRAIN_IMAGES_FILE)203    else:204        run_config[TRAIN_IMAGES_FILE] = ""205    if GT_IMAGES_FILE in run_config:206        print "run_config['{1}'] is {0}".format(207            run_config[GT_IMAGES_FILE],208            GT_IMAGES_FILE)209    else:210        run_config[GT_IMAGES_FILE] = ""211    print212    # make logs directory213    logs_dir = os.path.dirname(run_config[TEST_IMAGES_FILE])214    logs_dir = os.path.join(logs_dir, 'logs')215    logs_dir = os.path.join(logs_dir, 'segmentation')216    if not os.path.exists(logs_dir):217        os.makedirs(logs_dir)218    # make temp directory219    print "creating temp directory"220    temp_dir = os.path.dirname(run_config[TEST_IMAGES_FILE])221    temp_dir = os.path.join(temp_dir, 'segmentationTemp')222    images_processed_dir = os.path.join(temp_dir, 'imagesPreprocessed')223    hcsearch_output_dir = os.path.join(temp_dir, 'hcSearchOutput')224    if not os.path.exists(images_processed_dir):225        os.makedirs(images_processed_dir)226    if not os.path.exists(hcsearch_output_dir):227        os.makedirs(hcsearch_output_dir)228    # copy test images to temp folder229    print "copying test images to temp"230    test_temp_dir = os.path.dirname(run_config[TEST_IMAGES_FILE])231    test_temp_dir = os.path.join(test_temp_dir, 'segmentationTemp')232    test_temp_dir = os.path.join(test_temp_dir, 'images')233    if not os.path.exists(test_temp_dir):234        os.makedirs(test_temp_dir)235    with open(run_config[TEST_IMAGES_FILE], 'r') as f:236        for line in f:237            file_name_base = os.path.basename(line.strip())238            shutil.copyfile(line.strip(), os.path.join(test_temp_dir, file_name_base))239    if TRAIN_OPTION == 0:240        # copy shipped model files241        print "copying shipped models to temp"242        shipped_models_dir = os.path.join(THIS_DIR, 'models')243        shipped_models = ["codebook.txt", "edgeclassifier_model.txt",244            "edgeclassifier_training.txt", "initfunc_model.txt",245            "initfunc_training.txt"]246        for m in shipped_models:247            shutil.copyfile(os.path.join(shipped_models_dir, m),248                os.path.join(images_processed_dir, m))249        hcsearch_models_dir = os.path.join(hcsearch_output_dir, 'models')250        if not os.path.exists(hcsearch_models_dir):251            os.makedirs(hcsearch_models_dir)252        shipped_models = ["model_cost.txt", "model_cost.txt.model",253            "model_heuristic.txt", "model_heuristic.txt.model"]254        for m in shipped_models:255            shutil.copyfile(os.path.join(shipped_models_dir, m),256                os.path.join(hcsearch_models_dir, m))257        # set to not train images258        run_config[TRAIN_IMAGES_FILE] = ""259        run_config[GT_IMAGES_FILE] = ""260        shipped_training_images_file = ""261    elif TRAIN_OPTION == 1:262        # copy shipped training images263        print "copying shipped training images to temp"264        train_temp_dir = os.path.dirname(run_config[TEST_IMAGES_FILE])265        train_temp_dir = os.path.join(train_temp_dir, 'segmentationTemp')266        train_temp_dir = os.path.join(train_temp_dir, 'images')267        if not os.path.exists(train_temp_dir):268            os.makedirs(train_temp_dir)269        shipped_training_images_file = os.path.join(THIS_DIR, 'models/', 'training/', 'training_images_list.txt')270        training_images_dir = os.path.join(THIS_DIR, 'models/', 'training/', 'images/')271        gt_images_dir = os.path.join(THIS_DIR, 'models/', 'training/', 'groundtruth/')272        with open(shipped_training_images_file, 'r') as f:273            for line in f:274                file_name_base = line.strip()275                shutil.copyfile(os.path.join(training_images_dir, '{0}.jpg'.format(file_name_base)),276                    os.path.join(train_temp_dir, '{0}.jpg'.format(file_name_base)))277                shutil.copyfile(os.path.join(gt_images_dir, '{0}.jpg'.format(file_name_base)),278                    os.path.join(train_temp_dir, '{0}.jpg'.format(file_name_base)))279    else:280        print "Unknown/unimplemented TRAIN_OPTION: {0}".format(TRAIN_OPTION)281        print282        print "running step Error"283        exit(1)284    #285    #  call matlab...286    #287    matlab_func1 = "preprocess_for_hcsearch('{0}', '{1}', '{2}', '{3}', '{4}', '{5}')".format(288        run_config[SEGMENTATION_OUTPUT_DIR],289        run_config[TEST_IMAGES_FILE],290        run_config[TRAIN_IMAGES_FILE],291        run_config[GT_IMAGES_FILE],292        shipped_training_images_file,293        THIRD_PARTY_DIR)294    print 'running step Preprocessing'295    os.chdir(THIS_DIR)296    run_matlab_function(matlab_func1, "preprocess_for_hcsearch", logs_dir)297    #298    #  call HC-Search...299    #300    base_dir = os.path.join(THIS_DIR, 'nematocyst/')301    infer_only = TRAIN_OPTION == 0302    print 'running step Segmenting'303    os.chdir(THIS_DIR)304    run_hc_search(images_processed_dir, hcsearch_output_dir, TIME_BOUND, logs_dir, base_dir, infer_only)305    #306    #  call matlab...307    #308    matlab_func2 = "postprocess_for_hcsearch('{0}', '{1}', '{2}', '{3}', '{4}')".format(309        run_config[SEGMENTATION_OUTPUT_DIR],310        run_config[TEST_IMAGES_FILE],311        run_config[TRAIN_IMAGES_FILE],312        run_config[GT_IMAGES_FILE],313        TIME_BOUND)314    print 'running step Postprocessing'315    os.chdir(THIS_DIR)316    run_matlab_function(matlab_func2, "postprocess_for_hcsearch", logs_dir)317    #318    #  copy original images319    #320    # copy original images to segmentedData/*_orig.jpg321    print "copying original images"322    with open(run_config[TEST_IMAGES_FILE], 'r') as f:323        for line in f:324            orig_image_path = line.strip()325            old_file_name_base = os.path.splitext(os.path.basename(orig_image_path))[0]326            new_file_name = '{}_orig.jpg'.format(old_file_name_base)327            dest = os.path.join(run_config[SEGMENTATION_OUTPUT_DIR], new_file_name)328            shutil.copyfile(orig_image_path, dest)329    print 'run completed'330# run script (or do nothing on import)331if __name__ == "__main__":...

batSkullScore.py

Source:batSkullScore.py

1#!/usr/bin/env python2import argparse3import os4import subprocess5import platform6import time7import traceback8# paths to MaATLAB9MAC_MATLAB_PATH = "/Applications/MATLAB_R2015b.app/bin/matlab"10WIN_MATLAB_PATH = "C:\\Program Files\\MATLAB\\R2015b\\bin\\matlab.exe"11# whether to log MATLAB runs (for debugging)12LOG_MATLAB_RUNS = True13def remove_cache_directory(cache_dir):14    '''Delete the cache directory'''15    print "cache_dir is {0}".format(cache_dir)16    print17    os.remove(cache_dir)18def run_matlab_function(func_string, func_name, logs_dir):19    '''Wraps function with try-catch to exit MATLAB on errors'''20    wrapped_func_string = "try;{0};catch exception;disp(getReport(exception));exit(1);end;exit".format(func_string)21    print "executing: {0}".format(wrapped_func_string)22    print23    # check OS and use appropriate command/arguments24    logfile = os.path.join(logs_dir,25        'matlab_run_{0}_{1}.txt'.format(26            int(time.time()),27            func_name))28    application = []29    if platform.system() == 'Darwin': # Mac30        application = [31        MAC_MATLAB_PATH,32        "-nodisplay",33        '-r "{0}"'.format(wrapped_func_string)]34        # for Mac, have to separate arguments like this35        if LOG_MATLAB_RUNS:36            application.append('-logfile "{0}"'.format(logfile))37    elif platform.system() == 'Windows': # Windows38        application = [39        WIN_MATLAB_PATH,40        "-nosplash",41        "-wait",42        "-nodesktop",43        "-minimize",44        "-r",45        '"{0}"'.format(wrapped_func_string)]46        # for Windows, have to separate arguments like this47        if LOG_MATLAB_RUNS:48            application.append("-logfile")49            application.append('"{0}"'.format(logfile))50    elif platform.system() == 'Linux': # Linux51        print "Linux unsupported at this time."52        print53        print "running step Error"54        exit(1)55    else:56        print "Unrecognized OS/Platform: {0}".format(platform.system())57        print58        print "running step Error"59        exit(1)60    # execute MATLAB61    try:62        exit_status = subprocess.call(application)63    except:64        print "Could not successfully open MATLAB. Is this the correct path: "65        if platform.system() == 'Windows':66            print WIN_MATLAB_PATH67        elif platform.system() == 'Darwin':68            print MAC_MATLAB_PATH69        else:70            print "(unknown os)"71        traceback.print_exc()72        exit(1)73    if exit_status != 0:74        print "MATLAB exited with error. ({0})".format(func_name)75        print76        print "running step Error"77        exit(1)78    print "MATLAB exited successfully. ({0})".format(func_name)79def main():80    '''Main loop'''81    # parse arguments82    parser = argparse.ArgumentParser(description="Launch bat scoring algorithm")83    parser.add_argument("runConfigFileName", help="path to the runConfig_scoring.txt file that is generated from the avatol_cv program.")84    args = parser.parse_args()85    run_config_file_name = args.runConfigFileName86    # constants: paths87    THIS_DIR = os.path.dirname(os.path.realpath(__file__))88    # THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '3rdParty')89    # constants: keys in run_config_file_name90    TEST_IMAGES_FILE = "testImagesFile"91    TRAINING_DATA_DIR = "trainingDataDir"92    SCORING_OUTPUT_DIR = "scoringOutputDir"93    #94    # parse run config file95    #96    print "run_config_file_name is {0}".format(run_config_file_name)97    print98    # loop to parse run config file for key-value pairs99    run_config = {}100    with open(run_config_file_name, "r") as f:101        for line in f:102            key, value = line.partition("=")[::2]103            run_config[key.strip()] = value.strip()104    # check that run config file has all the expected key-value pairs105    # expecting: testImagesFile, trainingDataDir, scoringOutputDir106    if TEST_IMAGES_FILE not in run_config:107        print "scoringRunConfig file missing entry for {0}".format(TEST_IMAGES_FILE)108        exit(1)109    if TRAINING_DATA_DIR not in run_config:110        print "scoringRunConfig file missing entry for {0}".format(TRAINING_DATA_DIR)111        exit(1)112    if SCORING_OUTPUT_DIR not in run_config:113        print "scoringRunConfig file missing entry for {0}".format(SCORING_OUTPUT_DIR)114        exit(1)115    print "run_config['{1}'] is {0}".format(116        run_config[TEST_IMAGES_FILE],117        TEST_IMAGES_FILE)118    print "run_config['{1}'] is {0}".format(119        run_config[TRAINING_DATA_DIR],120        TRAINING_DATA_DIR)121    print "run_config['{1}'] is {0}".format(122        run_config[SCORING_OUTPUT_DIR],123        SCORING_OUTPUT_DIR)124    print125    # remove cache directory126    cache_dir = os.path.dirname(run_config[TEST_IMAGES_FILE])127    cache_dir = os.path.join(cache_dir, 'legacy_format', 'cache')128    # remove_cache_directory(cache_dir)129    # logs directory130    logs_dir = os.path.dirname(run_config[TEST_IMAGES_FILE])131    logs_dir = os.path.join(logs_dir, 'logs')132    logs_dir = os.path.join(logs_dir, 'scoring')133    if not os.path.exists(logs_dir):134        os.makedirs(logs_dir)135    #136    #  call matlab to translate input137    #138    matlab_func1 = "translate_input('{0}', '{1}', '{2}')".format(139        run_config[SCORING_OUTPUT_DIR],140        run_config[TRAINING_DATA_DIR],141        run_config[TEST_IMAGES_FILE])142    print 'running step Processing Inputs'143    os.chdir(THIS_DIR)144    run_matlab_function(matlab_func1, "translate_input", logs_dir)145    #146    #  call matlab to score147    #148    summary_file = os.path.dirname(run_config[TEST_IMAGES_FILE])149    summary_file = os.path.join(summary_file, 'legacy_format', 'input', 'summary.txt')150    print "summary_file is {0}".format(summary_file)151    matlab_func2 = "invoke_batskull_system('{0}','{1}')".format(152        summary_file,153        "regime2")154    print 'running step Training and Scoring'155    os.chdir(os.path.join('bat','chain_rpm'))156    run_matlab_function(matlab_func2, "invoke_batskull_system", logs_dir)157    os.chdir(THIS_DIR)158    #159    #  call matlab to translate output160    #161    matlab_func3 = "translate_output('{0}', '{1}', '{2}')".format(162        run_config[SCORING_OUTPUT_DIR],163        run_config[TRAINING_DATA_DIR],164        run_config[TEST_IMAGES_FILE])165    print 'running step Processing Outputs'166    os.chdir(THIS_DIR)167    run_matlab_function(matlab_func3, "translate_output", logs_dir)168    print 'run completed'169# run script (or do nothing on import)170if __name__ == "__main__":...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.