How to use config_files method in toolium

Best Python code snippet using toolium_python

eval_sweeps.py

Source:eval_sweeps.py

1import os2import numpy as np3import sys4import argparse5sys.path.append(os.path.abspath('../../'))6from sweeper import Sweeper7def extract_line(lines, max_steps, interval=0):8    """9    Retrieves num_steps at which episodes were completed10    and returns an array where the value at index i represents11    the number of episodes completed until step i12    """13    step = 014    # rewards_over_time = np.zeros(max_steps//interval+1) # for GANModel training15    rewards_over_time = np.zeros(max_steps // interval + 1) # for DQN training16    try:17        for line in lines:18            if 'total steps' not in line:19                continue20            num_steps = float(line.split("|")[1].split(",")[0].split(" ")[-1])21            if num_steps > max_steps:22                break23            # reward = float(line.split("|")[1].split(",")[1].split(" ")[2]) # for GANModel training24            reward = float(line.split("|")[1].split(",")[2].split("/")[0].split(" ")[-1]) # for DQN training25            rewards_over_time[step] = reward26            step += 127        return rewards_over_time28    except:29        print(line)30        print('step:{}'.format(step))31        raise32def get_max_steps(lines):33    for line in lines[::-1]:34        if 'total steps' in line:35            max_steps = float(line.split("|")[1].split(",")[0].split(" ")[-1])36            return max_steps37    return -138def _eval_lines(config_file, last_points, start_idx, end_idx, max_steps, interval=10000):39    print('config_files: {}'.format(config_file))40    project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))41    sweeper = Sweeper(os.path.join(project_root, config_file))42    eval = []43    eval_lines = []44    for k in range(sweeper.total_combinations):45        eval.append([])46        eval_lines.append([])47    for idx in range(start_idx, end_idx):48        cfg = sweeper.parse(idx)49        cfg.data_root = os.path.join(project_root, 'data', 'output')50        log_dir = cfg.get_log_dir()51        log_path = os.path.join(log_dir, 'log')52        try:53            with open(log_path, "r") as f:54                lines = f.readlines()55        except FileNotFoundError:56            continue57        if len(lines) == 0:58            continue59        # ugly parse based on the log_file format60        try:61            num_steps = get_max_steps(lines)62            if num_steps >= max_steps:63                assert idx % sweeper.total_combinations == cfg.param_setting64                avg_eval_steps = extract_line(lines, max_steps, interval=interval)65                # eval[id % sweeper.total_combinations].append(np.mean(avg_eval_steps[-int(len(avg_eval_steps)):]))66                eval[idx % sweeper.total_combinations].append(np.mean(avg_eval_steps[-last_points:]))67                # eval[id % sweeper.total_combinations].append(np.mean(avg_eval_steps))68        except IndexError:69            print(idx)70            raise71    summary = list(map(lambda x: (x[0], np.mean(x[1]), np.std(x[1]), len(x[1])), enumerate(eval)))72    summary = [x for x in summary if np.isnan(x[1]) == False]73    # new_summary = []74    # for s in summary:75    #     if np.isnan(s[1]) == False:76    #         new_summary.append(s)77    # print(summary[0])78    # print(new_summary[0])79    # quit()80    summary = sorted(summary, key=lambda s: s[1], reverse=True)81    for idx, mean, std, num_runs in summary:82        print("Param Setting # {:>3d} | Rewards: {:>10.10f} +/- {:>5.2f} ({:>2d} runs) {} | ".format(83            idx, mean, std, num_runs, sweeper.param_setting_from_id(idx)))84if __name__ == '__main__':85    parser = argparse.ArgumentParser(description="run_file")86    parser.add_argument('--cfg-file', default='default.json')87    parser.add_argument('--start', default=0, type=int)88    parser.add_argument('--end', default=72, type=int)89    parser.add_argument('--max-steps', default=50000, type=int)90    parser.add_argument('--interval', default=10, type=int)91    args = parser.parse_args()92    #############93    # Navigation94    # print("dqn_navigation")95    # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_navigation_c.json', last_points=15, start_idx=0, end_idx=40, max_steps=300000, interval=10000)96    # # print("zach_navigation")97    # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_navigation.json', last_points=15, start_idx=0, end_idx=200, max_steps=300000, interval=10000)98    # print("mve_avg_navigation")99    # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_navigation.json', last_points=15, start_idx=0, end_idx=160, max_steps=300000, interval=10000)100    # print("mve_avg_navigation_learnedv2")101    # _eval_lines(config_files='experiment/archive/mve_avg_learnedv2/navigation.json', last_points=15, start_idx=0, end_idx=60, max_steps=300000, interval=10000)102    # print("search_dt_navigation")103    # _eval_lines(config_files='experiment/archive/search_dt_perfect/navigation.json', last_points=15, start_idx=0, end_idx=80, max_steps=300000, interval=10000)104    #105    # print("search_dt_navigation_learnedv2")106    # _eval_lines(config_files='experiment/archive/search_dt_learnedv2/navigation.json', last_points=15, start_idx=0, end_idx=80, max_steps=200000, interval=10000)107    # print("mve_sw_model")108    # _eval_lines(config_files='experiment/archive/mve_sw_model/navigation.json', last_points=15, start_idx=0, end_idx=240, max_steps=300000, interval=10000)109    # print("mve_learned/navigation")110    # _eval_lines(config_files='experiment/config_files_2/mve_learned/navigation.json', last_points=15, start_idx=0, end_idx=160, max_steps=300000, interval=10000)111    # #############112    #############113    # Cartpole114    #115    # print("dqn_cartpole")116    # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_cartpole.json', last_points=12, start_idx=0, end_idx=40, max_steps=50000, interval=2000)117    # print("mve_cartpole")118    # _eval_lines(config_files='experiment/archive/mve_perfect/mve_cartpole.json', last_points=12, start_idx=0, end_idx=160, max_steps=50000, interval=2000)119    # print("mve_avg_cartpole")120    # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_cartpole.json', last_points=12, start_idx=0, end_idx=160, max_steps=50000, interval=2000)121    #122    # print("zach_cartpole")123    # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_cartpole.json', last_points=12, start_idx=0, end_idx=200, max_steps=50000, interval=2000)124    # print("mve_cartpole_learned")125    # _eval_lines(config_files='experiment/archive/mve_avg_learned/cartpole.json', last_points=12, start_idx=0, end_idx=80, max_steps=50000, interval=2000)126    #127    # print("mve_sw_model")128    # _eval_lines(config_files='experiment/archive/mve_sw_model/cartpole.json', last_points=12, start_idx=0, end_idx=240, max_steps=50000, interval=2000)129    # print("dqn_cartpole")130    # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_cartpole_b.json', last_points=12, start_idx=0, end_idx=40, max_steps=50000, interval=2000)131    #132    # print("zach_cartpole_b")133    # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_cartpole_b.json', last_points=12, start_idx=0, end_idx=120, max_steps=50000, interval=2000)134    #135    # print("zach_cartpole_c")136    # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_cartpole_c.json', last_points=12, start_idx=0, end_idx=200, max_steps=50000, interval=2000)137    # print("zach_cartpole_d")138    # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_cartpole_d.json', last_points=12, start_idx=0,139    #             end_idx=200, max_steps=50000, interval=2000)140    # #141    # print("search_dt_perfect")142    # _eval_lines(config_files='experiment/archive/search_dt_perfect/cartpole.json', last_points=12, start_idx=0, end_idx=120, max_steps=50000, interval=2000)143    # print("mve_avg_learnedv2")144    # _eval_lines(config_files='experiment/archive/mve_avg_learnedv2/cartpole.json', last_points=12, start_idx=0, end_idx=90, max_steps=50000, interval=2000)145    # print("search_dt_learnedv2")146    # _eval_lines(config_files='experiment/archive/search_dt_learnedv2/cartpole.json', last_points=12, start_idx=0, end_idx=120, max_steps=50000, interval=2000)147    # print("mve_learned/cartpole")148    # _eval_lines(config_files='experiment/config_files_2/mve_learned/cartpole.json', last_points=12, start_idx=0, end_idx=160, max_steps=50000, interval=2000)149    # ############150    #############151    # Catcher Raw152    # print("dqn_catcher_raw")153    # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_raw.json', last_points=20, start_idx=0, end_idx=40, max_steps=200000, interval=5000)154    #155    # print("dqn_catcher_raw_VF_sw")156    # _eval_lines(config_files='experiment/config_files_2/model_free/catcher_raw.json', last_points=20, start_idx=0, end_idx=270, max_steps=200000, interval=5000)157    # print("dqn_catcher_raw_VF_sw_huber")158    # _eval_lines(config_files='experiment/config_files_2/model_free/catcher_raw_huber.json', last_points=5, start_idx=0, end_idx=270, max_steps=200000, interval=5000)159    # print("dqn_catcher_raw_architecture_sweep")160    # _eval_lines(config_files='experiment/archive/model_free_v4/catcher_raw.json', last_points=20, start_idx=0, end_idx=160, max_steps=200000, interval=5000)161    #162    # print("dqn_catcher_raw_architecture_sweep_smaller_lr")163    # _eval_lines(config_files='experiment/archive/model_free_v4/catcher_raw_smaller_lr.json', last_points=20, start_idx=0, end_idx=150, max_steps=200000, interval=5000)164    # print("zach_catcher_raw")165    # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_catcher_raw.json', last_points=20, start_idx=0, end_idx=200, max_steps=200000, interval=5000)166    # print("zach_catcher_raw_b")167    # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_catcher_raw_b.json', last_points=20, start_idx=0, end_idx=160, max_steps=200000, interval=5000)168    #169    # print("mve_avg_catcher_raw")170    # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_catcher_raw.json', last_points=20, start_idx=0, end_idx=160, max_steps=200000, interval=5000)171    # print("mve_avg_catcher_learned")172    # _eval_lines(config_files='experiment/archive/mve_avg_learned/catcher_raw.json', last_points=20, start_idx=0, end_idx=90, max_steps=200000, interval=5000)173    #174    # print("mve_avg_catcher_learned_model_sw")175    # _eval_lines(config_files='experiment/archive/mve_sw_model/catcher_raw.json', last_points=20, start_idx=0, end_idx=90, max_steps=200000, interval=5000)176    # print("mve_avg_catcher_learned_error_aware")177    # _eval_lines(config_files='experiment/archive/mve_error_aware/catcher_raw.json', last_points=20, start_idx=0, end_idx=120, max_steps=200000, interval=5000)178    # print("mve_avg_catcher_learned_model_sweeps")179    # _eval_lines(config_files='experiment/archive/mve_sw_model/catcher_raw.json', last_points=20, start_idx=0, end_idx=240, max_steps=200000, interval=5000)180    # print("search_dt_perfect")181    # _eval_lines(config_files='experiment/archive/search_dt_perfect/catcher_raw.json', last_points=20, start_idx=0, end_idx=80, max_steps=200000, interval=5000)182    # print("search_dt_learned")183    # _eval_lines(config_files='experiment/archive/search_dt_learned/catcher_raw.json', last_points=20, start_idx=0, end_idx=80, max_steps=200000, interval=5000)184    #185    # print("search_dt_learnedv2")186    # _eval_lines(config_files='experiment/archive/search_dt_learnedv2/catcher_raw.json', last_points=20, start_idx=0, end_idx=80, max_steps=200000, interval=5000)187    # print("mve_avg_catcher_learnedv2")188    # _eval_lines(config_files='experiment/archive/mve_avg_learnedv2/catcher_raw.json', last_points=20, start_idx=0, end_idx=80, max_steps=200000, interval=5000)189    #190    # print("mve_avg_catcher_learned_error_aware")191    # _eval_lines(config_files='experiment/archive/mve_error_awarev2/catcher_raw.json', last_points=20, start_idx=0, end_idx=120, max_steps=200000, interval=5000)192    # print("mve_learned")193    # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw.json', last_points=20, start_idx=0, end_idx=200, max_steps=200000, interval=5000)194    # # print("mse_learned_mse")195    # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw_mse.json', last_points=20, start_idx=0,196    #             end_idx=200, max_steps=200000, interval=5000)197    # print("search_learned")198    # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw.json', last_points=20, start_idx=0, end_idx=200, max_steps=200000, interval=5000)199    # # print("search_learned_mse")200    # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_mse.json', last_points=20, start_idx=0, end_idx=200, max_steps=200000, interval=5000)201    # print("mve_learned_v2_mse")202    # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw_v2.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)203    #204    # print("mve_learned_v3_huber")205    # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw_v3.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)206    #207    # print("mve_learned_v4_mse_highlr")208    # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw_v4.json', last_points=20, start_idx=0, end_idx=90, max_steps=200000, interval=5000)209    #210    # print("search_learned_v2")211    # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_v2.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)212    #213    # print("search_learned_v3_huber")214    # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_v3.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)215    #216    # print("search_learned_v4_mse_highlr")217    # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_v4.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)218    # print("search_learned_v4_mse_huber")219    # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_v5.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)220    # print("capacity_catcher_raw")221    # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity_catcher_raw.json', last_points=20, start_idx=0, end_idx=1540, max_steps=200000, interval=5000)222    # print("dqn_catcher_raw_VF_sw_huber")223    # _eval_lines(config_files='experiment/config_files_2/model_free/catcher_raw_huber.json', last_points=20, start_idx=0, end_idx=270, max_steps=200000, interval=5000)224    # print("capacity_catcher_raw_8")225    # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_8.json', last_points=20, start_idx=0, end_idx=20, max_steps=200000, interval=5000)226    # print("capacity_catcher_raw_16")227    # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_16.json', last_points=20, start_idx=0, end_idx=30, max_steps=200000, interval=5000)228    # print("capacity_catcher_raw_32")229    # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_32.json', last_points=20, start_idx=0, end_idx=40, max_steps=200000, interval=5000)230    # print("capacity_catcher_raw_64")231    # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_64.json', last_points=20, start_idx=0, end_idx=50, max_steps=200000, interval=5000)232    # print("capacity_catcher_raw_128")233    # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_128.json', last_points=20, start_idx=0, end_idx=60, max_steps=200000, interval=5000)234    # print("capacity_catcher_raw_256")235    # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_256.json', last_points=20, start_idx=0, end_idx=70, max_steps=200000, interval=5000)236    # print("search_learned")237    # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_huber.json', last_points=20, start_idx=0, end_idx=300, max_steps=200000, interval=5000)238    # ###########239    ############240    # Acrobot241    # print("dqn_acrobot")242    # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_test.json', last_points=25, start_idx=0, end_idx=40, max_steps=100000, interval=2000)243    # print("mve_acrobot")244    # _eval_lines(config_files='experiment/archive/mve_perfect/mve_acrobot.json', last_points=25, start_idx=0, end_idx=160, max_steps=100000, interval=2000)245    #246    # print("mve_avg_acrobot")247    # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_acrobot.json', last_points=25, start_idx=0, end_idx=160, max_steps=100000, interval=2000)248    # #249    # print("mve_avg_acrobot_learnedv2")250    # _eval_lines(config_files='experiment/archive/mve_avg_learnedv2/acrobot.json', last_points=25, start_idx=0, end_idx=60, max_steps=100000, interval=2000)251    # #252    # print("mve_sw_model")253    # _eval_lines(config_files='experiment/archive/mve_sw_model/acrobot.json', last_points=25, start_idx=0, end_idx=240, max_steps=100000, interval=2000)254    #255    # print("zach_acrobot")256    # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_acrobot.json', last_points=25, start_idx=0, end_idx=200, max_steps=100000, interval=2000)257    #258    # print("search_dt_acrobot")259    # _eval_lines(config_files='experiment/archive/search_dt_perfect/acrobot.json', last_points=25, start_idx=0, end_idx=80, max_steps=100000, interval=2000)260    #261    # print("search_dt_acrobot_learnedv2")262    # _eval_lines(config_files='experiment/archive/search_dt_learnedv2/acrobot.json', last_points=25, start_idx=0, end_idx=80, max_steps=100000, interval=2000)263    # print("mve_learned/acrobot")264    # _eval_lines(config_files='experiment/config_files_2/mve_learned/acrobot.json', last_points=25, start_idx=0, end_idx=160, max_steps=100000, interval=2000)265    print("search_learned_fixed/acrobot")266    _eval_lines(config_file='experiment/config_files_2/search_learned_fixed/acrobot.json', last_points=25, start_idx=0, end_idx=500, max_steps=100000, interval=2000)267    print("search_learned_fixed/acrobot exponential")268    _eval_lines(config_file='experiment/config_files_2/search_learned_fixed/acrobot_exponential.json', last_points=25, start_idx=0, end_idx=500, max_steps=100000, interval=2000)269    # # ############270    ############271    # Catcher visual272    # print("dqn_catacher_visual")273    # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_visual.json', last_points=25, start_idx=0, end_idx=45, max_steps=500000, interval=10000)274    # print("mve_catcher_visual")275    # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_catcher_visual.json', last_points=25, start_idx=0, end_idx=120, max_steps=500000, interval=10000)276    # print("zach_catcher_visual")277    # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_catcher_visual_b.json', last_points=25, start_idx=0, end_idx=150, max_steps=500000, interval=10000)278    # ############279    ## model_free280    # print("dqn_catacher_raw")281    # _eval_lines(config_files='experiment/archive/model_free/dqn_catcher_raw.json', start_idx=0, end_idx=120, max_steps=400000, interval=10000)282    # print("dqn_catacher_raw_2h")283    # _eval_lines(config_files='experiment/archive/model_free/dqn_catcher_raw_2h.json', start_idx=0, end_idx=60, max_steps=400000, interval=10000)284    # print("dqn_catacher_visual_gray")285    # _eval_lines(config_files='experiment/archive/model_free/dqn_catcher_visual_gray.json', start_idx=0, end_idx=24, max_steps=200000, interval=10000)286    ## model_free_v2287    # print("dqn_catacher_raw")288    # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_raw.json', last_points=20, start_idx=0, end_idx=40, max_steps=200000, interval=5000)289    # print("dqn_acrobot")290    # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_test.json', last_points=10, start_idx=0, end_idx=359, max_steps=200000, interval=10000)291    # print("dqn_catacher_lite")292    # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_lite.json', start_idx=0, end_idx=60, max_steps=400000, interval=10000)293    # print("dqn_catacher_lite_b")294    # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_lite_b.json', start_idx=0, end_idx=80, max_steps=600000, interval=10000)295    # print("dqn_catacher_visual")296    # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_visual.json', start_idx=0, end_idx=45, max_steps=500000, interval=10000)297    # print("dqn_acrobot")298    # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_test.json', start_idx=0, end_idx=360, max_steps=200000, interval=10000)299    # print("mve_perfect_catcher_raw")300    # _eval_lines(config_files='experiment/archive/mve_perfect/mve_catcher_raw.json', last_points=20, start_idx=0, end_idx=160, max_steps=200000, interval=5000)301    # print("dqn_acrobot")302    # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_test.json', last_points=25, start_idx=0, end_idx=40, max_steps=100000, interval=2000)303    #304    # print("mve_acrobot")305    # _eval_lines(config_files='experiment/archive/mve_perfect/mve_acrobot.json', last_points=25, start_idx=0, end_idx=160, max_steps=100000, interval=2000)306    # print("dqn_navigation")307    # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_navigation_b.json', last_points=10, start_idx=0, end_idx=40, max_steps=200000, interval=10000)308    #309    # print("mve_navigation")310    # _eval_lines(config_files='experiment/archive/mve_perfect/mve_navigation.json', last_points=10, start_idx=0, end_idx=160, max_steps=200000, interval=10000)311    # print("dqn_cartpole")312    # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_cartpole.json', last_points=12, start_idx=0, end_idx=40, max_steps=50000, interval=2000)313    # #314    # print("mve_cartpole")315    # _eval_lines(config_files='experiment/archive/mve_perfect/mve_cartpole.json', last_points=12, start_idx=0, end_idx=160, max_steps=50000, interval=2000)316    # print("dqn_navigation_b")317    # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_navigation_c.json', last_points=15, start_idx=0, end_idx=40, max_steps=300000, interval=10000)318    #319    # print("mve_navigation_b")320    # _eval_lines(config_files='experiment/archive/mve_perfect/mve_navigation_c.json', last_points=15, start_idx=0, end_idx=160, max_steps=300000, interval=10000)321    # print("zach_navigation")322    # _eval_lines(config_files='experiment/archive/zach_perfect/zach_navigation.json', last_points=10, start_idx=0, end_idx=160, max_steps=200000, interval=10000)323    # print("dqn_catacher_visual")324    # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_visual.json', last_points=25, start_idx=0, end_idx=45, max_steps=500000, interval=10000)325    #326    # print("mve_catacher_visual")327    # _eval_lines(config_files='experiment/archive/mve_perfect/mve_catcher_visual.json', last_points=25,328    #             start_idx=0, end_idx=60, max_steps=500000, interval=10000)329    # TRACKING330    # Catcher Raw331    # print("sprint_3/catcher_raw/horizon_avg")332    # _eval_lines(config_files='experiment/config_files_3/search_learned/catcher_raw/horizon_avg.json', last_points=20, start_idx=0, end_idx=50, max_steps=200000, interval=5000)333    #334    #335    # print("sprint_3/catcher_raw/horizon_leaf")336    # _eval_lines(config_files='experiment/config_files_3/search_learned/catcher_raw/horizon_leaf.json', last_points=20, start_idx=0, end_idx=50, max_steps=200000, interval=5000)337    #338    #339    # print("sprint_3/catcher_raw/horizon_avg_tune_vf_lr")340    # _eval_lines(config_files='experiment/config_files_3/search_learned/catcher_raw/horizon_avg_tune_vf_lr.json', last_points=20, start_idx=0, end_idx=360, max_steps=200000, interval=5000)341    #342    #343    #344    #345    #...

test_config.py

Source:test_config.py

1"""Do an integration test for config file usage."""23import os4import subprocess56HTML_TEST_FILES = os.path.abspath(os.path.dirname(__file__))789def test_config_valid():10    """Config test for valid HTML"""11    assert subprocess.call([12        'html5validator',13        f'--config={HTML_TEST_FILES}/config_files/valid.yaml']) == 0141516def test_config_invalid():17    """Config test for invalid HTML"""18    assert subprocess.call([19        'html5validator',20        f'--config={HTML_TEST_FILES}/config_files/invalid.yaml']) == 1212223def test_config_skip():24    """Config test for skipping files"""25    assert subprocess.call([26        'html5validator',27        f'--config={HTML_TEST_FILES}/config_files/skip.yaml']) == 2282930def test_config_invalid_with_css():31    """Config test for CSS and HTML"""32    assert subprocess.call([33        'html5validator',34        f'--config={HTML_TEST_FILES}/config_files/invalid_css.yaml'35    ]) == 3363738def test_config_invalid_css_only():39    """Config test for CSSS only"""40    assert subprocess.call([41        'html5validator',42        f"--config={HTML_TEST_FILES}/config_files/invalid_css_only.yaml"]) == 2434445def test_config_invalid_single_file():46    """Config test for invalid single file"""47    assert subprocess.call([48        'html5validator',49        f"--config={HTML_TEST_FILES}/config_files/invalid_single_file.yaml"50    ]) == 1515253def test_config_warning():54    """Config test for warnings"""55    assert subprocess.call([56        'html5validator',57        f"--config={HTML_TEST_FILES}/config_files/warning.yaml"]) == 1585960def test_config_warning_but_pass():61    """Config test for allowed warnings"""62    assert subprocess.call([63        'html5validator',64        f'--config={HTML_TEST_FILES}/config_files/warning_pass.yaml']) == 0656667def test_config_return_value():68    """Config test for error code return value"""69    assert subprocess.call([70        'html5validator',71        f'--config={HTML_TEST_FILES}/config_files/return_254.yaml']) == 25472    assert subprocess.call([73        'html5validator',74        f'--config={HTML_TEST_FILES}/config_files/return_255.yaml']) == 25575    assert subprocess.call([76        'html5validator',77        f'--config={HTML_TEST_FILES}/config_files/return_255_256.yaml']) == 255787980def test_config_angularjs():81    """Config test for angularjs"""82    assert subprocess.call([83        'html5validator',84        f'--config={HTML_TEST_FILES}/config_files/angularjs.yaml']) == 0858687def test_config_angularjs_no_output_with_ignore():88    """Make sure there is no spurious output when messages are ignored."""89    assert subprocess.check_output([90        'html5validator',91        f'--config={HTML_TEST_FILES}/config_files/angularjs.yaml']) == b''929394def test_config_angularjs_normal_quotes():95    """Config test for normal angularjs"""96    assert subprocess.call([97        'html5validator',98        f'--config={HTML_TEST_FILES}/config_files/angularjs_normal.yaml']) == 099100101def test_config_multiple_ignoreres():102    """Config test for multiple regex ignores"""103    assert subprocess.call([104        'html5validator',105        f'--config={HTML_TEST_FILES}/config_files/multiple_ignores.yaml']) == 0106107108def test_config_ignore_and_ignorere():109    """Config test for ignore and regex ignore"""110    assert subprocess.call([111        'html5validator',112        f'--config={HTML_TEST_FILES}/config_files/ignore_and_ignorere.yaml'113    ]) == 0114115116def test_config_stack_size():117    """Config test for stack size"""118    assert subprocess.call([119        'html5validator',120        f'--config={HTML_TEST_FILES}/config_files/stack_size.yaml']) == 0121122123def test_config_valid_format_flags():124    """Config test for output format for valid files"""125    assert subprocess.call([126        'html5validator',127        f'--config={HTML_TEST_FILES}/config_files/format_flags/text_valid.yaml'128    ]) == 0129    assert subprocess.call([130        'html5validator',131        f'--config={HTML_TEST_FILES}/config_files/format_flags/gnu_valid.yaml'132    ]) == 0133    assert subprocess.call([134        'html5validator',135        f'--config={HTML_TEST_FILES}/config_files/format_flags/json_valid.yaml'136    ]) == 0137    assert subprocess.call([138        'html5validator',139        f'--config={HTML_TEST_FILES}/config_files/format_flags/xml_valid.yaml'140    ]) == 0141142143def test_config_invalid_format_flags():144    """Config test for output format with invalid files"""145    assert subprocess.call([146        'html5validator',147        f'--config={HTML_TEST_FILES}/config_files/format_flags/'148        'text_invalid.yaml'149    ]) == 3150    assert subprocess.call([151        'html5validator',152        f'--config={HTML_TEST_FILES}/config_files/format_flags/'153        'gnu_invalid.yaml'154    ]) == 1155    assert subprocess.call([156        'html5validator',157        f'--config={HTML_TEST_FILES}/config_files/format_flags/'158        'json_invalid.yaml'159    ]) == 1160    assert subprocess.call([161        'html5validator',162        f'--config={HTML_TEST_FILES}/config_files/format_flags/'163        'xml_invalid.yaml'164    ]) == 8165166167def test_config_log_file():168    """Config test for log file"""169    assert subprocess.call([170        'html5validator',171        f'--config={HTML_TEST_FILES}/config_files/log_file.yaml']) == 0172173174def test_config_extra():175    """Config test for vnu extra arguments"""176    assert subprocess.call([177        'html5validator',178        f'--config={HTML_TEST_FILES}/config_files/extra.yaml']) == 0179180181def test_config_nofiles():182    """Command line test for missing files"""183    assert subprocess.call([184        "html5validator",185        f'--config={HTML_TEST_FILES}/config_files/no_files.yaml'186    ]) == 1187188189if __name__ == '__main__':190    test_config_valid()191    test_config_invalid()192    test_config_return_value()193    test_config_angularjs()194    test_config_multiple_ignoreres()195    test_config_ignore_and_ignorere()196    test_config_stack_size()197    test_config_valid_format_flags()198    test_config_invalid_format_flags()199    test_config_log_file()
...

configtest.py

Source:configtest.py

...39                                             'test-python-configs',40                                             'summary',41                                            ],42                            require_config_file=require_config_file)43    def query_config_files(self):44        """This query method, much like others, caches its runtime45        settings in self.VAR so we don't have to figure out config_files46        multiple times.47        """48        if self.config_files:49            return self.config_files50        c = self.config51        if 'test_files' in c:52            self.config_files = c['test_files']53            return self.config_files54        self.debug("No --test-file(s) specified; defaulting to crawling the configs/ directory.")55        config_files = []56        for root, dirs, files in os.walk(os.path.join(sys.path[0], "..",57                                                      "configs")):58            for name in files:59                # Hardcode =P60                if name.endswith(".json") or name.endswith(".py"):61                    if not name.startswith("test_malformed"):62                        config_files.append(os.path.join(root, name))63        self.config_files = config_files64        return self.config_files65    def list_config_files(self):66        """ Non-default action that is mainly here to demonstrate how67        non-default actions work in a mozharness script.68        """69        config_files = self.query_config_files()70        for config_file in config_files:71            self.info(config_file)72    def test_json_configs(self):73        """ Currently only "is this well-formed json?"74        """75        config_files = self.query_config_files()76        filecount = [0, 0]77        for config_file in config_files:78            if config_file.endswith(".json"):79                filecount[0] += 180                self.info("Testing %s." % config_file)81                contents = self.read_from_file(config_file, verbose=False)82                try:83                    json.loads(contents)84                except ValueError:85                    self.add_summary("%s is invalid json." % config_file,86                                     level="error")87                    self.error(pprint.pformat(sys.exc_info()[1]))88                else:89                    self.info("Good.")90                    filecount[1] += 191        if filecount[0]:92            self.add_summary("%d of %d json config files were good." %93                             (filecount[1], filecount[0]))94        else:95            self.add_summary("No json config files to test.")96    def test_python_configs(self):97        """Currently only "will this give me a config dictionary?"98        """99        config_files = self.query_config_files()100        filecount = [0, 0]101        for config_file in config_files:102            if config_file.endswith(".py"):103                filecount[0] += 1104                self.info("Testing %s." % config_file)105                global_dict = {}106                local_dict = {}107                try:108                    execfile(config_file, global_dict, local_dict)109                except:110                    self.add_summary("%s is invalid python." % config_file,111                                     level="error")112                    self.error(pprint.pformat(sys.exc_info()[1]))113                else:...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.