Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use read_with_lock method in yandex-tank

Best Python code snippet using yandex-tank

util.py

Source:util.py

...586    def get_file(self, cache_size=None):587        cache_size = self.cache_size if not cache_size else cache_size588        fileobj = FileLike(self, cache_size)589        return fileobj590    def read_with_lock(self, pos, _len=None):591        """592        Reads {_len} characters if _len is not None else reads line593        :param pos: start reading position594        :param _len: number of characters to read595        :rtype: (string, int)596        """597        self.wait_lock()598        try:599            self._opened_file.seek(pos)600            result = self._opened_file.read(_len) if _len is not None else self._opened_file.readline()601            stop_pos = self._opened_file.tell()602        finally:603            self.unlock()604        if not result and self.stop.is_set():605            result = None606        return result, stop_pos607    @retry(wait_random_min=5, wait_random_max=20, stop_max_delay=10000,608           retry_on_exception=FileLockedError.retry, wrap_exception=True)609    def wait_lock(self):610        if self._is_locked:611            raise FileLockedError('Generator output file {} is locked'.format(self.filename))612        else:613            self._is_locked = True614            return True615    def unlock(self):616        self._is_locked = False617class FileLike(object):618    def __init__(self, multireader, cache_size):619        """620        :type multireader: FileMultiReader621        """622        self.multireader = multireader623        self.cache_size = cache_size624        self._cursor = 0625    def read(self, _len=None):626        _len = self.cache_size if not _len else _len627        result, self._cursor = self.multireader.read_with_lock(self._cursor, _len)628        return result629    def readline(self):630        result, self._cursor = self.multireader.read_with_lock(self._cursor)631        return result632def get_callstack():633    """634        Get call stack, clean wrapper functions from it and present635        in dotted notation form636    """637    stack = inspect.stack(context=0)638    cleaned = [frame[3] for frame in stack if frame[3] != 'wrapper']639    return '.'.join(cleaned[1:])640def timeit(min_duration_sec):641    def timeit_fixed(func):642        def wrapper(*args, **kwargs):643            start_time = time.time()644            result = func(*args, **kwargs)...

predict_keras.py

Source:predict_keras.py

...29from subseasonal_toolkit.utils.experiments_util import pandas2hdf30from subseasonal_toolkit.utils.general_util import make_directories31from subseasonal_toolkit.utils.experiments_util import get_target_date as get_target_date_eval32from subseasonal_toolkit.models.salient.salient_util import ONE_WEEK, ONE_DAY, WEEKS, dir_submodel_forecasts, dir_train_results, dir_train_data, dir_predict_data, year_fraction, get_target_date33def read_with_lock(filename):34    """Open an hdf file with a lock and read. 35    Args:36        filename (str): path of hdf file to be read with a lock.37    Returns:38        openned pandas dataframe.39    """40    #with FileLock(filename+'lock'):41    #    df = pd.read_hdf(filename)  42    df = pd.read_hdf(filename)  43    subprocess.call(f"rm {filename}lock", shell=True)44    return df45def generate_windows(x):46    """generates a sliding window over the data of length window_size.47    Args:48        x: array of input features used the NN49    Returns:50        reshaped array of input features so that each example is a concatenation of the past 10 weeks of data.51    """52    window_size = 1053    result = []54    for i in range(x.shape[0]+1-window_size):55        result.append(x[i:i+window_size, ...])56    return np.stack(result)57def compile_input(submodel_name, datestr):58    """Compile input data to be used by NN.59    Args:60        submodel_name (str): string consisting of the ground truth variable 61            ids used for training and the date of the last training example.62        datestr (str): YYMMDD string of first day of target 2-week period.63    Returns:64        input_data: array of input features used generate predictions by the NNs.65    """  66    67    #get input start and end date 68    target_date = datetime.strptime(datestr, '%Y%m%d') 69    end_date = target_date - timedelta(days=((target_date.weekday() - 5) % 7))70    end_date = datetime(end_date.year, end_date.month, end_date.day)71    start_date = end_date - ONE_WEEK * WEEKS + ONE_DAY 72    73    # load date data    74    date_data_file = os.path.join(dir_train_data, "date.pickle")75    date_vectors = pickle.load(open(date_data_file, 'rb'))76    date_vectors_all = sorted([d[0] for d in date_vectors])77    end_date_sn = datetime.strptime(submodel_name[-8:], '%Y%m%d')78    date_vectors = [d for d in date_vectors_all if (d.astype(object).year<=end_date_sn.year) and (d.astype(object).month<=end_date_sn.month) and (d.astype(object).day<=end_date_sn.day)]      79    last_i = len(date_vectors)80    # load training data81    # load sst data82    training_sst_data_file = os.path.join(dir_train_data, "sst.pickle")83    training_sst_vectors_all = pickle.load(open(training_sst_data_file, 'rb'))   84    # load time data85    training_time_data_file = os.path.join(dir_train_data, "time.pickle")86    training_time_vectors = pickle.load(open(training_time_data_file, 'rb'))87    training_time_vectors_all = np.reshape(training_time_vectors,(training_time_vectors.shape[0],1))88  89    # account for early train stop submodels90    training_sst_vectors = training_sst_vectors_all[:last_i,:]91    training_time_vectors = training_time_vectors_all[:last_i,:]92    ## Load input data to generate a prediction93    # load sst data94    sst_data_file = os.path.join(dir_predict_data, "salient_fri", datestr, "sst.pickle")95    if isfile(sst_data_file):96        sst_vectors = pickle.load(open(sst_data_file, 'rb'))97    else:98        #find datestr in dates99        d = [d for d in date_vectors_all if (d.astype(object).year==target_date.year) and (d.astype(object).month==target_date.month) and (d.astype(object).day==target_date.day)]100        i = date_vectors_all.index(d)101        sst_vectors = training_sst_vectors_all[i-10:i,:]102    sst_vectors = (sst_vectors - np.amin(training_sst_vectors)) * 1./(np.amax(training_sst_vectors) - np.amin(training_sst_vectors))103                104    # compile input data105    input_data = sst_vectors106    107               108    return input_data109def add_i_time(input_data, submodel_name, datestr, i_time=False):110    """add i_time feature to input data to be used by NN.111    Args:112        input_data (float): array of input features used generate predictions by the NNs.113        submodel_name (str): string consisting of the ground truth variable 114            ids used for training and the date of the last training example.115        datestr (str): YYMMDD string of first day of target 2-week period.116        i_time (bool): if True, include time vector as an input feature (default: False).117    Returns:118        input_data: array of input features used generate predictions by the NNs.119    """120    121    #get input start and end date 122    target_date = datetime.strptime(datestr, '%Y%m%d') 123    end_date = target_date - timedelta(days=((target_date.weekday() - 5) % 7))124    end_date = datetime(end_date.year, end_date.month, end_date.day)125    start_date = end_date - ONE_WEEK * WEEKS + ONE_DAY 126        127    #load time data128    if i_time:129        time_vectors = np.zeros((WEEKS, 1))130        day = start_date131        for i in range(time_vectors.shape[0]):132            time_vectors[i, 0] = year_fraction(day)133            day += ONE_WEEK134        135    # compile input data136    if i_time:137        input_data = np.concatenate((input_data, time_vectors), axis=1)138               139    return input_data140def main():141    #"""142    #example usage to generate original salient forecasts143    # python src/models/salient/predict_keras.py -d 20200811 -sn salient_20170201144    # input arguments145    parser = argparse.ArgumentParser()146    parser.add_argument('-d', '--date', help='Submission date')147    parser.add_argument('-sn', '--submodel_name', default='salient_fri_20170201')148    parser.add_argument('--region', '-r', default='contest')149    args = parser.parse_args()150    # set args151    submodel_name = args.submodel_name152    target_date, datestr = get_target_date('Generate predictions', args.date)153    region = args.region154    155    # Setup directory where the submodel's weights are saved.156    dir_weights = os.path.join(dir_train_results, submodel_name, f"{region}_{submodel_name}")157    # map locations with points of interest158    mask_f = os.path.join("data", "masks", "us_mask.nc")159    mask_ds = Dataset(mask_f) if region.startswith('us') else Dataset(mask_f.replace("us_", "fcstrodeo_"))160    mask_lat = mask_ds.variables['lat'][:]161    mask_lon = mask_ds.variables['lon'][:]162    points_idx = np.where(mask_ds.variables['mask'][:])163    points = np.array((points_idx[0], points_idx[1])).T164    165    # Import and run models166    model_names = glob.glob(os.path.join(dir_weights, "k_model_*.h5"))167    N_models = len(model_names)168    169    if region.startswith('us'):170        num_of_gc = 862 171    elif region.startswith('contest'):172        num_of_gc = 514173    else:174        num_of_gc = 348175    # Create empty template array for predictions to be generated176    precip_wk34_predictions = np.zeros((N_models,num_of_gc))177    precip_wk56_predictions = np.zeros((N_models,num_of_gc))178    temp_wk34_predictions = np.zeros((N_models,num_of_gc))179    temp_wk56_predictions = np.zeros((N_models,num_of_gc))180     181    # Compile input data182    input_data_all = compile_input(submodel_name, datestr)183    184    # Generate predictions for each of the top 10 selected NN members in the ensemble185    for i in range(N_models):186        187        # Load NN ensemble member188        model_name = model_names[i]189        model = keras.models.load_model(model_name)190        # If NN was trained on time as an input feature, add time to the compile input data191        result = re.search('time(.*).h5', model_name)192        input_set = int(result.group(1))193        input_data = add_i_time(input_data_all, submodel_name, datestr, i_time=bool(input_set))194        input_data = generate_windows(input_data)195        # Generate predictions196        prediction_i = model.predict(input_data)197        # predictions for a 2-week target period are the accumulation of precip  198        # the mean temperature over the target period 199        prediction_i = np.reshape(prediction_i,(8,num_of_gc))           200        precip_wk34_predictions[i,:] = np.sum(prediction_i[0:2,:], axis=0)201        precip_wk56_predictions[i,:] = np.sum(prediction_i[2:4,:], axis=0)202        temp_wk34_predictions[i,:] = np.mean(prediction_i[4:6,:], axis=0)203        temp_wk56_predictions[i,:] = np.mean(prediction_i[6:8,:], axis=0)204            205    206    # sum precip predictions and average temp predictions over the 2-week target period207    # clip precip predictions to zero since precipitations cannot be negative208    precip_wk34_prediction = np.mean(precip_wk34_predictions, axis=0)209    precip_wk56_prediction = np.mean(precip_wk56_predictions, axis=0)210    precip_wk34_prediction = precip_wk34_prediction.clip(0)211    precip_wk56_prediction = precip_wk56_prediction.clip(0)212    temp_wk34_prediction = np.mean(temp_wk34_predictions, axis=0)213    temp_wk56_prediction = np.mean(temp_wk56_predictions, axis=0)214    215    216    # Get target date objects217    deadline_date = datetime.strptime(datestr, '%Y%m%d')218    target_date_34w = get_target_date_eval(datestr, "34w")219    target_date_56w = get_target_date_eval(datestr, "56w")220    221    # Get target date strings222    target_date_str_34w = datetime.strftime(target_date_34w, '%Y%m%d')223    target_date_str_56w = datetime.strftime(target_date_56w, '%Y%m%d')224    225    # Get lat, lon and pred template arrays226    template_f = resource_filename("subseasonal_toolkit", os.path.join("models", "salient", "data", "apcp_week34_template.nc"))227    template_ds = Dataset(template_f)228    template_lat = template_ds.variables["lat"][:]229    template_lon = template_ds.variables["lon"][:]230    template_var = template_ds.variables["apcp_week34"][:]231    232    # Determine variables, horizons and corresponding predictions to be saved233    gt_vars = ["precip", "tmp2m"] 234    horizons =  ["34w", "56w"] 235    predictions = [precip_wk34_prediction, precip_wk56_prediction, temp_wk34_prediction, temp_wk56_prediction]236    tasks = [f"{region}_{g}_{t}" for g, t in itertools.product(gt_vars, horizons)]237    238    # Format predictions to standard pandas contest prediction format.239    for task, prediction in zip(tasks, predictions):240        out_dir = os.path.join(dir_submodel_forecasts, submodel_name, task) 241        make_directories(out_dir)242        243        pred_file =  os.path.join(dir_train_data, "contest_latlons.h5")244        pred = read_with_lock(pred_file)245        #pred = pd.read_hdf(pred_file)246        247        if "34w" in task:248            pred["start_date"] = target_date_34w249            out_file = os.path.join(out_dir, f"{task}-{target_date_str_34w}.h5")250        elif "56" in task:251            pred["start_date"] = target_date_56w252            out_file = os.path.join(out_dir, f"{task}-{target_date_str_56w}.h5")253                   254        pred["pred"] = np.nan   255        a = template_var 256        # save predictions into array257        for loc in range(len(prediction)):258            index = points[loc]...

predict_keras_hindcasts.py

Source:predict_keras_hindcasts.py

...27from subseasonal_toolkit.utils.general_util import make_directories28from subseasonal_toolkit.utils.experiments_util import pandas2hdf29from subseasonal_toolkit.utils.experiments_util import get_target_date as get_target_date_eval30from subseasonal_toolkit.models.salient.salient_util import ONE_DAY, dir_train_data, dir_train_results, dir_submodel_forecasts, get_target_date31def read_with_lock(filename):32    """Open an hdf file with a lock and read. 33    Args:34        filename (str): path of hdf file to be read with a lock.35    Returns:36        openned pandas dataframe.37    """38    #with FileLock(filename+'lock'):39    #    df = pd.read_hdf(filename)  40    df = pd.read_hdf(filename)  41    subprocess.call(f"rm {filename}lock", shell=True)42    return df43def generate_windows(x):44    """generates a sliding window over the data of length window_size.45    Args:46        x: array of input features used the NN47    Returns:48        reshaped array of input features so that each example is a concatenation of the past 10 weeks of data.49    """50    window_size = 1051    result = []52    for i in range(x.shape[0]+1-window_size):53        result.append(x[i:i+window_size, ...])54    return np.stack(result)55def compile_input(date, i_time=False):56    ## Load training data57    # load time data58    training_time_data_file = os.path.join(dir_train_data, "time.pickle") # (t,1)59    training_time_vectors = pickle.load(open(training_time_data_file, 'rb'))60    training_time_vectors = np.reshape(training_time_vectors,(training_time_vectors.shape[0],1))61    # load sst data62    training_sst_data_file = os.path.join(dir_train_data, "sst.pickle") # (t,loc)63    training_sst_vectors = pickle.load(open(training_sst_data_file, 'rb'))64    # load precipitation data65    training_location_precip_file = os.path.join(dir_train_data, "precip.pickle") # (loc,t)66    training_precip_data = pickle.load(open(training_location_precip_file, 'rb'))67    training_precip_data = training_precip_data.T68    # load temperature data69    training_location_temp_file = os.path.join(dir_train_data, "temp.pickle") # (loc,t)70    training_temp_data = pickle.load(open(training_location_temp_file, 'rb'))71    training_temp_data = training_temp_data.T72    # make precip data only as long as temp data73    training_precip_data = training_precip_data[:training_temp_data.shape[0],:]74    # ensure same length vectors and normalize75    training_time_vectors = training_time_vectors[:training_precip_data.shape[0],:]76    training_sst_vectors = training_sst_vectors[:training_precip_data.shape[0],:]77    ## Generate input data from date78    date_data_file = os.path.join(dir_train_data, "date.pickle") # (time,1)79    dates = pickle.load(open(date_data_file, 'rb'))80    dates = dates[:training_sst_vectors.shape[0]]81    # dates represent middle of the week (Wednesday) sst or sss data is for Sun-Sat82    # Offset our date by 3 days so we're sure we don't have future data in the83    # input set84    cutoff_date = datetime.strptime(date, '%Y%m%d') - ONE_DAY * 385    input_indices = np.where(dates < np.datetime64(str(cutoff_date.date())))[0]86    # Take the last 10 weeks87    input_indices = input_indices[-10:]88    # load sst data89    sst_vectors = training_sst_vectors[input_indices]90    # Normalize91    sst_vectors = (sst_vectors - np.amin(training_sst_vectors)) * 1./(np.amax(training_sst_vectors) - np.amin(training_sst_vectors))92    # compile input data93    input_data = sst_vectors94    if i_time:95        time_vectors = training_time_vectors[input_indices]96        time_vectors = np.reshape(time_vectors,(time_vectors.shape[0],1))97        input_data = np.concatenate((input_data, time_vectors), axis=1)98    return input_data99def main():100    # input arguments101    parser = argparse.ArgumentParser()102    parser.add_argument('-d', '--date', help='Submission date')103    parser.add_argument('-sn', '--submodel_name', default='salient_fri_hindcasts_2010')104    parser.add_argument('--region', '-r', default='contest')105    args = parser.parse_args()106    # set args107    submodel_name = args.submodel_name108    target_date, datestr = get_target_date('Generate predictions', args.date)109    region = args.region110    111    # Setup directory where the submodel's weights are saved.112    dir_weights = os.path.join(dir_train_results, submodel_name, f"{region}_{submodel_name}")113    114    # map locations with points of interest115    mask_f = os.path.join("data", "masks", "us_mask.nc")116    mask_ds = Dataset(mask_f) if region.startswith('us') else Dataset(mask_f.replace("us_", "fcstrodeo_"))117    mask_lat = mask_ds.variables['lat'][:]118    mask_lon = mask_ds.variables['lon'][:]119    points_idx = np.where(mask_ds.variables['mask'][:])120    points = np.array((points_idx[0], points_idx[1])).T121    122    # Import and run models123    model_names = glob.glob(os.path.join(dir_weights, f"k_model_*.h5"))124    N_models = len(model_names)125    126    if region.startswith('us'):127        num_of_gc = 862 128    elif region.startswith('contest'):129        num_of_gc = 514130    else:131        num_of_gc = 348132    133    # Create empty template array for predictions to be generated134    precip_wk34_predictions = np.zeros((N_models,num_of_gc))135    precip_wk56_predictions = np.zeros((N_models,num_of_gc))136    temp_wk34_predictions = np.zeros((N_models,num_of_gc))137    temp_wk56_predictions = np.zeros((N_models,num_of_gc))138    139    # Generate predictions for each of the top 10 selected NN members in the ensemble140    for i in range(N_models):141        142        # Load NN ensemble member143        model_name = model_names[i]144        model = keras.models.load_model(model_name)145        # If NN was trained on time as an input feature, add time to the compile input data146        result = re.search('time(.*).h5', model_name)147        input_set = int(result.group(1))148        # Compile input data149        input_data_all = compile_input(datestr, i_time=bool(input_set))150        input_data = generate_windows(input_data_all)151        # Generate predictions152        prediction_i = model.predict(input_data)153        # predictions for a 2-week target period are the accumulation of precip  154        # the mean temperature over the target period 155        prediction_i = np.reshape(prediction_i,(8,num_of_gc))           156        precip_wk34_predictions[i,:] = np.sum(prediction_i[0:2,:], axis=0)157        precip_wk56_predictions[i,:] = np.sum(prediction_i[2:4,:], axis=0)158        temp_wk34_predictions[i,:] = np.mean(prediction_i[4:6,:], axis=0)159        temp_wk56_predictions[i,:] = np.mean(prediction_i[6:8,:], axis=0)160            161    162    # sum precip predictions and average temp predictions over the 2-week target period163    # clip precip predictions to zero since precipitations cannot be negative164    precip_wk34_prediction = np.mean(precip_wk34_predictions, axis=0)165    precip_wk34_prediction = precip_wk34_prediction.clip(0)166    precip_wk56_prediction = np.mean(precip_wk56_predictions, axis=0)167    precip_wk56_prediction = precip_wk56_prediction.clip(0)168    temp_wk34_prediction = np.mean(temp_wk34_predictions, axis=0)169    temp_wk56_prediction = np.mean(temp_wk56_predictions, axis=0)170    171    # Get target date objects172    deadline_date = datetime.strptime(datestr, '%Y%m%d')173    target_date_34w = get_target_date_eval(datestr, "34w")174    target_date_56w = get_target_date_eval(datestr, "56w")175    176    # Get target date strings177    target_date_str_34w = datetime.strftime(target_date_34w, '%Y%m%d')178    target_date_str_56w = datetime.strftime(target_date_56w, '%Y%m%d')179    180    # Get lat, lon and pred template arrays181    template_f = os.path.join(dir_train_data, "apcp_week34_template.nc")182    template_ds = Dataset(template_f)183    template_lat = template_ds.variables["lat"][:]184    template_lon = template_ds.variables["lon"][:]185    template_var = template_ds.variables["apcp_week34"][:]186    187    # Determine variables, horizons and corresponding predictions to be saved188    gt_vars = ["precip", "tmp2m"] 189    horizons =  ["34w", "56w"] 190    predictions = [precip_wk34_prediction, precip_wk56_prediction, temp_wk34_prediction, temp_wk56_prediction]191    tasks = [f"{region}_{g}_{t}" for g, t in itertools.product(gt_vars, horizons)]192    193    # Format predictions to standard pandas contest prediction format.194    for task, prediction in zip(tasks, predictions):195        out_dir = os.path.join(dir_submodel_forecasts, submodel_name, task) 196        make_directories(out_dir)197        198        pred_file =  os.path.join(dir_train_data, "contest_latlons.h5")199        pred = read_with_lock(pred_file)200        #pred = pd.read_hdf(pred_file)201        202        if "34w" in task:203            pred["start_date"] = target_date_34w204            out_file = os.path.join(out_dir, f"{task}-{target_date_str_34w}.h5")205        elif "56" in task:206            pred["start_date"] = target_date_56w207            out_file = os.path.join(out_dir, f"{task}-{target_date_str_56w}.h5")208                   209        pred["pred"] = np.nan   210        a = template_var 211        # save predictions into array212        for loc in range(len(prediction)):213            index = points[loc]...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.