How to use read_with_lock method in yandex-tank
Best Python code snippet using yandex-tank
util.py
Source:util.py 
...586 def get_file(self, cache_size=None):587 cache_size = self.cache_size if not cache_size else cache_size588 fileobj = FileLike(self, cache_size)589 return fileobj590 def read_with_lock(self, pos, _len=None):591 """592 Reads {_len} characters if _len is not None else reads line593 :param pos: start reading position594 :param _len: number of characters to read595 :rtype: (string, int)596 """597 self.wait_lock()598 try:599 self._opened_file.seek(pos)600 result = self._opened_file.read(_len) if _len is not None else self._opened_file.readline()601 stop_pos = self._opened_file.tell()602 finally:603 self.unlock()604 if not result and self.stop.is_set():605 result = None606 return result, stop_pos607 @retry(wait_random_min=5, wait_random_max=20, stop_max_delay=10000,608 retry_on_exception=FileLockedError.retry, wrap_exception=True)609 def wait_lock(self):610 if self._is_locked:611 raise FileLockedError('Generator output file {} is locked'.format(self.filename))612 else:613 self._is_locked = True614 return True615 def unlock(self):616 self._is_locked = False617class FileLike(object):618 def __init__(self, multireader, cache_size):619 """620 :type multireader: FileMultiReader621 """622 self.multireader = multireader623 self.cache_size = cache_size624 self._cursor = 0625 def read(self, _len=None):626 _len = self.cache_size if not _len else _len627 result, self._cursor = self.multireader.read_with_lock(self._cursor, _len)628 return result629 def readline(self):630 result, self._cursor = self.multireader.read_with_lock(self._cursor)631 return result632def get_callstack():633 """634 Get call stack, clean wrapper functions from it and present635 in dotted notation form636 """637 stack = inspect.stack(context=0)638 cleaned = [frame[3] for frame in stack if frame[3] != 'wrapper']639 return '.'.join(cleaned[1:])640def timeit(min_duration_sec):641 def timeit_fixed(func):642 def wrapper(*args, **kwargs):643 start_time = time.time()644 result = func(*args, **kwargs)...
predict_keras.py
Source:predict_keras.py
...29from subseasonal_toolkit.utils.experiments_util import pandas2hdf30from subseasonal_toolkit.utils.general_util import make_directories31from subseasonal_toolkit.utils.experiments_util import get_target_date as get_target_date_eval32from subseasonal_toolkit.models.salient.salient_util import ONE_WEEK, ONE_DAY, WEEKS, dir_submodel_forecasts, dir_train_results, dir_train_data, dir_predict_data, year_fraction, get_target_date33def read_with_lock(filename):34 """Open an hdf file with a lock and read. 35 Args:36 filename (str): path of hdf file to be read with a lock.37 Returns:38 openned pandas dataframe.39 """40 #with FileLock(filename+'lock'):41 # df = pd.read_hdf(filename) 42 df = pd.read_hdf(filename) 43 subprocess.call(f"rm {filename}lock", shell=True)44 return df45def generate_windows(x):46 """generates a sliding window over the data of length window_size.47 Args:48 x: array of input features used the NN49 Returns:50 reshaped array of input features so that each example is a concatenation of the past 10 weeks of data.51 """52 window_size = 1053 result = []54 for i in range(x.shape[0]+1-window_size):55 result.append(x[i:i+window_size, ...])56 return np.stack(result)57def compile_input(submodel_name, datestr):58 """Compile input data to be used by NN.59 Args:60 submodel_name (str): string consisting of the ground truth variable 61 ids used for training and the date of the last training example.62 datestr (str): YYMMDD string of first day of target 2-week period.63 Returns:64 input_data: array of input features used generate predictions by the NNs.65 """ 66 67 #get input start and end date 68 target_date = datetime.strptime(datestr, '%Y%m%d') 69 end_date = target_date - timedelta(days=((target_date.weekday() - 5) % 7))70 end_date = datetime(end_date.year, end_date.month, end_date.day)71 start_date = end_date - ONE_WEEK * WEEKS + ONE_DAY 72 73 # load date data 74 date_data_file = os.path.join(dir_train_data, "date.pickle")75 date_vectors = pickle.load(open(date_data_file, 'rb'))76 date_vectors_all = sorted([d[0] for d in date_vectors])77 end_date_sn = datetime.strptime(submodel_name[-8:], '%Y%m%d')78 date_vectors = [d for d in date_vectors_all if (d.astype(object).year<=end_date_sn.year) and (d.astype(object).month<=end_date_sn.month) and (d.astype(object).day<=end_date_sn.day)] 79 last_i = len(date_vectors)80 # load training data81 # load sst data82 training_sst_data_file = os.path.join(dir_train_data, "sst.pickle")83 training_sst_vectors_all = pickle.load(open(training_sst_data_file, 'rb')) 84 # load time data85 training_time_data_file = os.path.join(dir_train_data, "time.pickle")86 training_time_vectors = pickle.load(open(training_time_data_file, 'rb'))87 training_time_vectors_all = np.reshape(training_time_vectors,(training_time_vectors.shape[0],1))88 89 # account for early train stop submodels90 training_sst_vectors = training_sst_vectors_all[:last_i,:]91 training_time_vectors = training_time_vectors_all[:last_i,:]92 ## Load input data to generate a prediction93 # load sst data94 sst_data_file = os.path.join(dir_predict_data, "salient_fri", datestr, "sst.pickle")95 if isfile(sst_data_file):96 sst_vectors = pickle.load(open(sst_data_file, 'rb'))97 else:98 #find datestr in dates99 d = [d for d in date_vectors_all if (d.astype(object).year==target_date.year) and (d.astype(object).month==target_date.month) and (d.astype(object).day==target_date.day)]100 i = date_vectors_all.index(d)101 sst_vectors = training_sst_vectors_all[i-10:i,:]102 sst_vectors = (sst_vectors - np.amin(training_sst_vectors)) * 1./(np.amax(training_sst_vectors) - np.amin(training_sst_vectors))103 104 # compile input data105 input_data = sst_vectors106 107 108 return input_data109def add_i_time(input_data, submodel_name, datestr, i_time=False):110 """add i_time feature to input data to be used by NN.111 Args:112 input_data (float): array of input features used generate predictions by the NNs.113 submodel_name (str): string consisting of the ground truth variable 114 ids used for training and the date of the last training example.115 datestr (str): YYMMDD string of first day of target 2-week period.116 i_time (bool): if True, include time vector as an input feature (default: False).117 Returns:118 input_data: array of input features used generate predictions by the NNs.119 """120 121 #get input start and end date 122 target_date = datetime.strptime(datestr, '%Y%m%d') 123 end_date = target_date - timedelta(days=((target_date.weekday() - 5) % 7))124 end_date = datetime(end_date.year, end_date.month, end_date.day)125 start_date = end_date - ONE_WEEK * WEEKS + ONE_DAY 126 127 #load time data128 if i_time:129 time_vectors = np.zeros((WEEKS, 1))130 day = start_date131 for i in range(time_vectors.shape[0]):132 time_vectors[i, 0] = year_fraction(day)133 day += ONE_WEEK134 135 # compile input data136 if i_time:137 input_data = np.concatenate((input_data, time_vectors), axis=1)138 139 return input_data140def main():141 #"""142 #example usage to generate original salient forecasts143 # python src/models/salient/predict_keras.py -d 20200811 -sn salient_20170201144 # input arguments145 parser = argparse.ArgumentParser()146 parser.add_argument('-d', '--date', help='Submission date')147 parser.add_argument('-sn', '--submodel_name', default='salient_fri_20170201')148 parser.add_argument('--region', '-r', default='contest')149 args = parser.parse_args()150 # set args151 submodel_name = args.submodel_name152 target_date, datestr = get_target_date('Generate predictions', args.date)153 region = args.region154 155 # Setup directory where the submodel's weights are saved.156 dir_weights = os.path.join(dir_train_results, submodel_name, f"{region}_{submodel_name}")157 # map locations with points of interest158 mask_f = os.path.join("data", "masks", "us_mask.nc")159 mask_ds = Dataset(mask_f) if region.startswith('us') else Dataset(mask_f.replace("us_", "fcstrodeo_"))160 mask_lat = mask_ds.variables['lat'][:]161 mask_lon = mask_ds.variables['lon'][:]162 points_idx = np.where(mask_ds.variables['mask'][:])163 points = np.array((points_idx[0], points_idx[1])).T164 165 # Import and run models166 model_names = glob.glob(os.path.join(dir_weights, "k_model_*.h5"))167 N_models = len(model_names)168 169 if region.startswith('us'):170 num_of_gc = 862 171 elif region.startswith('contest'):172 num_of_gc = 514173 else:174 num_of_gc = 348175 # Create empty template array for predictions to be generated176 precip_wk34_predictions = np.zeros((N_models,num_of_gc))177 precip_wk56_predictions = np.zeros((N_models,num_of_gc))178 temp_wk34_predictions = np.zeros((N_models,num_of_gc))179 temp_wk56_predictions = np.zeros((N_models,num_of_gc))180 181 # Compile input data182 input_data_all = compile_input(submodel_name, datestr)183 184 # Generate predictions for each of the top 10 selected NN members in the ensemble185 for i in range(N_models):186 187 # Load NN ensemble member188 model_name = model_names[i]189 model = keras.models.load_model(model_name)190 # If NN was trained on time as an input feature, add time to the compile input data191 result = re.search('time(.*).h5', model_name)192 input_set = int(result.group(1))193 input_data = add_i_time(input_data_all, submodel_name, datestr, i_time=bool(input_set))194 input_data = generate_windows(input_data)195 # Generate predictions196 prediction_i = model.predict(input_data)197 # predictions for a 2-week target period are the accumulation of precip 198 # the mean temperature over the target period 199 prediction_i = np.reshape(prediction_i,(8,num_of_gc)) 200 precip_wk34_predictions[i,:] = np.sum(prediction_i[0:2,:], axis=0)201 precip_wk56_predictions[i,:] = np.sum(prediction_i[2:4,:], axis=0)202 temp_wk34_predictions[i,:] = np.mean(prediction_i[4:6,:], axis=0)203 temp_wk56_predictions[i,:] = np.mean(prediction_i[6:8,:], axis=0)204 205 206 # sum precip predictions and average temp predictions over the 2-week target period207 # clip precip predictions to zero since precipitations cannot be negative208 precip_wk34_prediction = np.mean(precip_wk34_predictions, axis=0)209 precip_wk56_prediction = np.mean(precip_wk56_predictions, axis=0)210 precip_wk34_prediction = precip_wk34_prediction.clip(0)211 precip_wk56_prediction = precip_wk56_prediction.clip(0)212 temp_wk34_prediction = np.mean(temp_wk34_predictions, axis=0)213 temp_wk56_prediction = np.mean(temp_wk56_predictions, axis=0)214 215 216 # Get target date objects217 deadline_date = datetime.strptime(datestr, '%Y%m%d')218 target_date_34w = get_target_date_eval(datestr, "34w")219 target_date_56w = get_target_date_eval(datestr, "56w")220 221 # Get target date strings222 target_date_str_34w = datetime.strftime(target_date_34w, '%Y%m%d')223 target_date_str_56w = datetime.strftime(target_date_56w, '%Y%m%d')224 225 # Get lat, lon and pred template arrays226 template_f = resource_filename("subseasonal_toolkit", os.path.join("models", "salient", "data", "apcp_week34_template.nc"))227 template_ds = Dataset(template_f)228 template_lat = template_ds.variables["lat"][:]229 template_lon = template_ds.variables["lon"][:]230 template_var = template_ds.variables["apcp_week34"][:]231 232 # Determine variables, horizons and corresponding predictions to be saved233 gt_vars = ["precip", "tmp2m"] 234 horizons = ["34w", "56w"] 235 predictions = [precip_wk34_prediction, precip_wk56_prediction, temp_wk34_prediction, temp_wk56_prediction]236 tasks = [f"{region}_{g}_{t}" for g, t in itertools.product(gt_vars, horizons)]237 238 # Format predictions to standard pandas contest prediction format.239 for task, prediction in zip(tasks, predictions):240 out_dir = os.path.join(dir_submodel_forecasts, submodel_name, task) 241 make_directories(out_dir)242 243 pred_file = os.path.join(dir_train_data, "contest_latlons.h5")244 pred = read_with_lock(pred_file)245 #pred = pd.read_hdf(pred_file)246 247 if "34w" in task:248 pred["start_date"] = target_date_34w249 out_file = os.path.join(out_dir, f"{task}-{target_date_str_34w}.h5")250 elif "56" in task:251 pred["start_date"] = target_date_56w252 out_file = os.path.join(out_dir, f"{task}-{target_date_str_56w}.h5")253 254 pred["pred"] = np.nan 255 a = template_var 256 # save predictions into array257 for loc in range(len(prediction)):258 index = points[loc]...
predict_keras_hindcasts.py
Source:predict_keras_hindcasts.py
...27from subseasonal_toolkit.utils.general_util import make_directories28from subseasonal_toolkit.utils.experiments_util import pandas2hdf29from subseasonal_toolkit.utils.experiments_util import get_target_date as get_target_date_eval30from subseasonal_toolkit.models.salient.salient_util import ONE_DAY, dir_train_data, dir_train_results, dir_submodel_forecasts, get_target_date31def read_with_lock(filename):32 """Open an hdf file with a lock and read. 33 Args:34 filename (str): path of hdf file to be read with a lock.35 Returns:36 openned pandas dataframe.37 """38 #with FileLock(filename+'lock'):39 # df = pd.read_hdf(filename) 40 df = pd.read_hdf(filename) 41 subprocess.call(f"rm {filename}lock", shell=True)42 return df43def generate_windows(x):44 """generates a sliding window over the data of length window_size.45 Args:46 x: array of input features used the NN47 Returns:48 reshaped array of input features so that each example is a concatenation of the past 10 weeks of data.49 """50 window_size = 1051 result = []52 for i in range(x.shape[0]+1-window_size):53 result.append(x[i:i+window_size, ...])54 return np.stack(result)55def compile_input(date, i_time=False):56 ## Load training data57 # load time data58 training_time_data_file = os.path.join(dir_train_data, "time.pickle") # (t,1)59 training_time_vectors = pickle.load(open(training_time_data_file, 'rb'))60 training_time_vectors = np.reshape(training_time_vectors,(training_time_vectors.shape[0],1))61 # load sst data62 training_sst_data_file = os.path.join(dir_train_data, "sst.pickle") # (t,loc)63 training_sst_vectors = pickle.load(open(training_sst_data_file, 'rb'))64 # load precipitation data65 training_location_precip_file = os.path.join(dir_train_data, "precip.pickle") # (loc,t)66 training_precip_data = pickle.load(open(training_location_precip_file, 'rb'))67 training_precip_data = training_precip_data.T68 # load temperature data69 training_location_temp_file = os.path.join(dir_train_data, "temp.pickle") # (loc,t)70 training_temp_data = pickle.load(open(training_location_temp_file, 'rb'))71 training_temp_data = training_temp_data.T72 # make precip data only as long as temp data73 training_precip_data = training_precip_data[:training_temp_data.shape[0],:]74 # ensure same length vectors and normalize75 training_time_vectors = training_time_vectors[:training_precip_data.shape[0],:]76 training_sst_vectors = training_sst_vectors[:training_precip_data.shape[0],:]77 ## Generate input data from date78 date_data_file = os.path.join(dir_train_data, "date.pickle") # (time,1)79 dates = pickle.load(open(date_data_file, 'rb'))80 dates = dates[:training_sst_vectors.shape[0]]81 # dates represent middle of the week (Wednesday) sst or sss data is for Sun-Sat82 # Offset our date by 3 days so we're sure we don't have future data in the83 # input set84 cutoff_date = datetime.strptime(date, '%Y%m%d') - ONE_DAY * 385 input_indices = np.where(dates < np.datetime64(str(cutoff_date.date())))[0]86 # Take the last 10 weeks87 input_indices = input_indices[-10:]88 # load sst data89 sst_vectors = training_sst_vectors[input_indices]90 # Normalize91 sst_vectors = (sst_vectors - np.amin(training_sst_vectors)) * 1./(np.amax(training_sst_vectors) - np.amin(training_sst_vectors))92 # compile input data93 input_data = sst_vectors94 if i_time:95 time_vectors = training_time_vectors[input_indices]96 time_vectors = np.reshape(time_vectors,(time_vectors.shape[0],1))97 input_data = np.concatenate((input_data, time_vectors), axis=1)98 return input_data99def main():100 # input arguments101 parser = argparse.ArgumentParser()102 parser.add_argument('-d', '--date', help='Submission date')103 parser.add_argument('-sn', '--submodel_name', default='salient_fri_hindcasts_2010')104 parser.add_argument('--region', '-r', default='contest')105 args = parser.parse_args()106 # set args107 submodel_name = args.submodel_name108 target_date, datestr = get_target_date('Generate predictions', args.date)109 region = args.region110 111 # Setup directory where the submodel's weights are saved.112 dir_weights = os.path.join(dir_train_results, submodel_name, f"{region}_{submodel_name}")113 114 # map locations with points of interest115 mask_f = os.path.join("data", "masks", "us_mask.nc")116 mask_ds = Dataset(mask_f) if region.startswith('us') else Dataset(mask_f.replace("us_", "fcstrodeo_"))117 mask_lat = mask_ds.variables['lat'][:]118 mask_lon = mask_ds.variables['lon'][:]119 points_idx = np.where(mask_ds.variables['mask'][:])120 points = np.array((points_idx[0], points_idx[1])).T121 122 # Import and run models123 model_names = glob.glob(os.path.join(dir_weights, f"k_model_*.h5"))124 N_models = len(model_names)125 126 if region.startswith('us'):127 num_of_gc = 862 128 elif region.startswith('contest'):129 num_of_gc = 514130 else:131 num_of_gc = 348132 133 # Create empty template array for predictions to be generated134 precip_wk34_predictions = np.zeros((N_models,num_of_gc))135 precip_wk56_predictions = np.zeros((N_models,num_of_gc))136 temp_wk34_predictions = np.zeros((N_models,num_of_gc))137 temp_wk56_predictions = np.zeros((N_models,num_of_gc))138 139 # Generate predictions for each of the top 10 selected NN members in the ensemble140 for i in range(N_models):141 142 # Load NN ensemble member143 model_name = model_names[i]144 model = keras.models.load_model(model_name)145 # If NN was trained on time as an input feature, add time to the compile input data146 result = re.search('time(.*).h5', model_name)147 input_set = int(result.group(1))148 # Compile input data149 input_data_all = compile_input(datestr, i_time=bool(input_set))150 input_data = generate_windows(input_data_all)151 # Generate predictions152 prediction_i = model.predict(input_data)153 # predictions for a 2-week target period are the accumulation of precip 154 # the mean temperature over the target period 155 prediction_i = np.reshape(prediction_i,(8,num_of_gc)) 156 precip_wk34_predictions[i,:] = np.sum(prediction_i[0:2,:], axis=0)157 precip_wk56_predictions[i,:] = np.sum(prediction_i[2:4,:], axis=0)158 temp_wk34_predictions[i,:] = np.mean(prediction_i[4:6,:], axis=0)159 temp_wk56_predictions[i,:] = np.mean(prediction_i[6:8,:], axis=0)160 161 162 # sum precip predictions and average temp predictions over the 2-week target period163 # clip precip predictions to zero since precipitations cannot be negative164 precip_wk34_prediction = np.mean(precip_wk34_predictions, axis=0)165 precip_wk34_prediction = precip_wk34_prediction.clip(0)166 precip_wk56_prediction = np.mean(precip_wk56_predictions, axis=0)167 precip_wk56_prediction = precip_wk56_prediction.clip(0)168 temp_wk34_prediction = np.mean(temp_wk34_predictions, axis=0)169 temp_wk56_prediction = np.mean(temp_wk56_predictions, axis=0)170 171 # Get target date objects172 deadline_date = datetime.strptime(datestr, '%Y%m%d')173 target_date_34w = get_target_date_eval(datestr, "34w")174 target_date_56w = get_target_date_eval(datestr, "56w")175 176 # Get target date strings177 target_date_str_34w = datetime.strftime(target_date_34w, '%Y%m%d')178 target_date_str_56w = datetime.strftime(target_date_56w, '%Y%m%d')179 180 # Get lat, lon and pred template arrays181 template_f = os.path.join(dir_train_data, "apcp_week34_template.nc")182 template_ds = Dataset(template_f)183 template_lat = template_ds.variables["lat"][:]184 template_lon = template_ds.variables["lon"][:]185 template_var = template_ds.variables["apcp_week34"][:]186 187 # Determine variables, horizons and corresponding predictions to be saved188 gt_vars = ["precip", "tmp2m"] 189 horizons = ["34w", "56w"] 190 predictions = [precip_wk34_prediction, precip_wk56_prediction, temp_wk34_prediction, temp_wk56_prediction]191 tasks = [f"{region}_{g}_{t}" for g, t in itertools.product(gt_vars, horizons)]192 193 # Format predictions to standard pandas contest prediction format.194 for task, prediction in zip(tasks, predictions):195 out_dir = os.path.join(dir_submodel_forecasts, submodel_name, task) 196 make_directories(out_dir)197 198 pred_file = os.path.join(dir_train_data, "contest_latlons.h5")199 pred = read_with_lock(pred_file)200 #pred = pd.read_hdf(pred_file)201 202 if "34w" in task:203 pred["start_date"] = target_date_34w204 out_file = os.path.join(out_dir, f"{task}-{target_date_str_34w}.h5")205 elif "56" in task:206 pred["start_date"] = target_date_56w207 out_file = os.path.join(out_dir, f"{task}-{target_date_str_56w}.h5")208 209 pred["pred"] = np.nan 210 a = template_var 211 # save predictions into array212 for loc in range(len(prediction)):213 index = points[loc]...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run yandex-tank automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
