Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use isin method in pandera

Best Python code snippet using pandera_python

gbm.py

Source:gbm.py

1# Initialize data2import os3import gc4import numpy as np # linear algebra5import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)6from sklearn.metrics import roc_auc_score as ras7from sklearn.model_selection import cross_val_score8from sklearn.preprocessing import LabelEncoder9import lightgbm as lgb10#%%11path = "../../input/"12path2 = "../../output/"13test_df = pd.read_csv(path+"Challenge_20180423.csv")14train_df = pd.read_csv(path+"def_train.csv")15#dateMin = pd.read_csv(path+"dateMin.csv")16submission = pd.read_csv(path+"sample_submission.csv")17valid_df = pd.read_csv(path+"def_valid.csv")18#%%19ohe = pd.get_dummies(train_df["BuySell"])20columns = ohe.columns21for j in columns:22    train_df[j] = ohe[j]23train_df.drop("BuySell", axis=1, inplace=True)24# =============================================================================25ohe = pd.get_dummies(valid_df["BuySell"])26columns = ohe.columns27for j in columns: 28    valid_df[j] = ohe[j]29valid_df.drop("BuySell", axis=1, inplace=True)30# =============================================================================31# --> Test Treatment <-32test_df.drop("PredictionIdx", axis=1, inplace=True)33ohe = pd.get_dummies(test_df["BuySell"])34columns = ohe.columns35for j in columns:36    test_df[j] = ohe[j]37test_df.drop("BuySell", axis=1, inplace=True)38#%%39# --> Categorical Isin <--40isin_df = pd.read_csv(path+"isinTrat2.csv")41#isin_df['yNr'] = 042#isin_df['yNr'][isin_df['CompositeRating'] == 'NR'] = 143#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'NR'] = 044#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'D'] = 145#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'DD+'] = 246#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'DDD'] = 347#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'DDD+'] = 448#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'C'] = 549#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'C+'] = 650#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CC-'] = 751#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CC'] = 852#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CC+'] = 953#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CCC-'] = 1054#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CCC'] = 1155#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CCC+'] = 1256#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'B-'] = 1357#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'B'] = 1458#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'B+'] = 1559#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BB-'] = 1660#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BB'] = 1761#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BB+'] = 1862#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BBB-'] = 1963#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BBB'] = 2064#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BBB+'] = 2165#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'A-'] = 2266#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'A'] = 2367#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'A+'] = 2468#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AA-'] = 2569#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AA'] = 2670#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AA+'] = 2771#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AAA'] = 2872#isin_df['CompositeRating'] = isin_df['CompositeRating'].astype(int)73categorical_feats = [f for f in isin_df.columns if isin_df[f].dtype == 'object']74cat_feats_idx = np.unique([np.argmax(isin_df.columns == feat) for feat in categorical_feats])75all_cat = categorical_feats76for i in categorical_feats:77    if (len(isin_df[i].unique()) > 10):78        isin_df[i], indexer = pd.factorize(isin_df[i])79    else:80        ohe = pd.get_dummies(isin_df[i])81        columns = ohe.columns82        for j in columns:83            isin_df[j] = ohe[j]84        isin_df.drop(i, axis=1, inplace=True)85#categorical_feats = [f for f in isin_df.columns if isin_df[f].dtype == 'object']86print(categorical_feats)87# --> Categorical Customer <--88cost_df = pd.read_csv(path+"Customer.csv")89del cost_df['Region']90categorical_feats = [f for f in cost_df.columns if cost_df[f].dtype == 'object']91for i in categorical_feats:92    if (len(cost_df[i].unique()) > 10):93        cost_df[i], indexer = pd.factorize(cost_df[i])94    else:95        ohe = pd.get_dummies(cost_df[i])96        columns = ohe.columns97        for j in columns:98            cost_df[j] = ohe[j]99        cost_df.drop(i, axis=1, inplace=True)100#categorical_feats += [f for f in cost_df.columns if cost_df[f].dtype == 'object']101all_cat = all_cat + categorical_feats102print(categorical_feats)103print(all_cat)104#%%105# --> Merging <--106train_df = pd.merge(train_df, isin_df, on='IsinIdx', how='left', sort=False)107train_df = pd.merge(train_df, cost_df, on='CustomerIdx', how='left', sort=False)108#train_df = pd.merge(train_df, dateMin, on=['CustomerIdx', 'IsinIdx'], how='left', sort=False)109valid_df = pd.merge(valid_df, isin_df, on='IsinIdx', how='left', sort=False)110valid_df = pd.merge(valid_df, cost_df, on='CustomerIdx', how='left', sort=False)111#valid_df = pd.merge(valid_df, dateMin, on=['CustomerIdx', 'IsinIdx'], how='left', sort=False)112test_df = pd.merge(test_df, isin_df, on='IsinIdx', how='left', sort=False)113test_df = pd.merge(test_df, cost_df, on='CustomerIdx', how='left', sort=False)114#test_df = pd.merge(test_df, dateMin, on=['CustomerIdx', 'IsinIdx'], how='left', sort=False)115#train_df['dateMin'] = pd.to_datetime(train_df['dateMin'], format='%Y%m%d')116#train_df['dateAux'] = pd.to_datetime(train_df['DateKey'], format='%Y%m%d')117#train_df['dateMin'] = train_df['dateAux'] - train_df['dateMin']118#train_df['dateMin'] = train_df['dateMin'].dt.days119#train_df['dateMin'] = train_df['dateMin'].astype(int)120#train_df.drop('dateAux', axis=1, inplace=True)121#122#valid_df['dateMin'] = pd.to_datetime(valid_df['dateMin'], format='%Y%m%d')123#valid_df['dateAux'] = pd.to_datetime(valid_df['DateKey'], format='%Y%m%d')124#valid_df['dateMin'] = valid_df['dateAux'] - valid_df['dateMin']125#valid_df['dateMin'] = valid_df['dateMin'].dt.days126#valid_df['dateMin'] = valid_df['dateMin'].astype(int)127#valid_df.drop('dateAux', axis=1, inplace=True)128#129#test_df['dateMin'] = pd.to_datetime(test_df['dateMin'], format='%Y%m%d')130#test_df['dateAux'] = pd.to_datetime(test_df['DateKey'], format='%Y%m%d')131#test_df['dateMin'] = test_df['dateAux'] - test_df['dateMin']132#test_df['dateMin'] = test_df['dateMin'].dt.days133#test_df['dateMin'] = test_df['dateMin'].astype(int)134#test_df.drop('dateAux', axis=1, inplace=True)135train_df = train_df[train_df['DateKey'] != 20180416]136y_train = train_df['CustomerInterest']137y_valid = valid_df['CustomerInterest']138del test_df['DateKey']139del test_df['CustomerInterest']140del train_df['DateKey'] 141del train_df['CustomerInterest']142del valid_df['DateKey']143del valid_df['CustomerInterest']144lgb_train = lgb.Dataset(train_df, y_train, free_raw_data=False)145num_train, num_feature = train_df.shape146lgb_eval = lgb.Dataset(valid_df, y_valid, reference=lgb_train, free_raw_data=False)147# specify your configurations as a dict148params = {149    'boosting_type': 'gbdt',150    'objective': 'binary',151    'metric': 'auc',152    'num_leaves': 31,153    'learning_rate': 0.15154}155# generate a feature name156feature_name = ['feature_' + str(col) for col in range(num_feature)]157print('Start training...')158# feature_name and categorical_feature159gbm = lgb.train(params,160                lgb_train,161                num_boost_round=500,162                valid_sets=lgb_train,  # eval training data163                feature_name=feature_name)164y_pred = gbm.predict(valid_df)...

v_004.py

Source:v_004.py

1import numpy as np # linear algebra2import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)3import lightgbm as lgb4import xgboost as xgb5from sklearn.metrics import roc_auc_score6#%%7path = "../../input/"8path2 = "../../output/"9test_df = pd.read_csv(path+"Challenge_20180423.csv")10train_df = pd.read_csv(path+"def_train.csv")11#dateMin = pd.read_csv(path+"dateMin.csv")12submission = pd.read_csv(path+"sample_submission.csv")13valid_df = pd.read_csv(path+"def_valid.csv")14#%%15ohe = pd.get_dummies(train_df["BuySell"])16columns = ohe.columns17for j in columns:18    train_df[j] = ohe[j]19train_df.drop("BuySell", axis=1, inplace=True)20# =============================================================================21ohe = pd.get_dummies(valid_df["BuySell"])22columns = ohe.columns23for j in columns: 24    valid_df[j] = ohe[j]25valid_df.drop("BuySell", axis=1, inplace=True)26# =============================================================================27# --> Test Treatment <-28test_df.drop("PredictionIdx", axis=1, inplace=True)29ohe = pd.get_dummies(test_df["BuySell"])30columns = ohe.columns31for j in columns:32    test_df[j] = ohe[j]33test_df.drop("BuySell", axis=1, inplace=True)34#%%35# --> Categorical Isin <--36isin_df = pd.read_csv(path+"Isin.csv")37#isin_df['yNr'] = 038#isin_df['yNr'][isin_df['CompositeRating'] == 'NR'] = 139#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'NR'] = 040#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'D'] = 141#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'DD+'] = 242#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'DDD'] = 343#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'DDD+'] = 444#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'C'] = 545#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'C+'] = 646#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CC-'] = 747#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CC'] = 848#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CC+'] = 949#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CCC-'] = 1050#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CCC'] = 1151#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CCC+'] = 1252#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'B-'] = 1353#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'B'] = 1454#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'B+'] = 1555#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BB-'] = 1656#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BB'] = 1757#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BB+'] = 1858#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BBB-'] = 1959#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BBB'] = 2060#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BBB+'] = 2161#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'A-'] = 2262#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'A'] = 2363#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'A+'] = 2464#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AA-'] = 2565#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AA'] = 2666#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AA+'] = 2767#isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AAA'] = 2868#isin_df['CompositeRating'] = isin_df['CompositeRating'].astype(int)69categorical_feats = [f for f in isin_df.columns if isin_df[f].dtype == 'object']70all_cat = categorical_feats71for i in categorical_feats:72    if (len(isin_df[i].unique()) > 10):73        isin_df[i], indexer = pd.factorize(isin_df[i])74    else:75        ohe = pd.get_dummies(isin_df[i])76        columns = ohe.columns77        for j in columns:78            isin_df[j] = ohe[j]79        isin_df.drop(i, axis=1, inplace=True)80categorical_feats = [f for f in isin_df.columns if isin_df[f].dtype == 'object']81print(categorical_feats)82# --> Categorical Customer <--83cost_df = pd.read_csv(path+"Customer.csv")84del cost_df['Region']85categorical_feats += [f for f in cost_df.columns if cost_df[f].dtype == 'object']86for i in categorical_feats:87    if (len(cost_df[i].unique()) > 10):88        cost_df[i], indexer = pd.factorize(cost_df[i])89    else:90        ohe = pd.get_dummies(cost_df[i])91        columns = ohe.columns92        for j in columns:93            cost_df[j] = ohe[j]94        cost_df.drop(i, axis=1, inplace=True)95categorical_feats = [f for f in cost_df.columns if cost_df[f].dtype == 'object']96all_cat = all_cat + categorical_feats97print(categorical_feats)98print(all_cat)99#%%100# --> Merging <--101train_df = pd.merge(train_df, isin_df, on='IsinIdx', how='left', sort=False)102train_df = pd.merge(train_df, cost_df, on='CustomerIdx', how='left', sort=False)103#train_df = pd.merge(train_df, dateMin, on=['CustomerIdx', 'IsinIdx'], how='left', sort=False)104valid_df = pd.merge(valid_df, isin_df, on='IsinIdx', how='left', sort=False)105valid_df = pd.merge(valid_df, cost_df, on='CustomerIdx', how='left', sort=False)106#valid_df = pd.merge(valid_df, dateMin, on=['CustomerIdx', 'IsinIdx'], how='left', sort=False)107test_df = pd.merge(test_df, isin_df, on='IsinIdx', how='left', sort=False)108test_df = pd.merge(test_df, cost_df, on='CustomerIdx', how='left', sort=False)109#test_df = pd.merge(test_df, dateMin, on=['CustomerIdx', 'IsinIdx'], how='left', sort=False)110#train_df['dateMin'] = pd.to_datetime(train_df['dateMin'], format='%Y%m%d')111#train_df['dateAux'] = pd.to_datetime(train_df['DateKey'], format='%Y%m%d')112#train_df['dateMin'] = train_df['dateAux'] - train_df['dateMin']113#train_df['dateMin'] = train_df['dateMin'].dt.days114#train_df['dateMin'] = train_df['dateMin'].astype(int)115#train_df.drop('dateAux', axis=1, inplace=True)116#117#valid_df['dateMin'] = pd.to_datetime(valid_df['dateMin'], format='%Y%m%d')118#valid_df['dateAux'] = pd.to_datetime(valid_df['DateKey'], format='%Y%m%d')119#valid_df['dateMin'] = valid_df['dateAux'] - valid_df['dateMin']120#valid_df['dateMin'] = valid_df['dateMin'].dt.days121#valid_df['dateMin'] = valid_df['dateMin'].astype(int)122#valid_df.drop('dateAux', axis=1, inplace=True)123#124#test_df['dateMin'] = pd.to_datetime(test_df['dateMin'], format='%Y%m%d')125#test_df['dateAux'] = pd.to_datetime(test_df['DateKey'], format='%Y%m%d')126#test_df['dateMin'] = test_df['dateAux'] - test_df['dateMin']127#test_df['dateMin'] = test_df['dateMin'].dt.days128#test_df['dateMin'] = test_df['dateMin'].astype(int)129#test_df.drop('dateAux', axis=1, inplace=True)130# desligar se treinar pra geral131train_df = train_df[train_df['DateKey'] != 20180416]132y_train = train_df['CustomerInterest']133y_valid = valid_df['CustomerInterest']134del test_df['DateKey']135del test_df['CustomerInterest']136del train_df['DateKey'] 137del train_df['CustomerInterest']138del valid_df['DateKey']139del valid_df['CustomerInterest']140# --> Model <--141d_train = xgb.DMatrix(train_df, y_train)142d_valid = xgb.DMatrix(valid_df, y_valid)143#%%144watchlist = [(d_train, 'train'), (d_valid, 'valid')]145#watchlist = [(d_train, 'train')]146params = {147    'eta': 0.1,148    'max_depth': 10, 149    'subsample': 0.9, 150    'colsample_bytree': 0.9, 151    'colsample_bylevel':0.9,152    #'min_child_weight':10,153    #'alpha':4,154    'objective': 'binary:logistic',155    'eval_metric': 'auc', 156    'nthread':8,157    'seed' : 0,158    'silent': True}159    160model = xgb.train(params, d_train, 3000, watchlist, 161                  maximize=True, early_stopping_rounds = 20, 162                  verbose_eval=1)163#%%164preds = model.predict(xgb.DMatrix(test_df))165submission['CustomerInterest'] = preds166submission.to_csv(path2+"test2.csv", index=False)...

tratIsin3.py

Source:tratIsin3.py

1import numpy as np # linear algebra2import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)3import lightgbm as lgb4import xgboost as xgb5from sklearn.metrics import roc_auc_score6path = "../../input/"7path2 = "../../output/"8test_df = pd.read_csv(path+"Challenge_20180423.csv")9train_df = pd.read_csv(path+"trainDropVal.csv")10mm_df = pd.read_csv(path+"mmOnlyConversion.csv")11isin_df = pd.read_csv(path+"Isin.csv")12#dateMin = pd.read_csv(path+"dateMin.csv")13submission = pd.read_csv(path+"sample_submission.csv")14valid_df = pd.read_csv(path+"valid.csv")15aux1 = pd.to_datetime(isin_df['ActualMaturityDateKey'], format='%Y%m%d') 16aux2 = pd.to_datetime('20180423', format='%Y%m%d') 17aux3 = pd.to_datetime(isin_df['IssueDateKey'], format='%Y%m%d') 18isin_df['diffMatToEm'] = aux1 - aux319isin_df['diffMatToNow'] = aux1 - aux220isin_df['diffEmToNow'] = aux2 - aux321isin_df['diffMatToNow'] = isin_df['diffMatToNow'].dt.days22isin_df['diffMatToNow'] = isin_df['diffMatToNow'].astype(int)23isin_df['diffEmToNow'] = isin_df['diffEmToNow'].dt.days24isin_df['diffEmToNow'] = isin_df['diffEmToNow'].astype(int)25isin_df['diffMatToEm'] = isin_df['diffMatToEm'].dt.days26isin_df['diffMatToEm'] = isin_df['diffMatToEm'].astype(int)27del isin_df['ActualMaturityDateKey'],  isin_df['IssueDateKey']28isin_df['yNr'] = 029isin_df['yNr'][isin_df['CompositeRating'] == 'NR'] = 130isin_df['CompositeRating'][isin_df['CompositeRating'] == 'NR'] = 031isin_df['CompositeRating'][isin_df['CompositeRating'] == 'D'] = 132isin_df['CompositeRating'][isin_df['CompositeRating'] == 'DD+'] = 233isin_df['CompositeRating'][isin_df['CompositeRating'] == 'DDD'] = 334isin_df['CompositeRating'][isin_df['CompositeRating'] == 'DDD+'] = 435isin_df['CompositeRating'][isin_df['CompositeRating'] == 'C'] = 536isin_df['CompositeRating'][isin_df['CompositeRating'] == 'C+'] = 637isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CC-'] = 738isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CC'] = 839isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CC+'] = 940isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CCC-'] = 1041isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CCC'] = 1142isin_df['CompositeRating'][isin_df['CompositeRating'] == 'CCC+'] = 1243isin_df['CompositeRating'][isin_df['CompositeRating'] == 'B-'] = 1344isin_df['CompositeRating'][isin_df['CompositeRating'] == 'B'] = 1445isin_df['CompositeRating'][isin_df['CompositeRating'] == 'B+'] = 1546isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BB-'] = 1647isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BB'] = 1748isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BB+'] = 1849isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BBB-'] = 1950isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BBB'] = 2051isin_df['CompositeRating'][isin_df['CompositeRating'] == 'BBB+'] = 2152isin_df['CompositeRating'][isin_df['CompositeRating'] == 'A-'] = 2253isin_df['CompositeRating'][isin_df['CompositeRating'] == 'A'] = 2354isin_df['CompositeRating'][isin_df['CompositeRating'] == 'A+'] = 2455isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AA-'] = 2556isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AA'] = 2657isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AA+'] = 2758isin_df['CompositeRating'][isin_df['CompositeRating'] == 'AAA'] = 2859isin_df['CompositeRating'] = isin_df['CompositeRating'].astype(int)60#mm_df['USD'] = 161#mm_df['ARO'] = mm_df['ARS']62#mm_df['INR'] = mm_df['EUR']63#mm_df['FRF'] = mm_df['EUR']64#mm_df['ITL'] = mm_df['EUR']65#mm_df['DEM'] = mm_df['EUR']66#mm_df['CNH'] = mm_df['CNY']67#for i, r in isin_df.iterrows():68#    isin_df['IssuedAmount'][isin_df['IsinIdx'] == i] = isin_df['IssuedAmount'][isin_df['IsinIdx'] == i] * mm_df[r['Currency']].mean()...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.