From b9b7ff555c90c97215721e7e2b5cd356b210ebdf Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Fri, 13 Dec 2024 13:47:48 +0100 Subject: [PATCH 01/67] fix version for scikit learn --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a58fef6..cb7e998 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ doubleml joblib numpy pandas -scikit-learn +scikit-learn==1.5.2 lightgbm itables ipykernel \ No newline at end of file From 44f4186806b1ee32a228f7c159c06aab0dbbbb92 Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Fri, 13 Dec 2024 14:49:25 +0100 Subject: [PATCH 02/67] add script for irm sensitivity based on APO & framework --- doc/irm/irm.qmd | 2 - scripts/irm/irm_apo_sensitivity.py | 181 +++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 scripts/irm/irm_apo_sensitivity.py diff --git a/doc/irm/irm.qmd b/doc/irm/irm.qmd index 0425f32..0dd1299 100644 --- a/doc/irm/irm.qmd +++ b/doc/irm/irm.qmd @@ -242,7 +242,6 @@ display_columns = [ ```{python} #| echo: false -score = "partialling out" level = 0.95 df_ate_95 = df[(df['level'] == level)][display_columns] @@ -252,7 +251,6 @@ make_pretty(df_ate_95, level, n_rep) ```{python} #| echo: false -score = "partialling out" level = 0.9 df_ate_9 = df[(df['level'] == level)][display_columns] diff --git a/scripts/irm/irm_apo_sensitivity.py b/scripts/irm/irm_apo_sensitivity.py new file mode 100644 index 0000000..432e004 --- /dev/null +++ b/scripts/irm/irm_apo_sensitivity.py @@ -0,0 +1,181 @@ +import numpy as np +import pandas as pd +from datetime import datetime +import time +import sys + +from sklearn.linear_model import LinearRegression, LogisticRegression +from lightgbm import LGBMRegressor, LGBMClassifier + +import doubleml as dml +from doubleml.datasets import make_confounded_irm_data + +# Number of repetitions +n_rep = 500 +max_runtime = 5.5 * 3600 # 5.5 hours in seconds + +# DGP pars +n_obs = 5000 +theta = 5.0 +trimming_threshold = 0.05 + +dgp_pars = { + "gamma_a": 0.198, + "beta_a": 0.582, + "theta": theta, + "var_epsilon_y": 1.0, + "trimming_threshold": trimming_threshold, + "linear": False, +} + +# test inputs +np.random.seed(42) +dgp_dict = make_confounded_irm_data(n_obs=int(1e+6), **dgp_pars) + +oracle_dict = dgp_dict['oracle_values'] +rho = oracle_dict['rho_ate'] +cf_y = oracle_dict['cf_y'] +cf_d = oracle_dict['cf_d_ate'] + +print(f"Confounding factor for Y: {cf_y}") +print(f"Confounding factor for D: {cf_d}") +print(f"Rho: {rho}") + +# to get the best possible comparison between different learners (and settings) we first simulate all datasets +np.random.seed(42) +datasets = [] +for i in range(n_rep): + dgp_dict = make_confounded_irm_data(n_obs=n_obs, **dgp_pars) + datasets.append(dgp_dict) + +# set up hyperparameters +hyperparam_dict = { + "learner_g": [("Linear Reg.", LinearRegression()), + ("LGBM", LGBMRegressor(n_estimators=500, learning_rate=0.01, min_child_samples=10, verbose=-1))], + "learner_m": [("Logistic Regr.", LogisticRegression()), + ("LGBM", LGBMClassifier(n_estimators=500, learning_rate=0.01, min_child_samples=10, verbose=-1)),], + "level": [0.95, 0.90], + "treatment_levels": [0.0, 1.0], +} + +# set up the results dataframe +df_results_detailed = pd.DataFrame() +sensitivity_err_count = 0 + +# start simulation +np.random.seed(42) +start_time = time.time() + +for i_rep in range(n_rep): + print(f"Repetition: {i_rep}/{n_rep}", end="\r") + + # Check the elapsed time + elapsed_time = time.time() - start_time + if elapsed_time > max_runtime: + print("Maximum runtime exceeded. Stopping the simulation.") + break + + # define the DoubleML data object + dgp_dict = datasets[i_rep] + + x_cols = [f'X{i + 1}' for i in np.arange(dgp_dict['x'].shape[1])] + df = pd.DataFrame(np.column_stack((dgp_dict['x'], dgp_dict['y'], dgp_dict['d'])), columns=x_cols + ['y', 'd']) + obj_dml_data = dml.DoubleMLData(df, 'y', 'd') + + for learner_g_idx, (learner_g_name, ml_g) in enumerate(hyperparam_dict["learner_g"]): + for learner_m_idx, (learner_m_name, ml_m) in enumerate(hyperparam_dict["learner_m"]): + # Set machine learning methods for g & m + # calculate the APOs + dml_apos = dml.DoubleMLAPOS( + obj_dml_data=obj_dml_data, + ml_g=ml_g, + ml_m=ml_m, + treatment_levels=hyperparam_dict["treatment_levels"], + ) + + for level_idx, level in enumerate(hyperparam_dict["level"]): + dml_apos.fit(n_jobs_cv=5) + effects = dml_apos.coef + dml_apos.fit(n_jobs_cv=5) + effects = dml_apos.coef + + causal_contrast_model = dml_apos.causal_contrast(reference_levels=0) + estimate = causal_contrast_model.thetas + + for level_idx, level in enumerate(hyperparam_dict["level"]): + # estimate = causal_contrast_model.coef[0] + confint = causal_contrast_model.confint(level=level) + coverage = (confint.iloc[0, 0] < theta) & (theta < confint.iloc[0, 1]) + ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] + + # test sensitivity parameters + # try to run sensitivity analysis + try: + causal_contrast_model.sensitivity_analysis(cf_y=cf_y, cf_d=cf_d, rho=rho, level=level ,null_hypothesis=theta) + cover_lower = theta >= causal_contrast_model.sensitivity_params['ci']['lower'] + cover_upper = theta <= causal_contrast_model.sensitivity_params['ci']['upper'] + rv = causal_contrast_model.sensitivity_params['rv'] + rva = causal_contrast_model.sensitivity_params['rva'] + bias_lower = abs(theta - causal_contrast_model.sensitivity_params['theta']['lower']) + bias_upper = abs(theta - causal_contrast_model.sensitivity_params['theta']['upper']) + success_eval = 1 + except Exception as e: + sensitivity_err_count += 1 + continue + + df_results_detailed = pd.concat( + (df_results_detailed, + pd.DataFrame({ + "Coverage": coverage.astype(int), + "CI Length": confint.iloc[0, 1] - confint.iloc[0, 0], + "Bias": abs(estimate - theta), + "Coverage (Lower)": cover_lower.astype(int), + "Coverage (Upper)": cover_upper.astype(int), + "RV": rv, + "RVa": rva, + "Bias (Lower)": bias_lower, + "Bias (Upper)": bias_upper, + "Learner g": learner_g_name, + "Learner m": learner_m_name, + "level": level, + "repetition": i_rep, + "success_eval": success_eval}, index=[0])), + ignore_index=True) + +# aggregate results only if success_eval == 1 +df_results_detailed = df_results_detailed[df_results_detailed["success_eval"] == 1] + +df_results = df_results_detailed.groupby( + ["Learner g", "Learner m", "level"]).agg( + {"Coverage": "mean", + "CI Length": "mean", + "Bias": "mean", + "Coverage (Lower)": "mean", + "Coverage (Upper)": "mean", + "RV": "mean", + "RVa": "mean", + "Bias (Lower)": "mean", + "Bias (Upper)": "mean", + "repetition": "count"} + ).reset_index() +print(df_results) + +end_time = time.time() +total_runtime = end_time - start_time + +# save results +script_name = "irm_apo_sensitivity.py" +path = "results/irm/irm_apo_sensitivity" + +metadata = pd.DataFrame({ + 'DoubleML Version': [dml.__version__], + 'Script': [script_name], + 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], + 'Total Runtime (seconds)': [total_runtime], + 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Sensitivity Errors': [sensitivity_err_count], +}) +print(metadata) + +df_results.to_csv(f"{path}.csv", index=False) +metadata.to_csv(f"{path}_metadata.csv", index=False) From f375d90ba85ea040f425f176fb820c2cd82baf40 Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Fri, 13 Dec 2024 14:50:57 +0100 Subject: [PATCH 03/67] drop score that was not needed for results --- doc/irm/apo.qmd | 111 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 1 deletion(-) diff --git a/doc/irm/apo.qmd b/doc/irm/apo.qmd index 245d29e..befb83a 100644 --- a/doc/irm/apo.qmd +++ b/doc/irm/apo.qmd @@ -246,4 +246,113 @@ make_pretty(df_ate_95, level, n_rep) level = 0.9 df_ate_9 = df[df['level'] == level][display_columns] make_pretty(df_ate_9, level, n_rep) -``` \ No newline at end of file +``` + + +## Causal Contrast Sensitivity + +The simulations are based on the the ADD-DGP with $10,000$ observations. As the DGP is nonlinear, we will only use corresponding learners. Since the DGP includes an unobserved confounder, we would expect a bias in the ATE estimates, leading to low coverage of the true parameter. + +The confounding is set such that both sensitivity parameters are approximately $cf_y=cf_d=0.1$, such that the robustness value $RV$ should be approximately $10\%$. +Further, the corresponding confidence intervals are one-sided (since the direction of the bias is unkown), such that only one side should approximate the corresponding coverage level (here only the lower coverage is relevant since the bias is positive). Remark that for the coverage level the value of $\rho$ has to be correctly specified, such that the coverage level will be generally (significantly) larger than the nominal level under the conservative choice of $|\rho|=1$. + +```{python} +#| echo: false + +import numpy as np +import pandas as pd +from itables import init_notebook_mode, show, options + +init_notebook_mode(all_interactive=True) + +def highlight_range(s, level=0.95, dist=0.05, props=''): + color_grid = np.where((s >= level-dist) & + (s <= level+dist), props, '') + return color_grid + + +def color_coverage(df, level): + # color coverage column order is important + styled_df = df.apply( + highlight_range, + level=level, + dist=1.0, + props='color:black;background-color:red', + subset=["Coverage", "Coverage (Lower)"]) + styled_df = styled_df.apply( + highlight_range, + level=level, + dist=0.1, + props='color:black;background-color:yellow', + subset=["Coverage", "Coverage (Lower)"]) + styled_df = styled_df.apply( + highlight_range, + level=level, + dist=0.05, + props='color:white;background-color:darkgreen', + subset=["Coverage", "Coverage (Lower)"]) + + # set all coverage values to bold + styled_df = styled_df.set_properties( + **{'font-weight': 'bold'}, + subset=["Coverage", "Coverage (Lower)"]) + return styled_df + + +def make_pretty(df, level, n_rep): + styled_df = df.style.hide(axis="index") + # Format only float columns + float_cols = df.select_dtypes(include=['float']).columns + styled_df = styled_df.format({col: "{:.3f}" for col in float_cols}) + + # color coverage column order is important + styled_df = color_coverage(styled_df, level) + caption = f"Coverage for {level*100}%-Confidence Interval over {n_rep} Repetitions" + + return show(styled_df, caption=caption) +``` + +### ATE + +::: {.callout-note title="Metadata" collapse="true"} + +```{python} +#| echo: false +metadata_file = '../../results/irm/irm_apo_sensitivity_metadata.csv' +metadata_df = pd.read_csv(metadata_file) +print(metadata_df.T.to_string(header=False)) +``` + +::: + +```{python} +#| echo: false + +# set up data and rename columns +df = pd.read_csv("../../results/irm/irm_apo_sensitivity.csv", index_col=None) + +assert df["repetition"].nunique() == 1 +n_rep = df["repetition"].unique()[0] + +display_columns = [ + "Learner g", "Learner m", "Bias", "Bias (Lower)", "Bias (Upper)", "Coverage", "Coverage (Lower)", "Coverage (Upper)", "RV", "RVa"] +``` + +```{python} +#| echo: false +level = 0.95 + +df_ate_95 = df[(df['level'] == level)][display_columns] +df_ate_95.rename(columns={"Learner g": "Learner l"}, inplace=True) +make_pretty(df_ate_95, level, n_rep) +``` + +```{python} +#| echo: false +level = 0.9 + +df_ate_9 = df[(df['level'] == level)][display_columns] +df_ate_9.rename(columns={"Learner g": "Learner l"}, inplace=True) +make_pretty(df_ate_9, level, n_rep) +``` + From a813d34a99963dbedd04787f1cc01a94ae1bedf5 Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Fri, 13 Dec 2024 14:53:12 +0100 Subject: [PATCH 04/67] update workflow for apo sensitivity --- .github/workflows/apo_sim.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/apo_sim.yml b/.github/workflows/apo_sim.yml index 1284d2a..fdf9eaf 100644 --- a/.github/workflows/apo_sim.yml +++ b/.github/workflows/apo_sim.yml @@ -18,6 +18,7 @@ jobs: matrix: script: [ 'scripts/irm/irm_apo_coverage.py', + 'scripts/irm/irm_apo_sensitivity.py', ] steps: From 32e203ec72966910e303511c62a3cdf8f1183a09 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 13 Dec 2024 15:59:06 +0000 Subject: [PATCH 05/67] Update results from script: scripts/irm/irm_apo_coverage.py --- results/irm/irm_apo_coverage_apo.csv | 48 +++++++++---------- results/irm/irm_apo_coverage_apos.csv | 16 +++---- .../irm/irm_apo_coverage_apos_contrast.csv | 16 +++---- results/irm/irm_apo_coverage_metadata.csv | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv index ca4b92b..79c4a89 100644 --- a/results/irm/irm_apo_coverage_apo.csv +++ b/results/irm/irm_apo_coverage_apo.csv @@ -1,25 +1,25 @@ Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.0760855366229958,1000 -LGBM,LGBM,0.0,0.95,0.965,10.316274091547035,2.0760855366229958,1000 -LGBM,LGBM,1.0,0.9,0.915,38.23339285821166,9.180525200256618,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.180525200256618,1000 -LGBM,LGBM,2.0,0.9,0.89,37.49194764946096,9.610674399165429,1000 -LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.610674399165429,1000 -LGBM,Logistic,0.0,0.9,0.904,5.625897101886533,1.3345731610342202,1000 -LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.3345731610342202,1000 -LGBM,Logistic,1.0,0.9,0.924,7.423300143143786,1.697057819214677,1000 -LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.697057819214677,1000 -LGBM,Logistic,2.0,0.9,0.917,7.321275660150267,1.660826093332619,1000 -LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.660826093332619,1000 -Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.30423568882223,1000 -Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.30423568882223,1000 -Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.1264975614784354,1000 -Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.1264975614784354,1000 -Linear,LGBM,2.0,0.9,0.93,7.513644049429104,1.6338972741785736,1000 -Linear,LGBM,2.0,0.95,0.965,8.953059097926168,1.6338972741785736,1000 -Linear,Logistic,0.0,0.9,0.902,5.335670092717667,1.2832290098794459,1000 -Linear,Logistic,0.0,0.95,0.954,6.357843058957276,1.2832290098794459,1000 -Linear,Logistic,1.0,0.9,0.905,5.417512107920403,1.2812664745065607,1000 -Linear,Logistic,1.0,0.95,0.956,6.455363835025866,1.2812664745065607,1000 -Linear,Logistic,2.0,0.9,0.908,5.366403391397197,1.2775297528572058,1000 -Linear,Logistic,2.0,0.95,0.958,6.3944640430685675,1.2775297528572058,1000 +LGBM,LGBM,0.0,0.9,0.91,8.657690136121921,2.0771552393153954,1000 +LGBM,LGBM,0.0,0.95,0.966,10.316274091547035,2.0771552393153954,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.211837492627364,1000 +LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.211837492627364,1000 +LGBM,LGBM,2.0,0.9,0.894,37.49194764946096,9.572858395839285,1000 +LGBM,LGBM,2.0,0.95,0.953,44.67441108385784,9.572858395839285,1000 +LGBM,Logistic,0.0,0.9,0.904,5.625897101886533,1.3417933174607457,1000 +LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.3417933174607457,1000 +LGBM,Logistic,1.0,0.9,0.923,7.423300143143786,1.6937231294439623,1000 +LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6937231294439623,1000 +LGBM,Logistic,2.0,0.9,0.92,7.321275660150267,1.6623475358295177,1000 +LGBM,Logistic,2.0,0.95,0.965,8.723838024042964,1.6623475358295177,1000 +Linear,LGBM,0.0,0.9,0.9,5.498257423071024,1.3124155546719014,1000 +Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.3124155546719014,1000 +Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.128626055374095,1000 +Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.128626055374095,1000 +Linear,LGBM,2.0,0.9,0.934,7.513644049429104,1.63753078556302,1000 +Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.63753078556302,1000 +Linear,Logistic,0.0,0.9,0.904,5.335670092717667,1.2914800726015643,1000 +Linear,Logistic,0.0,0.95,0.955,6.357843058957276,1.2914800726015643,1000 +Linear,Logistic,1.0,0.9,0.908,5.417512107920403,1.2803038033844159,1000 +Linear,Logistic,1.0,0.95,0.956,6.455363835025866,1.2803038033844159,1000 +Linear,Logistic,2.0,0.9,0.906,5.366403391397197,1.287341166689217,1000 +Linear,Logistic,2.0,0.95,0.957,6.3944640430685675,1.287341166689217,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv index 48a0fba..a214628 100644 --- a/results/irm/irm_apo_coverage_apos.csv +++ b/results/irm/irm_apo_coverage_apos.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.906,28.21021843292038,7.120657478889542,0.924,36.1943517986729,1000 -LGBM,LGBM,0.95,0.9586666666666667,33.61454856442563,7.120657478889542,0.974,40.868020474452145,1000 -LGBM,Logistic,0.9,0.918,6.789316986971467,1.5705977914191496,0.925,8.394070597867524,1000 -LGBM,Logistic,0.95,0.962,8.089970168806186,1.5705977914191496,0.961,9.592284619044477,1000 -Linear,LGBM,0.9,0.9276666666666666,7.903418354805325,1.7245563336966274,0.94,9.850445774370156,1000 -Linear,LGBM,0.95,0.9666666666666667,9.41750382912841,1.7245563336966274,0.975,11.234392064837424,1000 -Linear,Logistic,0.9,0.903,5.372532806955153,1.278875142245648,0.906,5.7964894175017925,1000 -Linear,Logistic,0.95,0.9556666666666667,6.4017676921854445,1.278875142245648,0.953,6.8182385960659,1000 +LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.116365925800573,0.925,36.1943517986729,1000 +LGBM,LGBM,0.95,0.958,33.61454856442563,7.116365925800573,0.973,40.868020474452145,1000 +LGBM,Logistic,0.9,0.9133333333333333,6.789316986971467,1.5740169263514348,0.922,8.394070597867524,1000 +LGBM,Logistic,0.95,0.962,8.089970168806186,1.5740169263514348,0.96,9.592284619044477,1000 +Linear,LGBM,0.9,0.9266666666666666,7.903418354805325,1.7298579127669884,0.936,9.850445774370156,1000 +Linear,LGBM,0.95,0.968,9.41750382912841,1.7298579127669884,0.974,11.234392064837424,1000 +Linear,Logistic,0.9,0.9053333333333333,5.372532806955153,1.2839893621990757,0.901,5.7964894175017925,1000 +Linear,Logistic,0.95,0.9556666666666667,6.4017676921854445,1.2839893621990757,0.95,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv index 36dea44..9c8c331 100644 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.788503277093351,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.9485,45.13128131893863,9.788503277093351,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.9275,5.725128733165455,1.267381146072165,0.927,6.774304315459576,1000 -LGBM,Logistic,0.95,0.963,6.821911652197591,1.267381146072165,0.962,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.5064356327113417,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.989,8.85452711998085,1.5064356327113417,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.872,1.1425883251377777,0.29482653859949187,0.871,1.3505222299078565,1000 -Linear,Logistic,0.95,0.936,1.3614779635902856,0.29482653859949187,0.92,1.5496710496635275,1000 +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.784120711881009,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.949,45.13128131893863,9.784120711881009,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.928,5.725128733165455,1.268560055277062,0.926,6.774304315459576,1000 +LGBM,Logistic,0.95,0.962,6.821911652197591,1.268560055277062,0.961,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.5069627173157734,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.9885,8.85452711998085,1.5069627173157734,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.8715,1.1425883251377777,0.299993836487866,0.865,1.3505222299078565,1000 +Linear,Logistic,0.95,0.9295,1.3614779635902856,0.299993836487866,0.914,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv index d2aff99..ed46e11 100644 --- a/results/irm/irm_apo_coverage_metadata.csv +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.0,irm_apo_coverage.py,2024-09-09 10:19:31,10550.167822599411,3.12.5 +0.10.dev0,irm_apo_coverage.py,2024-12-13 15:59:02,11213.07107424736,3.12.8 From 5823ca2742f9644684ebf85ffdfc5cfcf0c6508e Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 16:10:03 +0000 Subject: [PATCH 06/67] Update results from script: scripts/irm/irm_apo_coverage.py --- results/irm/irm_apo_coverage_apo.csv | 48 +++++++++---------- results/irm/irm_apo_coverage_apos.csv | 16 +++---- .../irm/irm_apo_coverage_apos_contrast.csv | 16 +++---- results/irm/irm_apo_coverage_metadata.csv | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv index 79c4a89..b7044df 100644 --- a/results/irm/irm_apo_coverage_apo.csv +++ b/results/irm/irm_apo_coverage_apo.csv @@ -1,25 +1,25 @@ Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.91,8.657690136121921,2.0771552393153954,1000 -LGBM,LGBM,0.0,0.95,0.966,10.316274091547035,2.0771552393153954,1000 -LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.211837492627364,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.211837492627364,1000 -LGBM,LGBM,2.0,0.9,0.894,37.49194764946096,9.572858395839285,1000 -LGBM,LGBM,2.0,0.95,0.953,44.67441108385784,9.572858395839285,1000 -LGBM,Logistic,0.0,0.9,0.904,5.625897101886533,1.3417933174607457,1000 -LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.3417933174607457,1000 -LGBM,Logistic,1.0,0.9,0.923,7.423300143143786,1.6937231294439623,1000 -LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6937231294439623,1000 -LGBM,Logistic,2.0,0.9,0.92,7.321275660150267,1.6623475358295177,1000 -LGBM,Logistic,2.0,0.95,0.965,8.723838024042964,1.6623475358295177,1000 -Linear,LGBM,0.0,0.9,0.9,5.498257423071024,1.3124155546719014,1000 -Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.3124155546719014,1000 -Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.128626055374095,1000 -Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.128626055374095,1000 -Linear,LGBM,2.0,0.9,0.934,7.513644049429104,1.63753078556302,1000 -Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.63753078556302,1000 -Linear,Logistic,0.0,0.9,0.904,5.335670092717667,1.2914800726015643,1000 -Linear,Logistic,0.0,0.95,0.955,6.357843058957276,1.2914800726015643,1000 -Linear,Logistic,1.0,0.9,0.908,5.417512107920403,1.2803038033844159,1000 -Linear,Logistic,1.0,0.95,0.956,6.455363835025866,1.2803038033844159,1000 -Linear,Logistic,2.0,0.9,0.906,5.366403391397197,1.287341166689217,1000 -Linear,Logistic,2.0,0.95,0.957,6.3944640430685675,1.287341166689217,1000 +LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.0765932719699256,1000 +LGBM,LGBM,0.0,0.95,0.966,10.316274091547035,2.0765932719699256,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.20664647418475,1000 +LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.20664647418475,1000 +LGBM,LGBM,2.0,0.9,0.892,37.49194764946096,9.581059230174954,1000 +LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.581059230174954,1000 +LGBM,Logistic,0.0,0.9,0.905,5.625897101886533,1.3392657230189877,1000 +LGBM,Logistic,0.0,0.95,0.954,6.7036698705295725,1.3392657230189877,1000 +LGBM,Logistic,1.0,0.9,0.924,7.423300143143786,1.6941251940538382,1000 +LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6941251940538382,1000 +LGBM,Logistic,2.0,0.9,0.92,7.321275660150267,1.6613918913323027,1000 +LGBM,Logistic,2.0,0.95,0.969,8.723838024042964,1.6613918913323027,1000 +Linear,LGBM,0.0,0.9,0.9,5.498257423071024,1.3101118376643717,1000 +Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.3101118376643717,1000 +Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.1280748702898498,1000 +Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.1280748702898498,1000 +Linear,LGBM,2.0,0.9,0.933,7.513644049429104,1.6361363019410766,1000 +Linear,LGBM,2.0,0.95,0.967,8.953059097926168,1.6361363019410766,1000 +Linear,Logistic,0.0,0.9,0.903,5.335670092717667,1.2889815574567558,1000 +Linear,Logistic,0.0,0.95,0.953,6.357843058957276,1.2889815574567558,1000 +Linear,Logistic,1.0,0.9,0.907,5.417512107920403,1.2801718914567304,1000 +Linear,Logistic,1.0,0.95,0.954,6.455363835025866,1.2801718914567304,1000 +Linear,Logistic,2.0,0.9,0.906,5.366403391397197,1.2844950358842981,1000 +Linear,Logistic,2.0,0.95,0.957,6.3944640430685675,1.2844950358842981,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv index a214628..1250e31 100644 --- a/results/irm/irm_apo_coverage_apos.csv +++ b/results/irm/irm_apo_coverage_apos.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.116365925800573,0.925,36.1943517986729,1000 -LGBM,LGBM,0.95,0.958,33.61454856442563,7.116365925800573,0.973,40.868020474452145,1000 -LGBM,Logistic,0.9,0.9133333333333333,6.789316986971467,1.5740169263514348,0.922,8.394070597867524,1000 -LGBM,Logistic,0.95,0.962,8.089970168806186,1.5740169263514348,0.96,9.592284619044477,1000 -Linear,LGBM,0.9,0.9266666666666666,7.903418354805325,1.7298579127669884,0.936,9.850445774370156,1000 -Linear,LGBM,0.95,0.968,9.41750382912841,1.7298579127669884,0.974,11.234392064837424,1000 -Linear,Logistic,0.9,0.9053333333333333,5.372532806955153,1.2839893621990757,0.901,5.7964894175017925,1000 -Linear,Logistic,0.95,0.9556666666666667,6.4017676921854445,1.2839893621990757,0.95,6.8182385960659,1000 +LGBM,LGBM,0.9,0.9066666666666666,28.21021843292038,7.117624906701142,0.925,36.1943517986729,1000 +LGBM,LGBM,0.95,0.958,33.61454856442563,7.117624906701142,0.973,40.868020474452145,1000 +LGBM,Logistic,0.9,0.9136666666666666,6.789316986971467,1.572818265732698,0.922,8.394070597867524,1000 +LGBM,Logistic,0.95,0.963,8.089970168806186,1.572818265732698,0.96,9.592284619044477,1000 +Linear,LGBM,0.9,0.927,7.903418354805325,1.7283333079765255,0.937,9.850445774370156,1000 +Linear,LGBM,0.95,0.9676666666666667,9.41750382912841,1.7283333079765255,0.974,11.234392064837424,1000 +Linear,Logistic,0.9,0.9043333333333333,5.372532806955153,1.282319600330098,0.902,5.7964894175017925,1000 +Linear,Logistic,0.95,0.957,6.4017676921854445,1.282319600330098,0.952,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv index 9c8c331..df8717c 100644 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.784120711881009,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.949,45.13128131893863,9.784120711881009,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.928,5.725128733165455,1.268560055277062,0.926,6.774304315459576,1000 -LGBM,Logistic,0.95,0.962,6.821911652197591,1.268560055277062,0.961,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.5069627173157734,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.9885,8.85452711998085,1.5069627173157734,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.8715,1.1425883251377777,0.299993836487866,0.865,1.3505222299078565,1000 -Linear,Logistic,0.95,0.9295,1.3614779635902856,0.299993836487866,0.914,1.5496710496635275,1000 +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.785831227953715,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.9485,45.13128131893863,9.785831227953715,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.927,5.725128733165455,1.268042718065415,0.926,6.774304315459576,1000 +LGBM,Logistic,0.95,0.962,6.821911652197591,1.268042718065415,0.961,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.5067276240415115,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.9885,8.85452711998085,1.5067276240415115,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.8735,1.1425883251377777,0.2978290007254657,0.87,1.3505222299078565,1000 +Linear,Logistic,0.95,0.933,1.3614779635902856,0.2978290007254657,0.916,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv index ed46e11..1a3395e 100644 --- a/results/irm/irm_apo_coverage_metadata.csv +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_apo_coverage.py,2024-12-13 15:59:02,11213.07107424736,3.12.8 +0.10.dev0,irm_apo_coverage.py,2024-12-18 16:09:59,11286.377284288406,3.12.8 From a5b1727da4db1beeb577d9eb7fc48a9dc76fe0fd Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 18:32:02 +0000 Subject: [PATCH 07/67] Update results from script: scripts/irm/irm_apo_sensitivity.py --- results/irm/irm_apo_sensitivity.csv | 9 +++++++++ results/irm/irm_apo_sensitivity_metadata.csv | 2 ++ 2 files changed, 11 insertions(+) create mode 100644 results/irm/irm_apo_sensitivity.csv create mode 100644 results/irm/irm_apo_sensitivity_metadata.csv diff --git a/results/irm/irm_apo_sensitivity.csv b/results/irm/irm_apo_sensitivity.csv new file mode 100644 index 0000000..65a7ec9 --- /dev/null +++ b/results/irm/irm_apo_sensitivity.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition +LGBM,LGBM,0.9,0.08571428571428572,0.2668542738595025,0.18297137425202115,0.14285714285714285,1.0,0.6114310427610333,0.190586622442589,0.16060561738439572,0.2053371311196466,35 +LGBM,LGBM,0.95,0.2571428571428571,0.31797647967896914,0.18297137425202115,0.4,1.0,0.6114310427610333,0.1291525667553449,0.16060561738439572,0.2053371311196466,35 +LGBM,Logistic Regr.,0.9,0.2689655172413793,0.2574480013692081,0.1509161799548195,0.18620689655172415,1.0,0.5421706160096337,0.19323512174098637,0.12952194639765482,0.17231041351198423,145 +LGBM,Logistic Regr.,0.95,0.5448275862068965,0.3067682146955884,0.1509161799548195,0.5241379310344828,1.0,0.5421706160096337,0.10745562617986258,0.12952194639765482,0.17231041351198423,145 +Linear Reg.,LGBM,0.9,0.0975609756097561,0.2668393325191035,0.17524217372808007,0.07317073170731707,1.0,0.6166184854541432,0.21038949010727379,0.15590555899808603,0.19457878845807408,41 +Linear Reg.,LGBM,0.95,0.2926829268292683,0.31795867597377425,0.17524217372808007,0.2682926829268293,1.0,0.6166184854541432,0.13803374916742117,0.15590555899808603,0.19457878845807408,41 +Linear Reg.,Logistic Regr.,0.9,0.8811188811188811,0.25897564216705166,0.08760514512781448,0.8041958041958042,1.0,0.37872148149083257,0.03858402699843757,0.06806645558318873,0.10850473740047888,143 +Linear Reg.,Logistic Regr.,0.95,0.986013986013986,0.3085885109797244,0.08760514512781448,0.9790209790209791,1.0,0.37872148149083257,0.007354698850087891,0.06806645558318873,0.10850473740047888,143 diff --git a/results/irm/irm_apo_sensitivity_metadata.csv b/results/irm/irm_apo_sensitivity_metadata.csv new file mode 100644 index 0000000..5f9ff27 --- /dev/null +++ b/results/irm/irm_apo_sensitivity_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Sensitivity Errors +0.10.dev0,irm_apo_sensitivity.py,2024-12-18 18:32:01,19817.343207597733,3.12.8,736 From bd686ed930862c408c23f143c4db3f7f825d06f0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 3 Feb 2025 20:00:27 +0000 Subject: [PATCH 08/67] Update results from script: scripts/irm/irm_gate_coverage.py --- results/irm/irm_gate_coverage.csv | 2 +- results/irm/irm_gate_coverage_metadata.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/results/irm/irm_gate_coverage.csv b/results/irm/irm_gate_coverage.csv index f5dde96..4208c4b 100644 --- a/results/irm/irm_gate_coverage.csv +++ b/results/irm/irm_gate_coverage.csv @@ -1,7 +1,7 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition LGBM,LGBM,0.9,0.941,2.1611526373873855,0.48511146422993706,1.0,5.063646757352436,1000 LGBM,LGBM,0.95,0.9766666666666667,2.5751722007164304,0.48511146422993706,1.0,5.085406768554818,1000 -LGBM,Logistic Regression,0.9,0.916,0.3904280687581478,0.08854865701618204,0.997,0.9186129412168421,1000 +LGBM,Logistic Regression,0.9,0.916,0.3904280687581478,0.08854865701618204,0.997,0.9186129412168422,1000 LGBM,Logistic Regression,0.95,0.9606666666666667,0.4652237383199531,0.08854865701618204,0.998,0.9197763724365766,1000 Lasso,LGBM,0.9,0.9043333333333333,2.047596498760145,0.49693265710872336,1.0,4.807194790243196,1000 Lasso,LGBM,0.95,0.959,2.43986171576749,0.49693265710872336,1.0,4.819825928994352,1000 diff --git a/results/irm/irm_gate_coverage_metadata.csv b/results/irm/irm_gate_coverage_metadata.csv index b7034be..5b3a787 100644 --- a/results/irm/irm_gate_coverage_metadata.csv +++ b/results/irm/irm_gate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_gate_coverage.py,2025-01-08 13:08:04,3153.3980271816254,3.12.8 +0.10.dev0,irm_gate_coverage.py,2025-02-03 20:00:25,2082.8255705833435,3.12.8 From fe48bcbc25873d57abea9cb2265d5f62e4e3ce84 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 3 Feb 2025 20:25:21 +0000 Subject: [PATCH 09/67] Update results from script: scripts/irm/irm_ate_coverage.py --- results/irm/irm_ate_coverage.csv | 16 ++++++++-------- results/irm/irm_ate_coverage_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv index 0de042e..d0229af 100644 --- a/results/irm/irm_ate_coverage.csv +++ b/results/irm/irm_ate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536219,1000 -Lasso,Logistic Regression,0.95,0.935,0.5573839547492132,0.12336884150536219,1000 -Lasso,Random Forest,0.9,0.906,0.6047145364621751,0.1478860899164366,1000 -Lasso,Random Forest,0.95,0.959,0.7205618135094181,0.1478860899164366,1000 -Random Forest,Logistic Regression,0.9,0.803,0.5176227656141883,0.15019297474976653,1000 -Random Forest,Logistic Regression,0.95,0.883,0.6167855677602845,0.15019297474976653,1000 -Random Forest,Random Forest,0.9,0.899,0.6330777189487165,0.15360554047241584,1000 -Random Forest,Random Forest,0.95,0.959,0.7543586299857805,0.15360554047241584,1000 +Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536215,1000 +Lasso,Logistic Regression,0.95,0.935,0.5573839547492131,0.12336884150536215,1000 +Lasso,Random Forest,0.9,0.909,0.5986699126576687,0.1437194448121057,1000 +Lasso,Random Forest,0.95,0.958,0.7133591999985225,0.1437194448121057,1000 +Random Forest,Logistic Regression,0.9,0.8,0.5165274317458133,0.14990700956657027,1000 +Random Forest,Logistic Regression,0.95,0.883,0.6154803969548793,0.14990700956657027,1000 +Random Forest,Random Forest,0.9,0.891,0.6264098177762909,0.1516133190457161,1000 +Random Forest,Random Forest,0.95,0.952,0.7464133356834249,0.1516133190457161,1000 diff --git a/results/irm/irm_ate_coverage_metadata.csv b/results/irm/irm_ate_coverage_metadata.csv index a6fbf57..00e4d29 100644 --- a/results/irm/irm_ate_coverage_metadata.csv +++ b/results/irm/irm_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_coverage.py,2025-01-08 13:14:33,3543.555382013321,3.12.8 +0.10.dev0,irm_ate_coverage.py,2025-02-03 20:25:20,3578.0215225219727,3.12.8 From dd1ee9fc9ac72c5672240688aee1d0906bbb54fb Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 3 Feb 2025 20:25:21 +0000 Subject: [PATCH 10/67] Update results from script: scripts/irm/irm_atte_coverage.py --- results/irm/irm_atte_coverage.csv | 16 ++++++++-------- results/irm/irm_atte_coverage_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv index 3f486c5..2593e74 100644 --- a/results/irm/irm_atte_coverage.csv +++ b/results/irm/irm_atte_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.887,0.5331759172799809,0.13448723008182129,1000 -Lasso,Logistic Regression,0.95,0.942,0.6353182910443254,0.13448723008182129,1000 -Lasso,Random Forest,0.9,0.895,0.7320685495582887,0.18096827990713532,1000 -Lasso,Random Forest,0.95,0.948,0.8723134799586961,0.18096827990713532,1000 -Random Forest,Logistic Regression,0.9,0.871,0.5508997607447915,0.1495507044107987,1000 -Random Forest,Logistic Regression,0.95,0.928,0.6564375531412434,0.1495507044107987,1000 -Random Forest,Random Forest,0.9,0.901,0.7500491227406757,0.18282767194851973,1000 -Random Forest,Random Forest,0.95,0.948,0.8937386543823805,0.18282767194851973,1000 +Lasso,Logistic Regression,0.9,0.888,0.5331759172799809,0.1346025659304845,1000 +Lasso,Logistic Regression,0.95,0.942,0.6353182910443254,0.1346025659304845,1000 +Lasso,Random Forest,0.9,0.892,0.736544703634449,0.18600544721077814,1000 +Lasso,Random Forest,0.95,0.948,0.8776471465140515,0.18600544721077814,1000 +Random Forest,Logistic Regression,0.9,0.87,0.549683356573939,0.14976166337094526,1000 +Random Forest,Logistic Regression,0.95,0.914,0.6549881181724104,0.14976166337094526,1000 +Random Forest,Random Forest,0.9,0.902,0.7467330863654522,0.18215062754687864,1000 +Random Forest,Random Forest,0.95,0.947,0.889787353330196,0.18215062754687864,1000 diff --git a/results/irm/irm_atte_coverage_metadata.csv b/results/irm/irm_atte_coverage_metadata.csv index 18af13a..2cc142c 100644 --- a/results/irm/irm_atte_coverage_metadata.csv +++ b/results/irm/irm_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_coverage.py,2025-01-08 13:14:10,3538.650551557541,3.12.8 +0.10.dev0,irm_atte_coverage.py,2025-02-03 20:25:20,3578.3450458049774,3.12.8 From 2bb97ffd0175b05d34b8c64c302148f2ee3b26c9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 3 Feb 2025 20:56:48 +0000 Subject: [PATCH 11/67] Update results from script: scripts/irm/irm_cate_coverage.py --- results/irm/irm_cate_coverage.csv | 4 ++-- results/irm/irm_cate_coverage_metadata.csv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/results/irm/irm_cate_coverage.csv b/results/irm/irm_cate_coverage.csv index 42b5454..788025e 100644 --- a/results/irm/irm_cate_coverage.csv +++ b/results/irm/irm_cate_coverage.csv @@ -5,5 +5,5 @@ LGBM,Logistic Regression,0.9,0.8889914999999999,0.2354452920928757,0.05850699314 LGBM,Logistic Regression,0.95,0.942037,0.28055036951027373,0.05850699314525499,0.996,0.5990789624833718,1000 Lasso,LGBM,0.9,0.896289,0.6428742805642967,0.15829333167540963,1.0,1.6320898187101593,1000 Lasso,LGBM,0.95,0.9491539999999999,0.7660319531461224,0.15829333167540963,1.0,1.6337799284392311,1000 -Lasso,Logistic Regression,0.9,0.8892920000000001,0.2473839966772624,0.061668240816652016,0.998,0.629839636678611,1000 -Lasso,Logistic Regression,0.95,0.9413365,0.29477621345410787,0.061668240816652016,0.997,0.6288417962868702,1000 +Lasso,Logistic Regression,0.9,0.8892920000000001,0.24738399667726244,0.061668240816652016,0.998,0.629839636678611,1000 +Lasso,Logistic Regression,0.95,0.9413365,0.29477621345410787,0.061668240816652016,0.997,0.6288417962868703,1000 diff --git a/results/irm/irm_cate_coverage_metadata.csv b/results/irm/irm_cate_coverage_metadata.csv index 24d54ab..1826629 100644 --- a/results/irm/irm_cate_coverage_metadata.csv +++ b/results/irm/irm_cate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_cate_coverage.py,2025-01-08 14:19:05,7429.032528162003,3.12.8 +0.10.dev0,irm_cate_coverage.py,2025-02-03 20:56:46,5460.1229820251465,3.12.8 From 8c11e3b80a207227ec9282f7ed9c451211a8eaf7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 3 Feb 2025 21:02:16 +0000 Subject: [PATCH 12/67] Update results from script: scripts/irm/irm_apo_coverage.py --- results/irm/irm_apo_coverage_apo.csv | 25 +++++++++++++++++++ results/irm/irm_apo_coverage_apos.csv | 9 +++++++ .../irm/irm_apo_coverage_apos_contrast.csv | 9 +++++++ results/irm/irm_apo_coverage_metadata.csv | 2 ++ 4 files changed, 45 insertions(+) create mode 100644 results/irm/irm_apo_coverage_apo.csv create mode 100644 results/irm/irm_apo_coverage_apos.csv create mode 100644 results/irm/irm_apo_coverage_apos_contrast.csv create mode 100644 results/irm/irm_apo_coverage_metadata.csv diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv new file mode 100644 index 0000000..013fc38 --- /dev/null +++ b/results/irm/irm_apo_coverage_apo.csv @@ -0,0 +1,25 @@ +Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition +LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.075985425408154,1000 +LGBM,LGBM,0.0,0.95,0.964,10.316274091547035,2.075985425408154,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.189360948372517,1000 +LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.189360948372517,1000 +LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.600729846680576,1000 +LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.600729846680576,1000 +LGBM,Logistic,0.0,0.9,0.902,5.625897101886533,1.3356673109449309,1000 +LGBM,Logistic,0.0,0.95,0.956,6.7036698705295725,1.3356673109449309,1000 +LGBM,Logistic,1.0,0.9,0.92,7.423300143143785,1.6958813914305944,1000 +LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6958813914305944,1000 +LGBM,Logistic,2.0,0.9,0.918,7.321275660150268,1.660500602699947,1000 +LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.660500602699947,1000 +Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.305981267915353,1000 +Linear,LGBM,0.0,0.95,0.948,6.551577812380716,1.305981267915353,1000 +Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.126828037202422,1000 +Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.126828037202422,1000 +Linear,LGBM,2.0,0.9,0.932,7.513644049429104,1.6342111609274832,1000 +Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.6342111609274832,1000 +Linear,Logistic,0.0,0.9,0.902,5.335670092717667,1.2845963390782078,1000 +Linear,Logistic,0.0,0.95,0.953,6.357843058957276,1.2845963390782078,1000 +Linear,Logistic,1.0,0.9,0.904,5.417512107920403,1.280503259417172,1000 +Linear,Logistic,1.0,0.95,0.954,6.455363835025866,1.280503259417172,1000 +Linear,Logistic,2.0,0.9,0.907,5.366403391397197,1.2793569219717145,1000 +Linear,Logistic,2.0,0.95,0.96,6.3944640430685675,1.2793569219717145,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv new file mode 100644 index 0000000..81371cc --- /dev/null +++ b/results/irm/irm_apo_coverage_apos.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.119510083508661,0.924,36.1943517986729,1000 +LGBM,LGBM,0.95,0.9583333333333334,33.61454856442563,7.119510083508661,0.974,40.868020474452145,1000 +LGBM,Logistic,0.9,0.9173333333333333,6.789316986971467,1.570973449234106,0.923,8.394070597867524,1000 +LGBM,Logistic,0.95,0.963,8.089970168806186,1.570973449234106,0.96,9.592284619044477,1000 +Linear,LGBM,0.9,0.926,7.903418354805325,1.725523010236133,0.939,9.850445774370156,1000 +Linear,LGBM,0.95,0.967,9.41750382912841,1.725523010236133,0.974,11.234392064837424,1000 +Linear,Logistic,0.9,0.9036666666666666,5.372532806955153,1.2795639356934292,0.905,5.7964894175017925,1000 +Linear,Logistic,0.95,0.9573333333333334,6.4017676921854445,1.2795639356934292,0.953,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv new file mode 100644 index 0000000..bffe6b6 --- /dev/null +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.787707441089529,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.9485,45.13128131893863,9.787707441089529,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.9275,5.725128733165455,1.267568231095916,0.927,6.774304315459575,1000 +LGBM,Logistic,0.95,0.9625,6.821911652197591,1.267568231095916,0.962,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.5065164682583199,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.989,8.85452711998085,1.5065164682583199,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.873,1.1425883251377777,0.29572275431639516,0.872,1.3505222299078565,1000 +Linear,Logistic,0.95,0.936,1.3614779635902856,0.29572275431639516,0.919,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv new file mode 100644 index 0000000..214d397 --- /dev/null +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version +0.10.dev0,irm_apo_coverage.py,2025-02-03 21:02:12,5798.276986837387,3.12.8 From d6aa56ff260da7ec9391a14e2264c502a5fb2efe Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 3 Feb 2025 21:20:42 +0000 Subject: [PATCH 13/67] Update results from script: scripts/irm/irm_ate_sensitivity.py --- results/irm/irm_ate_sensitivity_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv index c038f24..6dc906e 100644 --- a/results/irm/irm_ate_sensitivity_metadata.csv +++ b/results/irm/irm_ate_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_sensitivity.py,2025-01-08 14:51:20,9351.407655954361,3.12.8 +0.10.dev0,irm_ate_sensitivity.py,2025-02-03 21:20:41,6893.791319608688,3.12.8 From 9a2ed398ac06168c73d6a0add10e5ba7543902b4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 3 Feb 2025 21:20:32 +0000 Subject: [PATCH 14/67] Update results from script: scripts/irm/irm_atte_sensitivity.py --- results/irm/irm_atte_sensitivity.csv | 2 +- results/irm/irm_atte_sensitivity_metadata.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/results/irm/irm_atte_sensitivity.csv b/results/irm/irm_atte_sensitivity.csv index 7483e17..901b16f 100644 --- a/results/irm/irm_atte_sensitivity.csv +++ b/results/irm/irm_atte_sensitivity.csv @@ -1,6 +1,6 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition LGBM,LGBM,0.9,0.702,0.348892741328716,0.13535473129404849,0.95,1.0,0.10509293446782587,0.024288450266824614,0.0649958270073569,0.25876253126104387,500 -LGBM,LGBM,0.95,0.826,0.41573134306126747,0.13535473129404849,0.982,1.0,0.10509293446782587,0.012452066983782133,0.0649958270073569,0.25876253126104387,500 +LGBM,LGBM,0.95,0.826,0.41573134306126747,0.13535473129404849,0.982,1.0,0.10509293446782587,0.012452066983782135,0.0649958270073569,0.25876253126104387,500 LGBM,Logistic Regr.,0.9,0.714,0.34666910502599596,0.13078975736827733,0.964,1.0,0.09817018528214369,0.022246998237972562,0.06545442330246619,0.2589976808508975,500 LGBM,Logistic Regr.,0.95,0.834,0.41308171698108886,0.13078975736827733,0.984,1.0,0.09817018528214369,0.010949342084431752,0.06545442330246619,0.2589976808508975,500 Linear Reg.,LGBM,0.9,0.754,0.3496967006881292,0.12455057551341779,0.962,1.0,0.09867724125956986,0.02021759355041513,0.06504946816195573,0.24393419014571052,500 diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv index 4353523..449b919 100644 --- a/results/irm/irm_atte_sensitivity_metadata.csv +++ b/results/irm/irm_atte_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_sensitivity.py,2025-01-08 15:14:53,10782.067339658737,3.12.8 +0.10.dev0,irm_atte_sensitivity.py,2025-02-03 21:20:31,6886.662088632584,3.12.8 From 7bf58076b791fd8e63946eb3316f6aaae945cb6c Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 00:55:59 +0000 Subject: [PATCH 15/67] Update results from script: scripts/irm/irm_apo_sensitivity.py --- results/irm/irm_apo_sensitivity.csv | 16 ++++++++-------- results/irm/irm_apo_sensitivity_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_apo_sensitivity.csv b/results/irm/irm_apo_sensitivity.csv index 65a7ec9..606bee8 100644 --- a/results/irm/irm_apo_sensitivity.csv +++ b/results/irm/irm_apo_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition -LGBM,LGBM,0.9,0.08571428571428572,0.2668542738595025,0.18297137425202115,0.14285714285714285,1.0,0.6114310427610333,0.190586622442589,0.16060561738439572,0.2053371311196466,35 -LGBM,LGBM,0.95,0.2571428571428571,0.31797647967896914,0.18297137425202115,0.4,1.0,0.6114310427610333,0.1291525667553449,0.16060561738439572,0.2053371311196466,35 -LGBM,Logistic Regr.,0.9,0.2689655172413793,0.2574480013692081,0.1509161799548195,0.18620689655172415,1.0,0.5421706160096337,0.19323512174098637,0.12952194639765482,0.17231041351198423,145 -LGBM,Logistic Regr.,0.95,0.5448275862068965,0.3067682146955884,0.1509161799548195,0.5241379310344828,1.0,0.5421706160096337,0.10745562617986258,0.12952194639765482,0.17231041351198423,145 -Linear Reg.,LGBM,0.9,0.0975609756097561,0.2668393325191035,0.17524217372808007,0.07317073170731707,1.0,0.6166184854541432,0.21038949010727379,0.15590555899808603,0.19457878845807408,41 -Linear Reg.,LGBM,0.95,0.2926829268292683,0.31795867597377425,0.17524217372808007,0.2682926829268293,1.0,0.6166184854541432,0.13803374916742117,0.15590555899808603,0.19457878845807408,41 -Linear Reg.,Logistic Regr.,0.9,0.8811188811188811,0.25897564216705166,0.08760514512781448,0.8041958041958042,1.0,0.37872148149083257,0.03858402699843757,0.06806645558318873,0.10850473740047888,143 -Linear Reg.,Logistic Regr.,0.95,0.986013986013986,0.3085885109797244,0.08760514512781448,0.9790209790209791,1.0,0.37872148149083257,0.007354698850087891,0.06806645558318873,0.10850473740047888,143 +LGBM,LGBM,0.9,0.09836065573770492,0.26980333552727825,0.18205252584345485,1.0,1.0,0.09239844030018919,0.04045631137696363,0.035066111350142995,0.3810774239324017,244 +LGBM,LGBM,0.95,0.3073770491803279,0.32149050339653273,0.18205252584345485,1.0,1.0,0.09239844030018919,0.025884873713769935,0.035066111350142995,0.3810774239324017,244 +LGBM,Logistic Regr.,0.9,0.3073770491803279,0.2578060603442706,0.14817801851576512,1.0,1.0,0.07187480682741719,0.024545078956797563,0.06295056021181017,0.35846457343559074,244 +LGBM,Logistic Regr.,0.95,0.5614754098360656,0.30719486827981307,0.14817801851576512,1.0,1.0,0.07187480682741719,0.012961043152938044,0.06295056021181017,0.35846457343559074,244 +Linear Reg.,LGBM,0.9,0.10655737704918032,0.26955592225259073,0.18010332737150503,1.0,1.0,0.0941627548347164,0.04084948000316684,0.03472113416565001,0.37380556783576596,244 +Linear Reg.,LGBM,0.95,0.29918032786885246,0.3211956922961776,0.18010332737150503,1.0,1.0,0.0941627548347164,0.026040282694101653,0.03472113416565001,0.37380556783576596,244 +Linear Reg.,Logistic Regr.,0.9,0.8852459016393442,0.25925313646720843,0.08765249652827344,1.0,1.0,0.044052229523588664,0.0042325141887964,0.11824259236043642,0.2930872570585111,244 +Linear Reg.,Logistic Regr.,0.95,0.9836065573770492,0.30891916583272205,0.08765249652827344,1.0,1.0,0.044052229523588664,0.0008081901182391807,0.11824259236043642,0.2930872570585111,244 diff --git a/results/irm/irm_apo_sensitivity_metadata.csv b/results/irm/irm_apo_sensitivity_metadata.csv index 5f9ff27..3b0ed0e 100644 --- a/results/irm/irm_apo_sensitivity_metadata.csv +++ b/results/irm/irm_apo_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Sensitivity Errors -0.10.dev0,irm_apo_sensitivity.py,2024-12-18 18:32:01,19817.343207597733,3.12.8,736 +0.10.dev0,irm_apo_sensitivity.py,2025-02-04 00:55:57,19823.67243719101,3.12.8,0 From f609aee6bfe42c5e7e2f4b50e3015ce69b8814b9 Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Tue, 4 Feb 2025 09:44:03 +0100 Subject: [PATCH 16/67] reduce n_rep to 250 for IRM and APO --- scripts/irm/irm_apo_sensitivity.py | 2 +- scripts/irm/irm_ate_sensitivity.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/irm/irm_apo_sensitivity.py b/scripts/irm/irm_apo_sensitivity.py index 432e004..d6359d3 100644 --- a/scripts/irm/irm_apo_sensitivity.py +++ b/scripts/irm/irm_apo_sensitivity.py @@ -11,7 +11,7 @@ from doubleml.datasets import make_confounded_irm_data # Number of repetitions -n_rep = 500 +n_rep = 250 max_runtime = 5.5 * 3600 # 5.5 hours in seconds # DGP pars diff --git a/scripts/irm/irm_ate_sensitivity.py b/scripts/irm/irm_ate_sensitivity.py index d29fa77..aac9702 100644 --- a/scripts/irm/irm_ate_sensitivity.py +++ b/scripts/irm/irm_ate_sensitivity.py @@ -11,7 +11,7 @@ from doubleml.datasets import make_confounded_irm_data # Number of repetitions -n_rep = 500 +n_rep = 250 max_runtime = 5.5 * 3600 # 5.5 hours in seconds # DGP pars From 34683d16926a218c14e7a72a476fe4db37531487 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 09:18:07 +0000 Subject: [PATCH 17/67] Update results from script: scripts/irm/irm_gate_coverage.py --- results/irm/irm_gate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_gate_coverage_metadata.csv b/results/irm/irm_gate_coverage_metadata.csv index 5b3a787..50b8fa4 100644 --- a/results/irm/irm_gate_coverage_metadata.csv +++ b/results/irm/irm_gate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_gate_coverage.py,2025-02-03 20:00:25,2082.8255705833435,3.12.8 +0.10.dev0,irm_gate_coverage.py,2025-02-04 09:18:06,1950.9954471588135,3.12.8 From 8b7b4201b8b07aff3aa613b91187b700d40969ba Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 09:41:49 +0000 Subject: [PATCH 18/67] Update results from script: scripts/irm/irm_ate_sensitivity.py --- results/irm/irm_ate_sensitivity.csv | 16 ++++++++-------- results/irm/irm_ate_sensitivity_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_ate_sensitivity.csv b/results/irm/irm_ate_sensitivity.csv index 6e10770..4ced543 100644 --- a/results/irm/irm_ate_sensitivity.csv +++ b/results/irm/irm_ate_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition -LGBM,LGBM,0.9,0.112,0.266748233354866,0.17891290135375168,0.962,1.0,0.12379347892727971,0.05409589192160397,0.04254708028278409,0.32210978560617337,500 -LGBM,LGBM,0.95,0.318,0.31785012462427936,0.17891290135375168,0.998,1.0,0.12379347892727971,0.03441021667548556,0.04254708028278409,0.32210978560617337,500 -LGBM,Logistic Regr.,0.9,0.292,0.2577778025822409,0.14922926552528684,1.0,1.0,0.10066571951295798,0.03493291437943745,0.029012990398602386,0.2979424530565633,500 -LGBM,Logistic Regr.,0.95,0.548,0.30716119707955875,0.14922926552528684,1.0,1.0,0.10066571951295798,0.01869752301454861,0.029012990398602386,0.2979424530565633,500 -Linear Reg.,LGBM,0.9,0.122,0.2675665174758639,0.17873104426193565,0.964,1.0,0.12647219547900976,0.05512739569620471,0.04513946154555041,0.31857328180879246,500 -Linear Reg.,LGBM,0.95,0.314,0.31882517029399604,0.17873104426193565,0.998,1.0,0.12647219547900976,0.035017588858111126,0.04513946154555041,0.31857328180879246,500 -Linear Reg.,Logistic Regr.,0.9,0.86,0.2592281409673473,0.08970251629543106,1.0,1.0,0.06300567732617765,0.006719868195974334,0.05720312141493262,0.23496869651774063,500 -Linear Reg.,Logistic Regr.,0.95,0.974,0.30888938185760084,0.08970251629543106,1.0,1.0,0.06300567732617765,0.0014945204694376396,0.05720312141493262,0.23496869651774063,500 +LGBM,LGBM,0.9,0.124,0.26677505130115525,0.1765053922782123,0.976,1.0,0.12228979172211582,0.052362889318383975,0.04091552128481434,0.3196297552079361,250 +LGBM,LGBM,0.95,0.344,0.31788208017826,0.1765053922782123,1.0,1.0,0.12228979172211582,0.032810864641097386,0.04091552128481434,0.3196297552079361,250 +LGBM,Logistic Regr.,0.9,0.304,0.2579277291971309,0.14752449816798544,1.0,1.0,0.09953613548630397,0.03393799918544193,0.028764647084079123,0.29629144306647,250 +LGBM,Logistic Regr.,0.95,0.56,0.3073398456600121,0.14752449816798544,1.0,1.0,0.09953613548630397,0.017771194883930825,0.028764647084079123,0.29629144306647,250 +Linear Reg.,LGBM,0.9,0.128,0.26773478876971546,0.17729378162308507,0.972,1.0,0.12555203747310065,0.0541289777049747,0.04449232158924936,0.3171219058204927,250 +Linear Reg.,LGBM,0.95,0.312,0.3190256779076688,0.17729378162308507,0.996,1.0,0.12555203747310065,0.03412475848766324,0.04449232158924936,0.3171219058204927,250 +Linear Reg.,Logistic Regr.,0.9,0.884,0.25926273338823186,0.0876812649368125,1.0,1.0,0.06165194867300403,0.005960209905468325,0.05893348758391341,0.23285106977109127,250 +Linear Reg.,Logistic Regr.,0.95,0.984,0.3089306012694445,0.0876812649368125,1.0,1.0,0.06165194867300403,0.0011440372027301212,0.05893348758391341,0.23285106977109127,250 diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv index 6dc906e..c048eb5 100644 --- a/results/irm/irm_ate_sensitivity_metadata.csv +++ b/results/irm/irm_ate_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_sensitivity.py,2025-02-03 21:20:41,6893.791319608688,3.12.8 +0.10.dev0,irm_ate_sensitivity.py,2025-02-04 09:41:49,3372.520791053772,3.12.8 From 981ce25cb8ee7ca8ad5e491047242289bea1773b Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 09:45:42 +0000 Subject: [PATCH 19/67] Update results from script: scripts/irm/irm_ate_coverage.py --- results/irm/irm_ate_coverage.csv | 12 ++++++------ results/irm/irm_ate_coverage_metadata.csv | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv index d0229af..cd71d61 100644 --- a/results/irm/irm_ate_coverage.csv +++ b/results/irm/irm_ate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536215,1000 Lasso,Logistic Regression,0.95,0.935,0.5573839547492131,0.12336884150536215,1000 -Lasso,Random Forest,0.9,0.909,0.5986699126576687,0.1437194448121057,1000 -Lasso,Random Forest,0.95,0.958,0.7133591999985225,0.1437194448121057,1000 -Random Forest,Logistic Regression,0.9,0.8,0.5165274317458133,0.14990700956657027,1000 -Random Forest,Logistic Regression,0.95,0.883,0.6154803969548793,0.14990700956657027,1000 -Random Forest,Random Forest,0.9,0.891,0.6264098177762909,0.1516133190457161,1000 -Random Forest,Random Forest,0.95,0.952,0.7464133356834249,0.1516133190457161,1000 +Lasso,Random Forest,0.9,0.912,0.5966096622110802,0.1474915784047085,1000 +Lasso,Random Forest,0.95,0.959,0.7109042601739195,0.1474915784047085,1000 +Random Forest,Logistic Regression,0.9,0.803,0.5166450978834383,0.15028882582171493,1000 +Random Forest,Logistic Regression,0.95,0.879,0.6156206048056972,0.15028882582171493,1000 +Random Forest,Random Forest,0.9,0.901,0.6352355229138156,0.15259149216742338,1000 +Random Forest,Random Forest,0.95,0.953,0.7569298119973559,0.15259149216742338,1000 diff --git a/results/irm/irm_ate_coverage_metadata.csv b/results/irm/irm_ate_coverage_metadata.csv index 00e4d29..75ae854 100644 --- a/results/irm/irm_ate_coverage_metadata.csv +++ b/results/irm/irm_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_coverage.py,2025-02-03 20:25:20,3578.0215225219727,3.12.8 +0.10.dev0,irm_ate_coverage.py,2025-02-04 09:45:41,3598.778997898102,3.12.8 From 10cff5d3669822cd6d28195785197e5057987e5a Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 09:44:49 +0000 Subject: [PATCH 20/67] Update results from script: scripts/irm/irm_atte_coverage.py --- results/irm/irm_atte_coverage.csv | 16 ++++++++-------- results/irm/irm_atte_coverage_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv index 2593e74..e698f59 100644 --- a/results/irm/irm_atte_coverage.csv +++ b/results/irm/irm_atte_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.888,0.5331759172799809,0.1346025659304845,1000 -Lasso,Logistic Regression,0.95,0.942,0.6353182910443254,0.1346025659304845,1000 -Lasso,Random Forest,0.9,0.892,0.736544703634449,0.18600544721077814,1000 -Lasso,Random Forest,0.95,0.948,0.8776471465140515,0.18600544721077814,1000 -Random Forest,Logistic Regression,0.9,0.87,0.549683356573939,0.14976166337094526,1000 -Random Forest,Logistic Regression,0.95,0.914,0.6549881181724104,0.14976166337094526,1000 -Random Forest,Random Forest,0.9,0.902,0.7467330863654522,0.18215062754687864,1000 -Random Forest,Random Forest,0.95,0.947,0.889787353330196,0.18215062754687864,1000 +Lasso,Logistic Regression,0.9,0.889,0.5331759172799809,0.13455425950247413,1000 +Lasso,Logistic Regression,0.95,0.94,0.6353182910443254,0.13455425950247413,1000 +Lasso,Random Forest,0.9,0.897,0.7349651395902699,0.18174347679882027,1000 +Lasso,Random Forest,0.95,0.948,0.8757649798658234,0.18174347679882027,1000 +Random Forest,Logistic Regression,0.9,0.872,0.5507712587449259,0.14924464481045588,1000 +Random Forest,Logistic Regression,0.95,0.92,0.6562844335641874,0.14924464481045588,1000 +Random Forest,Random Forest,0.9,0.905,0.7483114802258213,0.1823577375111621,1000 +Random Forest,Random Forest,0.95,0.953,0.8916681256184124,0.1823577375111621,1000 diff --git a/results/irm/irm_atte_coverage_metadata.csv b/results/irm/irm_atte_coverage_metadata.csv index 2cc142c..dead2bd 100644 --- a/results/irm/irm_atte_coverage_metadata.csv +++ b/results/irm/irm_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_coverage.py,2025-02-03 20:25:20,3578.3450458049774,3.12.8 +0.10.dev0,irm_atte_coverage.py,2025-02-04 09:44:48,3555.030492067337,3.12.8 From a0840b7c9227a605b1af67341874f28ba85605b4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 10:15:16 +0000 Subject: [PATCH 21/67] Update results from script: scripts/irm/irm_cate_coverage.py --- results/irm/irm_cate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_cate_coverage_metadata.csv b/results/irm/irm_cate_coverage_metadata.csv index 1826629..60b1e2e 100644 --- a/results/irm/irm_cate_coverage_metadata.csv +++ b/results/irm/irm_cate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_cate_coverage.py,2025-02-03 20:56:46,5460.1229820251465,3.12.8 +0.10.dev0,irm_cate_coverage.py,2025-02-04 10:15:15,5378.220800638199,3.12.8 From 695b793e876615b2c2c56f3ff5bcaaad1750c69b Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 10:20:00 +0000 Subject: [PATCH 22/67] Update results from script: scripts/irm/irm_apo_coverage.py --- results/irm/irm_apo_coverage_apo.csv | 48 +++++++++---------- results/irm/irm_apo_coverage_apos.csv | 16 +++---- .../irm/irm_apo_coverage_apos_contrast.csv | 16 +++---- results/irm/irm_apo_coverage_metadata.csv | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv index 013fc38..572e32d 100644 --- a/results/irm/irm_apo_coverage_apo.csv +++ b/results/irm/irm_apo_coverage_apo.csv @@ -1,25 +1,25 @@ Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.075985425408154,1000 -LGBM,LGBM,0.0,0.95,0.964,10.316274091547035,2.075985425408154,1000 -LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.189360948372517,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.189360948372517,1000 -LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.600729846680576,1000 -LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.600729846680576,1000 -LGBM,Logistic,0.0,0.9,0.902,5.625897101886533,1.3356673109449309,1000 -LGBM,Logistic,0.0,0.95,0.956,6.7036698705295725,1.3356673109449309,1000 -LGBM,Logistic,1.0,0.9,0.92,7.423300143143785,1.6958813914305944,1000 -LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6958813914305944,1000 -LGBM,Logistic,2.0,0.9,0.918,7.321275660150268,1.660500602699947,1000 -LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.660500602699947,1000 -Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.305981267915353,1000 -Linear,LGBM,0.0,0.95,0.948,6.551577812380716,1.305981267915353,1000 -Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.126828037202422,1000 -Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.126828037202422,1000 -Linear,LGBM,2.0,0.9,0.932,7.513644049429104,1.6342111609274832,1000 -Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.6342111609274832,1000 -Linear,Logistic,0.0,0.9,0.902,5.335670092717667,1.2845963390782078,1000 -Linear,Logistic,0.0,0.95,0.953,6.357843058957276,1.2845963390782078,1000 -Linear,Logistic,1.0,0.9,0.904,5.417512107920403,1.280503259417172,1000 -Linear,Logistic,1.0,0.95,0.954,6.455363835025866,1.280503259417172,1000 -Linear,Logistic,2.0,0.9,0.907,5.366403391397197,1.2793569219717145,1000 -Linear,Logistic,2.0,0.95,0.96,6.3944640430685675,1.2793569219717145,1000 +LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.075993749929611,1000 +LGBM,LGBM,0.0,0.95,0.964,10.316274091547035,2.075993749929611,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.186542742710675,1000 +LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.186542742710675,1000 +LGBM,LGBM,2.0,0.9,0.89,37.49194764946096,9.603990986009524,1000 +LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.603990986009524,1000 +LGBM,Logistic,0.0,0.9,0.903,5.625897101886533,1.3353194514431521,1000 +LGBM,Logistic,0.0,0.95,0.957,6.7036698705295725,1.3353194514431521,1000 +LGBM,Logistic,1.0,0.9,0.922,7.423300143143785,1.6962452101686065,1000 +LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6962452101686065,1000 +LGBM,Logistic,2.0,0.9,0.918,7.321275660150268,1.6605428913390372,1000 +LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.6605428913390372,1000 +Linear,LGBM,0.0,0.9,0.9,5.498257423071024,1.3054063118189718,1000 +Linear,LGBM,0.0,0.95,0.948,6.551577812380716,1.3054063118189718,1000 +Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.1266852211803364,1000 +Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.1266852211803364,1000 +Linear,LGBM,2.0,0.9,0.932,7.513644049429104,1.6340563160334893,1000 +Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.6340563160334893,1000 +Linear,Logistic,0.0,0.9,0.902,5.335670092717667,1.2841438731760095,1000 +Linear,Logistic,0.0,0.95,0.954,6.357843058957276,1.2841438731760095,1000 +Linear,Logistic,1.0,0.9,0.904,5.417512107920403,1.280708898509647,1000 +Linear,Logistic,1.0,0.95,0.954,6.455363835025866,1.280708898509647,1000 +Linear,Logistic,2.0,0.9,0.906,5.366403391397197,1.278693826210701,1000 +Linear,Logistic,2.0,0.95,0.959,6.3944640430685675,1.278693826210701,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv index 81371cc..956ad22 100644 --- a/results/irm/irm_apo_coverage_apos.csv +++ b/results/irm/irm_apo_coverage_apos.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.119510083508661,0.924,36.1943517986729,1000 -LGBM,LGBM,0.95,0.9583333333333334,33.61454856442563,7.119510083508661,0.974,40.868020474452145,1000 -LGBM,Logistic,0.9,0.9173333333333333,6.789316986971467,1.570973449234106,0.923,8.394070597867524,1000 -LGBM,Logistic,0.95,0.963,8.089970168806186,1.570973449234106,0.96,9.592284619044477,1000 -Linear,LGBM,0.9,0.926,7.903418354805325,1.725523010236133,0.939,9.850445774370156,1000 -Linear,LGBM,0.95,0.967,9.41750382912841,1.725523010236133,0.974,11.234392064837424,1000 -Linear,Logistic,0.9,0.9036666666666666,5.372532806955153,1.2795639356934292,0.905,5.7964894175017925,1000 -Linear,Logistic,0.95,0.9573333333333334,6.4017676921854445,1.2795639356934292,0.953,6.8182385960659,1000 +LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.119905917981641,0.924,36.1943517986729,1000 +LGBM,LGBM,0.95,0.9586666666666667,33.61454856442563,7.119905917981641,0.974,40.868020474452145,1000 +LGBM,Logistic,0.9,0.9176666666666666,6.789316986971467,1.570839912306491,0.924,8.394070597867524,1000 +LGBM,Logistic,0.95,0.963,8.089970168806186,1.570839912306491,0.959,9.592284619044477,1000 +Linear,LGBM,0.9,0.9273333333333333,7.903418354805325,1.7251987666785773,0.94,9.850445774370156,1000 +Linear,LGBM,0.95,0.967,9.41750382912841,1.7251987666785773,0.975,11.234392064837424,1000 +Linear,Logistic,0.9,0.904,5.372532806955153,1.279293151465328,0.906,5.7964894175017925,1000 +Linear,Logistic,0.95,0.957,6.4017676921854445,1.279293151465328,0.953,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv index bffe6b6..8dc1901 100644 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.787707441089529,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.9485,45.13128131893863,9.787707441089529,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.9275,5.725128733165455,1.267568231095916,0.927,6.774304315459575,1000 -LGBM,Logistic,0.95,0.9625,6.821911652197591,1.267568231095916,0.962,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.5065164682583199,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.989,8.85452711998085,1.5065164682583199,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.873,1.1425883251377777,0.29572275431639516,0.872,1.3505222299078565,1000 -Linear,Logistic,0.95,0.936,1.3614779635902856,0.29572275431639516,0.919,1.5496710496635275,1000 +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.788054543020683,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.9485,45.13128131893863,9.788054543020683,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2675085831936368,0.927,6.774304315459575,1000 +LGBM,Logistic,0.95,0.9625,6.821911652197591,1.2675085831936368,0.962,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.5065020389813555,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.989,8.85452711998085,1.5065020389813555,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.873,1.1425883251377777,0.295522974485943,0.872,1.3505222299078565,1000 +Linear,Logistic,0.95,0.936,1.3614779635902856,0.295522974485943,0.919,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv index 214d397..fe2d7cb 100644 --- a/results/irm/irm_apo_coverage_metadata.csv +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_apo_coverage.py,2025-02-03 21:02:12,5798.276986837387,3.12.8 +0.10.dev0,irm_apo_coverage.py,2025-02-04 10:19:56,5644.753345966339,3.12.8 From 41520d3bc1e696fe43e16d34d12b97ebe0c13125 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 10:36:22 +0000 Subject: [PATCH 23/67] Update results from script: scripts/irm/irm_atte_sensitivity.py --- results/irm/irm_atte_sensitivity_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv index 449b919..d613ea1 100644 --- a/results/irm/irm_atte_sensitivity_metadata.csv +++ b/results/irm/irm_atte_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_sensitivity.py,2025-02-03 21:20:31,6886.662088632584,3.12.8 +0.10.dev0,irm_atte_sensitivity.py,2025-02-04 10:36:21,6639.717920541763,3.12.8 From 5e6419b753ecd936311b5e537310effa8c44b479 Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Tue, 4 Feb 2025 14:41:23 +0100 Subject: [PATCH 24/67] add CI bound length for sensitivity results --- scripts/irm/irm_apo_sensitivity.py | 3 +++ scripts/irm/irm_ate_sensitivity.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/scripts/irm/irm_apo_sensitivity.py b/scripts/irm/irm_apo_sensitivity.py index d6359d3..5a80340 100644 --- a/scripts/irm/irm_apo_sensitivity.py +++ b/scripts/irm/irm_apo_sensitivity.py @@ -114,6 +114,7 @@ causal_contrast_model.sensitivity_analysis(cf_y=cf_y, cf_d=cf_d, rho=rho, level=level ,null_hypothesis=theta) cover_lower = theta >= causal_contrast_model.sensitivity_params['ci']['lower'] cover_upper = theta <= causal_contrast_model.sensitivity_params['ci']['upper'] + ci_bound_width = causal_contrast_model.sensitivity_params['ci']['upper'][0]- causal_contrast_model.sensitivity_params['ci']['lower'][0] rv = causal_contrast_model.sensitivity_params['rv'] rva = causal_contrast_model.sensitivity_params['rva'] bias_lower = abs(theta - causal_contrast_model.sensitivity_params['theta']['lower']) @@ -135,6 +136,7 @@ "RVa": rva, "Bias (Lower)": bias_lower, "Bias (Upper)": bias_upper, + "CI Bound Length": ci_bound_width, "Learner g": learner_g_name, "Learner m": learner_m_name, "level": level, @@ -156,6 +158,7 @@ "RVa": "mean", "Bias (Lower)": "mean", "Bias (Upper)": "mean", + "CI Bound Length": "mean", "repetition": "count"} ).reset_index() print(df_results) diff --git a/scripts/irm/irm_ate_sensitivity.py b/scripts/irm/irm_ate_sensitivity.py index aac9702..d91f830 100644 --- a/scripts/irm/irm_ate_sensitivity.py +++ b/scripts/irm/irm_ate_sensitivity.py @@ -101,6 +101,7 @@ dml_irm.sensitivity_analysis(cf_y=cf_y, cf_d=cf_d, rho=rho, level=level, null_hypothesis=theta) cover_lower = theta >= dml_irm.sensitivity_params['ci']['lower'] cover_upper = theta <= dml_irm.sensitivity_params['ci']['upper'] + ci_bound_width = dml_irm.sensitivity_params['ci']['upper'][0]- dml_irm.sensitivity_params['ci']['lower'][0] rv = dml_irm.sensitivity_params['rv'] rva = dml_irm.sensitivity_params['rva'] bias_lower = abs(theta - dml_irm.sensitivity_params['theta']['lower']) @@ -118,6 +119,7 @@ "RVa": rva, "Bias (Lower)": bias_lower, "Bias (Upper)": bias_upper, + "CI Bound Length": ci_bound_width, "Learner g": learner_g_name, "Learner m": learner_m_name, "level": level, @@ -135,6 +137,7 @@ "RVa": "mean", "Bias (Lower)": "mean", "Bias (Upper)": "mean", + "CI Bound Length": "mean", "repetition": "count"} ).reset_index() print(df_results) From 806b710c852e43d1cf194cb91c27076456028783 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 14:17:40 +0000 Subject: [PATCH 25/67] Update results from script: scripts/irm/irm_gate_coverage.py --- results/irm/irm_gate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_gate_coverage_metadata.csv b/results/irm/irm_gate_coverage_metadata.csv index 50b8fa4..5e92180 100644 --- a/results/irm/irm_gate_coverage_metadata.csv +++ b/results/irm/irm_gate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_gate_coverage.py,2025-02-04 09:18:06,1950.9954471588135,3.12.8 +0.10.dev0,irm_gate_coverage.py,2025-02-04 14:17:39,2070.3476033210754,3.12.8 From 35d46a6eb4314379a5d11af935145c7519cd5320 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 14:43:45 +0000 Subject: [PATCH 26/67] Update results from script: scripts/irm/irm_atte_coverage.py --- results/irm/irm_atte_coverage.csv | 16 ++++++++-------- results/irm/irm_atte_coverage_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv index e698f59..d07986d 100644 --- a/results/irm/irm_atte_coverage.csv +++ b/results/irm/irm_atte_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.889,0.5331759172799809,0.13455425950247413,1000 -Lasso,Logistic Regression,0.95,0.94,0.6353182910443254,0.13455425950247413,1000 -Lasso,Random Forest,0.9,0.897,0.7349651395902699,0.18174347679882027,1000 -Lasso,Random Forest,0.95,0.948,0.8757649798658234,0.18174347679882027,1000 -Random Forest,Logistic Regression,0.9,0.872,0.5507712587449259,0.14924464481045588,1000 -Random Forest,Logistic Regression,0.95,0.92,0.6562844335641874,0.14924464481045588,1000 -Random Forest,Random Forest,0.9,0.905,0.7483114802258213,0.1823577375111621,1000 -Random Forest,Random Forest,0.95,0.953,0.8916681256184124,0.1823577375111621,1000 +Lasso,Logistic Regression,0.9,0.89,0.5331759172799809,0.1350007505870614,1000 +Lasso,Logistic Regression,0.95,0.937,0.6353182910443254,0.1350007505870614,1000 +Lasso,Random Forest,0.9,0.9,0.7322748202898294,0.1827151434075385,1000 +Lasso,Random Forest,0.95,0.95,0.8725592666951327,0.1827151434075385,1000 +Random Forest,Logistic Regression,0.9,0.869,0.5487830931792036,0.15020896899050296,1000 +Random Forest,Logistic Regression,0.95,0.916,0.6539153881729932,0.15020896899050296,1000 +Random Forest,Random Forest,0.9,0.9,0.7513607898901226,0.18519153423783763,1000 +Random Forest,Random Forest,0.95,0.944,0.8953016021915328,0.18519153423783763,1000 diff --git a/results/irm/irm_atte_coverage_metadata.csv b/results/irm/irm_atte_coverage_metadata.csv index dead2bd..56ee932 100644 --- a/results/irm/irm_atte_coverage_metadata.csv +++ b/results/irm/irm_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_coverage.py,2025-02-04 09:44:48,3555.030492067337,3.12.8 +0.10.dev0,irm_atte_coverage.py,2025-02-04 14:43:44,3635.82240152359,3.12.8 From 07288ace4f75c3798b1fb89779463b5c7595d0fb Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 14:40:23 +0000 Subject: [PATCH 27/67] Update results from script: scripts/irm/irm_ate_sensitivity.py --- results/irm/irm_ate_sensitivity_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv index c048eb5..6a48c17 100644 --- a/results/irm/irm_ate_sensitivity_metadata.csv +++ b/results/irm/irm_ate_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_sensitivity.py,2025-02-04 09:41:49,3372.520791053772,3.12.8 +0.10.dev0,irm_ate_sensitivity.py,2025-02-04 14:40:22,3430.5844337940216,3.12.8 From a0717d303e3d70df84136bada30e72f6c300c54e Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 14:43:02 +0000 Subject: [PATCH 28/67] Update results from script: scripts/irm/irm_ate_coverage.py --- results/irm/irm_ate_coverage.csv | 12 ++++++------ results/irm/irm_ate_coverage_metadata.csv | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv index cd71d61..d1b75ad 100644 --- a/results/irm/irm_ate_coverage.csv +++ b/results/irm/irm_ate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536215,1000 Lasso,Logistic Regression,0.95,0.935,0.5573839547492131,0.12336884150536215,1000 -Lasso,Random Forest,0.9,0.912,0.5966096622110802,0.1474915784047085,1000 -Lasso,Random Forest,0.95,0.959,0.7109042601739195,0.1474915784047085,1000 -Random Forest,Logistic Regression,0.9,0.803,0.5166450978834383,0.15028882582171493,1000 -Random Forest,Logistic Regression,0.95,0.879,0.6156206048056972,0.15028882582171493,1000 -Random Forest,Random Forest,0.9,0.901,0.6352355229138156,0.15259149216742338,1000 -Random Forest,Random Forest,0.95,0.953,0.7569298119973559,0.15259149216742338,1000 +Lasso,Random Forest,0.9,0.912,0.6033525996000427,0.1470798884283924,1000 +Lasso,Random Forest,0.95,0.961,0.7189389656430433,0.1470798884283924,1000 +Random Forest,Logistic Regression,0.9,0.796,0.5170183758598037,0.14999135577981693,1000 +Random Forest,Logistic Regression,0.95,0.879,0.6160653929484903,0.14999135577981693,1000 +Random Forest,Random Forest,0.9,0.898,0.6346634640451655,0.1547114295252091,1000 +Random Forest,Random Forest,0.95,0.954,0.7562481618120631,0.1547114295252091,1000 diff --git a/results/irm/irm_ate_coverage_metadata.csv b/results/irm/irm_ate_coverage_metadata.csv index 75ae854..6966b69 100644 --- a/results/irm/irm_ate_coverage_metadata.csv +++ b/results/irm/irm_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_coverage.py,2025-02-04 09:45:41,3598.778997898102,3.12.8 +0.10.dev0,irm_ate_coverage.py,2025-02-04 14:43:01,3590.5973885059357,3.12.8 From b3974fe954bb72fbcc1e962853815b2c618ffc9c Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 15:10:17 +0000 Subject: [PATCH 29/67] Update results from script: scripts/irm/irm_cate_coverage.py --- results/irm/irm_cate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_cate_coverage_metadata.csv b/results/irm/irm_cate_coverage_metadata.csv index 60b1e2e..28caa05 100644 --- a/results/irm/irm_cate_coverage_metadata.csv +++ b/results/irm/irm_cate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_cate_coverage.py,2025-02-04 10:15:15,5378.220800638199,3.12.8 +0.10.dev0,irm_cate_coverage.py,2025-02-04 15:10:16,5228.195962905884,3.12.8 From ef5116f0248086fc950413ad2bb39cecd8c3533f Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 15:19:02 +0000 Subject: [PATCH 30/67] Update results from script: scripts/irm/irm_apo_coverage.py --- results/irm/irm_apo_coverage_apo.csv | 48 +++++++++---------- results/irm/irm_apo_coverage_apos.csv | 16 +++---- .../irm/irm_apo_coverage_apos_contrast.csv | 16 +++---- results/irm/irm_apo_coverage_metadata.csv | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv index 572e32d..69ee2bf 100644 --- a/results/irm/irm_apo_coverage_apo.csv +++ b/results/irm/irm_apo_coverage_apo.csv @@ -1,25 +1,25 @@ Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.075993749929611,1000 -LGBM,LGBM,0.0,0.95,0.964,10.316274091547035,2.075993749929611,1000 -LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.186542742710675,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.186542742710675,1000 -LGBM,LGBM,2.0,0.9,0.89,37.49194764946096,9.603990986009524,1000 -LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.603990986009524,1000 -LGBM,Logistic,0.0,0.9,0.903,5.625897101886533,1.3353194514431521,1000 -LGBM,Logistic,0.0,0.95,0.957,6.7036698705295725,1.3353194514431521,1000 -LGBM,Logistic,1.0,0.9,0.922,7.423300143143785,1.6962452101686065,1000 -LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6962452101686065,1000 -LGBM,Logistic,2.0,0.9,0.918,7.321275660150268,1.6605428913390372,1000 -LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.6605428913390372,1000 -Linear,LGBM,0.0,0.9,0.9,5.498257423071024,1.3054063118189718,1000 -Linear,LGBM,0.0,0.95,0.948,6.551577812380716,1.3054063118189718,1000 -Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.1266852211803364,1000 -Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.1266852211803364,1000 -Linear,LGBM,2.0,0.9,0.932,7.513644049429104,1.6340563160334893,1000 -Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.6340563160334893,1000 -Linear,Logistic,0.0,0.9,0.902,5.335670092717667,1.2841438731760095,1000 -Linear,Logistic,0.0,0.95,0.954,6.357843058957276,1.2841438731760095,1000 -Linear,Logistic,1.0,0.9,0.904,5.417512107920403,1.280708898509647,1000 -Linear,Logistic,1.0,0.95,0.954,6.455363835025866,1.280708898509647,1000 -Linear,Logistic,2.0,0.9,0.906,5.366403391397197,1.278693826210701,1000 -Linear,Logistic,2.0,0.95,0.959,6.3944640430685675,1.278693826210701,1000 +LGBM,LGBM,0.0,0.9,0.91,8.657690136121921,2.0774322102310396,1000 +LGBM,LGBM,0.0,0.95,0.966,10.316274091547035,2.0774322102310396,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.221499001374493,1000 +LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.221499001374493,1000 +LGBM,LGBM,2.0,0.9,0.894,37.49194764946096,9.569628503931227,1000 +LGBM,LGBM,2.0,0.95,0.953,44.67441108385784,9.569628503931227,1000 +LGBM,Logistic,0.0,0.9,0.903,5.625897101886533,1.3427898044782796,1000 +LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.3427898044782796,1000 +LGBM,Logistic,1.0,0.9,0.924,7.423300143143785,1.6931903229654406,1000 +LGBM,Logistic,1.0,0.95,0.97,8.84540769378873,1.6931903229654406,1000 +LGBM,Logistic,2.0,0.9,0.92,7.321275660150268,1.6627857804697925,1000 +LGBM,Logistic,2.0,0.95,0.965,8.723838024042964,1.6627857804697925,1000 +Linear,LGBM,0.0,0.9,0.902,5.498257423071024,1.31333599452238,1000 +Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.31333599452238,1000 +Linear,LGBM,1.0,0.9,0.947,10.700720020780512,2.129828839351603,1000 +Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.129828839351603,1000 +Linear,LGBM,2.0,0.9,0.934,7.513644049429104,1.6381350879200112,1000 +Linear,LGBM,2.0,0.95,0.967,8.953059097926168,1.6381350879200112,1000 +Linear,Logistic,0.0,0.9,0.904,5.335670092717667,1.2923995163085065,1000 +Linear,Logistic,0.0,0.95,0.954,6.357843058957276,1.2923995163085065,1000 +Linear,Logistic,1.0,0.9,0.908,5.417512107920403,1.2807077813820356,1000 +Linear,Logistic,1.0,0.95,0.959,6.455363835025866,1.2807077813820356,1000 +Linear,Logistic,2.0,0.9,0.908,5.366403391397197,1.2885877554305896,1000 +Linear,Logistic,2.0,0.95,0.956,6.3944640430685675,1.2885877554305896,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv index 956ad22..b6d385a 100644 --- a/results/irm/irm_apo_coverage_apos.csv +++ b/results/irm/irm_apo_coverage_apos.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.119905917981641,0.924,36.1943517986729,1000 -LGBM,LGBM,0.95,0.9586666666666667,33.61454856442563,7.119905917981641,0.974,40.868020474452145,1000 -LGBM,Logistic,0.9,0.9176666666666666,6.789316986971467,1.570839912306491,0.924,8.394070597867524,1000 -LGBM,Logistic,0.95,0.963,8.089970168806186,1.570839912306491,0.959,9.592284619044477,1000 -Linear,LGBM,0.9,0.9273333333333333,7.903418354805325,1.7251987666785773,0.94,9.850445774370156,1000 -Linear,LGBM,0.95,0.967,9.41750382912841,1.7251987666785773,0.975,11.234392064837424,1000 -Linear,Logistic,0.9,0.904,5.372532806955153,1.279293151465328,0.906,5.7964894175017925,1000 -Linear,Logistic,0.95,0.957,6.4017676921854445,1.279293151465328,0.953,6.8182385960659,1000 +LGBM,LGBM,0.9,0.9066666666666666,28.21021843292038,7.118050410548393,0.925,36.1943517986729,1000 +LGBM,LGBM,0.95,0.958,33.61454856442563,7.118050410548393,0.973,40.868020474452145,1000 +LGBM,Logistic,0.9,0.9136666666666666,6.789316986971467,1.57447223146738,0.921,8.394070597867524,1000 +LGBM,Logistic,0.95,0.9626666666666667,8.089970168806186,1.57447223146738,0.959,9.592284619044477,1000 +Linear,LGBM,0.9,0.9273333333333333,7.903418354805325,1.7307958124719784,0.937,9.850445774370156,1000 +Linear,LGBM,0.95,0.9686666666666667,9.41750382912841,1.7307958124719784,0.975,11.234392064837424,1000 +Linear,Logistic,0.9,0.9053333333333333,5.372532806955153,1.284757885951191,0.902,5.7964894175017925,1000 +Linear,Logistic,0.95,0.9553333333333334,6.4017676921854445,1.284757885951191,0.952,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv index 8dc1901..9f5e5b1 100644 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.788054543020683,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.9485,45.13128131893863,9.788054543020683,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2675085831936368,0.927,6.774304315459575,1000 -LGBM,Logistic,0.95,0.9625,6.821911652197591,1.2675085831936368,0.962,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.5065020389813555,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.989,8.85452711998085,1.5065020389813555,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.873,1.1425883251377777,0.295522974485943,0.872,1.3505222299078565,1000 -Linear,Logistic,0.95,0.936,1.3614779635902856,0.295522974485943,0.919,1.5496710496635275,1000 +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.786633570698548,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.9485,45.13128131893863,9.786633570698548,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2678647082222205,0.926,6.774304315459575,1000 +LGBM,Logistic,0.95,0.962,6.821911652197591,1.2678647082222205,0.961,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.5066569199658195,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.9885,8.85452711998085,1.5066569199658195,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.874,1.1425883251377777,0.2971374642918569,0.871,1.3505222299078565,1000 +Linear,Logistic,0.95,0.934,1.3614779635902856,0.2971374642918569,0.917,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv index fe2d7cb..ed6a0ab 100644 --- a/results/irm/irm_apo_coverage_metadata.csv +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_apo_coverage.py,2025-02-04 10:19:56,5644.753345966339,3.12.8 +0.10.dev0,irm_apo_coverage.py,2025-02-04 15:18:58,5762.698203086853,3.12.8 From da516fd137886c0a31fac463446ce9ee67f682f7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 15:37:16 +0000 Subject: [PATCH 31/67] Update results from script: scripts/irm/irm_atte_sensitivity.py --- results/irm/irm_atte_sensitivity_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv index d613ea1..108a6db 100644 --- a/results/irm/irm_atte_sensitivity_metadata.csv +++ b/results/irm/irm_atte_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_sensitivity.py,2025-02-04 10:36:21,6639.717920541763,3.12.8 +0.10.dev0,irm_atte_sensitivity.py,2025-02-04 15:37:15,6846.028452157974,3.12.8 From 8caf302fab0c006f632c0a2e99ae67ce8c526ffc Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 4 Feb 2025 19:14:04 +0000 Subject: [PATCH 32/67] Update results from script: scripts/irm/irm_apo_sensitivity.py --- results/irm/irm_apo_sensitivity.csv | 16 ++++++++-------- results/irm/irm_apo_sensitivity_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_apo_sensitivity.csv b/results/irm/irm_apo_sensitivity.csv index 606bee8..62a9182 100644 --- a/results/irm/irm_apo_sensitivity.csv +++ b/results/irm/irm_apo_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition -LGBM,LGBM,0.9,0.09836065573770492,0.26980333552727825,0.18205252584345485,1.0,1.0,0.09239844030018919,0.04045631137696363,0.035066111350142995,0.3810774239324017,244 -LGBM,LGBM,0.95,0.3073770491803279,0.32149050339653273,0.18205252584345485,1.0,1.0,0.09239844030018919,0.025884873713769935,0.035066111350142995,0.3810774239324017,244 -LGBM,Logistic Regr.,0.9,0.3073770491803279,0.2578060603442706,0.14817801851576512,1.0,1.0,0.07187480682741719,0.024545078956797563,0.06295056021181017,0.35846457343559074,244 -LGBM,Logistic Regr.,0.95,0.5614754098360656,0.30719486827981307,0.14817801851576512,1.0,1.0,0.07187480682741719,0.012961043152938044,0.06295056021181017,0.35846457343559074,244 -Linear Reg.,LGBM,0.9,0.10655737704918032,0.26955592225259073,0.18010332737150503,1.0,1.0,0.0941627548347164,0.04084948000316684,0.03472113416565001,0.37380556783576596,244 -Linear Reg.,LGBM,0.95,0.29918032786885246,0.3211956922961776,0.18010332737150503,1.0,1.0,0.0941627548347164,0.026040282694101653,0.03472113416565001,0.37380556783576596,244 -Linear Reg.,Logistic Regr.,0.9,0.8852459016393442,0.25925313646720843,0.08765249652827344,1.0,1.0,0.044052229523588664,0.0042325141887964,0.11824259236043642,0.2930872570585111,244 -Linear Reg.,Logistic Regr.,0.95,0.9836065573770492,0.30891916583272205,0.08765249652827344,1.0,1.0,0.044052229523588664,0.0008081901182391807,0.11824259236043642,0.2930872570585111,244 +LGBM,LGBM,0.9,0.09795918367346938,0.26979767912769265,0.1821217753924097,1.0,1.0,0.09243594786866559,0.040494673522359054,0.034931083457801146,0.3811385489487934,245 +LGBM,LGBM,0.95,0.30612244897959184,0.3214837633807111,0.1821217753924097,1.0,1.0,0.09243594786866559,0.025920252504384952,0.034931083457801146,0.3811385489487934,245 +LGBM,Logistic Regr.,0.9,0.30612244897959184,0.25780889388055256,0.1481921276548859,1.0,1.0,0.07187715697817153,0.024546744165995487,0.06294768349694302,0.35849335183083353,245 +LGBM,Logistic Regr.,0.95,0.563265306122449,0.30719824464654294,0.1481921276548859,1.0,1.0,0.07187715697817153,0.012953227730808654,0.06294768349694302,0.35849335183083353,245 +Linear Reg.,LGBM,0.9,0.10612244897959183,0.2695747491766244,0.18019285630080534,1.0,1.0,0.09421113740884988,0.04089415485221311,0.03462367597552903,0.37388485920180015,245 +Linear Reg.,LGBM,0.95,0.2979591836734694,0.32121812595984284,0.18019285630080534,1.0,1.0,0.09421113740884988,0.026079753922994376,0.03462367597552903,0.37388485920180015,245 +Linear Reg.,Logistic Regr.,0.9,0.8857142857142857,0.2592552019315289,0.08773320288603256,1.0,1.0,0.044089226784792754,0.004228149001636824,0.11817382692543607,0.2931817832302881,245 +Linear Reg.,Logistic Regr.,0.95,0.9836734693877551,0.30892162698526066,0.08773320288603256,1.0,1.0,0.044089226784792754,0.0008049142525428561,0.11817382692543607,0.2931817832302881,245 diff --git a/results/irm/irm_apo_sensitivity_metadata.csv b/results/irm/irm_apo_sensitivity_metadata.csv index 3b0ed0e..b50c486 100644 --- a/results/irm/irm_apo_sensitivity_metadata.csv +++ b/results/irm/irm_apo_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Sensitivity Errors -0.10.dev0,irm_apo_sensitivity.py,2025-02-04 00:55:57,19823.67243719101,3.12.8,0 +0.10.dev0,irm_apo_sensitivity.py,2025-02-04 19:14:02,19864.97425556183,3.12.8,0 From 51bffd6b2872ca02db987f6b10a5f61704885aae Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Wed, 5 Feb 2025 08:19:45 +0100 Subject: [PATCH 33/67] reduce n_rep to 100 --- scripts/irm/irm_apo_sensitivity.py | 2 +- scripts/irm/irm_ate_sensitivity.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/irm/irm_apo_sensitivity.py b/scripts/irm/irm_apo_sensitivity.py index 5a80340..e61b1e2 100644 --- a/scripts/irm/irm_apo_sensitivity.py +++ b/scripts/irm/irm_apo_sensitivity.py @@ -11,7 +11,7 @@ from doubleml.datasets import make_confounded_irm_data # Number of repetitions -n_rep = 250 +n_rep = 100 max_runtime = 5.5 * 3600 # 5.5 hours in seconds # DGP pars diff --git a/scripts/irm/irm_ate_sensitivity.py b/scripts/irm/irm_ate_sensitivity.py index d91f830..7b48ae2 100644 --- a/scripts/irm/irm_ate_sensitivity.py +++ b/scripts/irm/irm_ate_sensitivity.py @@ -11,7 +11,7 @@ from doubleml.datasets import make_confounded_irm_data # Number of repetitions -n_rep = 250 +n_rep = 100 max_runtime = 5.5 * 3600 # 5.5 hours in seconds # DGP pars From ccd6ba66b93ead9d86ebac9915464776e6e1d040 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 07:49:10 +0000 Subject: [PATCH 34/67] Update results from script: scripts/irm/irm_ate_sensitivity.py --- results/irm/irm_ate_sensitivity.csv | 18 +++++++++--------- results/irm/irm_ate_sensitivity_metadata.csv | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/results/irm/irm_ate_sensitivity.csv b/results/irm/irm_ate_sensitivity.csv index 4ced543..3f47f2c 100644 --- a/results/irm/irm_ate_sensitivity.csv +++ b/results/irm/irm_ate_sensitivity.csv @@ -1,9 +1,9 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition -LGBM,LGBM,0.9,0.124,0.26677505130115525,0.1765053922782123,0.976,1.0,0.12228979172211582,0.052362889318383975,0.04091552128481434,0.3196297552079361,250 -LGBM,LGBM,0.95,0.344,0.31788208017826,0.1765053922782123,1.0,1.0,0.12228979172211582,0.032810864641097386,0.04091552128481434,0.3196297552079361,250 -LGBM,Logistic Regr.,0.9,0.304,0.2579277291971309,0.14752449816798544,1.0,1.0,0.09953613548630397,0.03393799918544193,0.028764647084079123,0.29629144306647,250 -LGBM,Logistic Regr.,0.95,0.56,0.3073398456600121,0.14752449816798544,1.0,1.0,0.09953613548630397,0.017771194883930825,0.028764647084079123,0.29629144306647,250 -Linear Reg.,LGBM,0.9,0.128,0.26773478876971546,0.17729378162308507,0.972,1.0,0.12555203747310065,0.0541289777049747,0.04449232158924936,0.3171219058204927,250 -Linear Reg.,LGBM,0.95,0.312,0.3190256779076688,0.17729378162308507,0.996,1.0,0.12555203747310065,0.03412475848766324,0.04449232158924936,0.3171219058204927,250 -Linear Reg.,Logistic Regr.,0.9,0.884,0.25926273338823186,0.0876812649368125,1.0,1.0,0.06165194867300403,0.005960209905468325,0.05893348758391341,0.23285106977109127,250 -Linear Reg.,Logistic Regr.,0.95,0.984,0.3089306012694445,0.0876812649368125,1.0,1.0,0.06165194867300403,0.0011440372027301212,0.05893348758391341,0.23285106977109127,250 +Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),CI Bound Length,repetition +LGBM,LGBM,0.9,0.08,0.2670216845863586,0.17940963620485714,0.96,1.0,0.1241372280116238,0.05432610289837227,0.041818765832749474,0.32254502881613434,0.4945097406195433,100 +LGBM,LGBM,0.95,0.35,0.3181759618638197,0.17940963620485714,1.0,1.0,0.1241372280116238,0.03439763897795268,0.041818765832749474,0.32254502881613434,0.5535425909510789,100 +LGBM,Logistic Regr.,0.9,0.3,0.2581285280201102,0.1489250720013938,1.0,1.0,0.10039281695402444,0.034610324016033665,0.027819425637511053,0.29775753820387224,0.4988408323876356,100 +LGBM,Logistic Regr.,0.95,0.56,0.3075791122152416,0.1489250720013938,1.0,1.0,0.10039281695402444,0.017870308168433494,0.027819425637511053,0.29775753820387224,0.5558714046518355,100 +Linear Reg.,LGBM,0.9,0.09,0.2680454420037482,0.1789353408854105,0.97,1.0,0.12697023327157428,0.055500063642588984,0.04546847329030372,0.31838417542283604,0.4879323625959303,100 +Linear Reg.,LGBM,0.95,0.29,0.3193958440673857,0.1789353408854105,0.99,1.0,0.12697023327157428,0.03516077097784765,0.04546847329030372,0.31838417542283604,0.5471907936286992,100 +Linear Reg.,Logistic Regr.,0.9,0.88,0.2593925192155475,0.09058684260970791,1.0,1.0,0.0636454347862647,0.00654738335028408,0.05695175811582821,0.2354964421984655,0.4929868678265177,100 +Linear Reg.,Logistic Regr.,0.95,0.97,0.3090852506212617,0.09058684260970791,1.0,1.0,0.0636454347862647,0.0014769375799795156,0.05695175811582821,0.2354964421984655,0.5502941581407065,100 diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv index 6a48c17..23018f0 100644 --- a/results/irm/irm_ate_sensitivity_metadata.csv +++ b/results/irm/irm_ate_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_sensitivity.py,2025-02-04 14:40:22,3430.5844337940216,3.12.8 +0.10.dev0,irm_ate_sensitivity.py,2025-02-05 07:49:10,1345.4034423828125,3.12.8 From 02286bbd3dcf8d0a0bec7032e0d62359d7a0cc0c Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 08:00:50 +0000 Subject: [PATCH 35/67] Update results from script: scripts/irm/irm_gate_coverage.py --- results/irm/irm_gate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_gate_coverage_metadata.csv b/results/irm/irm_gate_coverage_metadata.csv index 5e92180..6610d3a 100644 --- a/results/irm/irm_gate_coverage_metadata.csv +++ b/results/irm/irm_gate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_gate_coverage.py,2025-02-04 14:17:39,2070.3476033210754,3.12.8 +0.10.dev0,irm_gate_coverage.py,2025-02-05 08:00:49,2046.5704367160797,3.12.8 From 3dd30fdd156927544a62b999c41178a49852dc00 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 08:26:29 +0000 Subject: [PATCH 36/67] Update results from script: scripts/irm/irm_atte_coverage.py --- results/irm/irm_atte_coverage.csv | 16 ++++++++-------- results/irm/irm_atte_coverage_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv index d07986d..83a5c9a 100644 --- a/results/irm/irm_atte_coverage.csv +++ b/results/irm/irm_atte_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.89,0.5331759172799809,0.1350007505870614,1000 -Lasso,Logistic Regression,0.95,0.937,0.6353182910443254,0.1350007505870614,1000 -Lasso,Random Forest,0.9,0.9,0.7322748202898294,0.1827151434075385,1000 -Lasso,Random Forest,0.95,0.95,0.8725592666951327,0.1827151434075385,1000 -Random Forest,Logistic Regression,0.9,0.869,0.5487830931792036,0.15020896899050296,1000 -Random Forest,Logistic Regression,0.95,0.916,0.6539153881729932,0.15020896899050296,1000 -Random Forest,Random Forest,0.9,0.9,0.7513607898901226,0.18519153423783763,1000 -Random Forest,Random Forest,0.95,0.944,0.8953016021915328,0.18519153423783763,1000 +Lasso,Logistic Regression,0.9,0.887,0.5331759172799809,0.13448727909967034,1000 +Lasso,Logistic Regression,0.95,0.942,0.6353182910443254,0.13448727909967034,1000 +Lasso,Random Forest,0.9,0.901,0.732502989221806,0.17916410379672257,1000 +Lasso,Random Forest,0.95,0.953,0.8728311467467906,0.17916410379672257,1000 +Random Forest,Logistic Regression,0.9,0.872,0.5516888939848164,0.14967076659539238,1000 +Random Forest,Logistic Regression,0.95,0.923,0.6573778633938455,0.14967076659539238,1000 +Random Forest,Random Forest,0.9,0.907,0.7398909146073201,0.18007616950951408,1000 +Random Forest,Random Forest,0.95,0.956,0.8816344028170062,0.18007616950951408,1000 diff --git a/results/irm/irm_atte_coverage_metadata.csv b/results/irm/irm_atte_coverage_metadata.csv index 56ee932..400efb2 100644 --- a/results/irm/irm_atte_coverage_metadata.csv +++ b/results/irm/irm_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_coverage.py,2025-02-04 14:43:44,3635.82240152359,3.12.8 +0.10.dev0,irm_atte_coverage.py,2025-02-05 08:26:28,3586.8515162467957,3.12.8 From 3508bd87470d07a1ab0821729dee5d4c9716f9d5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 08:26:25 +0000 Subject: [PATCH 37/67] Update results from script: scripts/irm/irm_ate_coverage.py --- results/irm/irm_ate_coverage.csv | 12 ++++++------ results/irm/irm_ate_coverage_metadata.csv | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv index d1b75ad..6f7d66d 100644 --- a/results/irm/irm_ate_coverage.csv +++ b/results/irm/irm_ate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536215,1000 Lasso,Logistic Regression,0.95,0.935,0.5573839547492131,0.12336884150536215,1000 -Lasso,Random Forest,0.9,0.912,0.6033525996000427,0.1470798884283924,1000 -Lasso,Random Forest,0.95,0.961,0.7189389656430433,0.1470798884283924,1000 -Random Forest,Logistic Regression,0.9,0.796,0.5170183758598037,0.14999135577981693,1000 -Random Forest,Logistic Regression,0.95,0.879,0.6160653929484903,0.14999135577981693,1000 -Random Forest,Random Forest,0.9,0.898,0.6346634640451655,0.1547114295252091,1000 -Random Forest,Random Forest,0.95,0.954,0.7562481618120631,0.1547114295252091,1000 +Lasso,Random Forest,0.9,0.9,0.6010940693795692,0.1456325105684905,1000 +Lasso,Random Forest,0.95,0.958,0.7162477608953431,0.1456325105684905,1000 +Random Forest,Logistic Regression,0.9,0.8,0.5174327575265436,0.15025537456897348,1000 +Random Forest,Logistic Regression,0.95,0.884,0.616559159159268,0.15025537456897348,1000 +Random Forest,Random Forest,0.9,0.901,0.6280042419978804,0.15067906184239996,1000 +Random Forest,Random Forest,0.95,0.953,0.748313209325182,0.15067906184239996,1000 diff --git a/results/irm/irm_ate_coverage_metadata.csv b/results/irm/irm_ate_coverage_metadata.csv index 6966b69..4e6c67c 100644 --- a/results/irm/irm_ate_coverage_metadata.csv +++ b/results/irm/irm_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_coverage.py,2025-02-04 14:43:01,3590.5973885059357,3.12.8 +0.10.dev0,irm_ate_coverage.py,2025-02-05 08:26:24,3583.4047524929047,3.12.8 From 8cc4618dcd45dcee976bc45667ab4117933acf1a Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 08:54:24 +0000 Subject: [PATCH 38/67] Update results from script: scripts/irm/irm_cate_coverage.py --- results/irm/irm_cate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_cate_coverage_metadata.csv b/results/irm/irm_cate_coverage_metadata.csv index 28caa05..fc0c8c2 100644 --- a/results/irm/irm_cate_coverage_metadata.csv +++ b/results/irm/irm_cate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_cate_coverage.py,2025-02-04 15:10:16,5228.195962905884,3.12.8 +0.10.dev0,irm_cate_coverage.py,2025-02-05 08:54:23,5263.5848553180695,3.12.8 From 2748dd579b28060be049c740046b113309fda5b9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 08:56:57 +0000 Subject: [PATCH 39/67] Update results from script: scripts/irm/irm_apo_coverage.py --- results/irm/irm_apo_coverage_apo.csv | 48 +++++++++---------- results/irm/irm_apo_coverage_apos.csv | 16 +++---- .../irm/irm_apo_coverage_apos_contrast.csv | 16 +++---- results/irm/irm_apo_coverage_metadata.csv | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv index 69ee2bf..1660c79 100644 --- a/results/irm/irm_apo_coverage_apo.csv +++ b/results/irm/irm_apo_coverage_apo.csv @@ -1,25 +1,25 @@ Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.91,8.657690136121921,2.0774322102310396,1000 -LGBM,LGBM,0.0,0.95,0.966,10.316274091547035,2.0774322102310396,1000 -LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.221499001374493,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.221499001374493,1000 -LGBM,LGBM,2.0,0.9,0.894,37.49194764946096,9.569628503931227,1000 -LGBM,LGBM,2.0,0.95,0.953,44.67441108385784,9.569628503931227,1000 -LGBM,Logistic,0.0,0.9,0.903,5.625897101886533,1.3427898044782796,1000 -LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.3427898044782796,1000 -LGBM,Logistic,1.0,0.9,0.924,7.423300143143785,1.6931903229654406,1000 -LGBM,Logistic,1.0,0.95,0.97,8.84540769378873,1.6931903229654406,1000 -LGBM,Logistic,2.0,0.9,0.92,7.321275660150268,1.6627857804697925,1000 -LGBM,Logistic,2.0,0.95,0.965,8.723838024042964,1.6627857804697925,1000 -Linear,LGBM,0.0,0.9,0.902,5.498257423071024,1.31333599452238,1000 -Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.31333599452238,1000 -Linear,LGBM,1.0,0.9,0.947,10.700720020780512,2.129828839351603,1000 -Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.129828839351603,1000 -Linear,LGBM,2.0,0.9,0.934,7.513644049429104,1.6381350879200112,1000 -Linear,LGBM,2.0,0.95,0.967,8.953059097926168,1.6381350879200112,1000 -Linear,Logistic,0.0,0.9,0.904,5.335670092717667,1.2923995163085065,1000 -Linear,Logistic,0.0,0.95,0.954,6.357843058957276,1.2923995163085065,1000 -Linear,Logistic,1.0,0.9,0.908,5.417512107920403,1.2807077813820356,1000 -Linear,Logistic,1.0,0.95,0.959,6.455363835025866,1.2807077813820356,1000 -Linear,Logistic,2.0,0.9,0.908,5.366403391397197,1.2885877554305896,1000 -Linear,Logistic,2.0,0.95,0.956,6.3944640430685675,1.2885877554305896,1000 +LGBM,LGBM,0.0,0.9,0.911,8.657690136121921,2.076224364155097,1000 +LGBM,LGBM,0.0,0.95,0.965,10.316274091547035,2.076224364155097,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.204502862976703,1000 +LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.204502862976703,1000 +LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.589817842879643,1000 +LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.589817842879643,1000 +LGBM,Logistic,0.0,0.9,0.902,5.625897101886533,1.3373065397720765,1000 +LGBM,Logistic,0.0,0.95,0.954,6.7036698705295725,1.3373065397720765,1000 +LGBM,Logistic,1.0,0.9,0.923,7.423300143143785,1.6943068560206218,1000 +LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6943068560206218,1000 +LGBM,Logistic,2.0,0.9,0.92,7.321275660150268,1.6607553978369354,1000 +LGBM,Logistic,2.0,0.95,0.969,8.723838024042964,1.6607553978369354,1000 +Linear,LGBM,0.0,0.9,0.902,5.498257423071024,1.3081785987296133,1000 +Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.3081785987296133,1000 +Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.12789243928373,1000 +Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.12789243928373,1000 +Linear,LGBM,2.0,0.9,0.933,7.513644049429104,1.6350244914768306,1000 +Linear,LGBM,2.0,0.95,0.967,8.953059097926168,1.6350244914768306,1000 +Linear,Logistic,0.0,0.9,0.901,5.335670092717667,1.2867920134236264,1000 +Linear,Logistic,0.0,0.95,0.952,6.357843058957276,1.2867920134236264,1000 +Linear,Logistic,1.0,0.9,0.907,5.417512107920403,1.280123448265588,1000 +Linear,Logistic,1.0,0.95,0.955,6.455363835025866,1.280123448265588,1000 +Linear,Logistic,2.0,0.9,0.905,5.366403391397197,1.2819715669841822,1000 +Linear,Logistic,2.0,0.95,0.958,6.3944640430685675,1.2819715669841822,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv index b6d385a..67da75c 100644 --- a/results/irm/irm_apo_coverage_apos.csv +++ b/results/irm/irm_apo_coverage_apos.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9066666666666666,28.21021843292038,7.118050410548393,0.925,36.1943517986729,1000 -LGBM,LGBM,0.95,0.958,33.61454856442563,7.118050410548393,0.973,40.868020474452145,1000 -LGBM,Logistic,0.9,0.9136666666666666,6.789316986971467,1.57447223146738,0.921,8.394070597867524,1000 -LGBM,Logistic,0.95,0.9626666666666667,8.089970168806186,1.57447223146738,0.959,9.592284619044477,1000 -Linear,LGBM,0.9,0.9273333333333333,7.903418354805325,1.7307958124719784,0.937,9.850445774370156,1000 -Linear,LGBM,0.95,0.9686666666666667,9.41750382912841,1.7307958124719784,0.975,11.234392064837424,1000 -Linear,Logistic,0.9,0.9053333333333333,5.372532806955153,1.284757885951191,0.902,5.7964894175017925,1000 -Linear,Logistic,0.95,0.9553333333333334,6.4017676921854445,1.284757885951191,0.952,6.8182385960659,1000 +LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.1200033701167555,0.925,36.1943517986729,1000 +LGBM,LGBM,0.95,0.958,33.61454856442563,7.1200033701167555,0.973,40.868020474452145,1000 +LGBM,Logistic,0.9,0.9136666666666666,6.789316986971467,1.5717307211497997,0.926,8.394070597867524,1000 +LGBM,Logistic,0.95,0.9626666666666667,8.089970168806186,1.5717307211497997,0.96,9.592284619044477,1000 +Linear,LGBM,0.9,0.927,7.903418354805325,1.727110032102625,0.939,9.850445774370156,1000 +Linear,LGBM,0.95,0.9676666666666667,9.41750382912841,1.727110032102625,0.974,11.234392064837424,1000 +Linear,Logistic,0.9,0.904,5.372532806955153,1.2808273819157843,0.904,5.7964894175017925,1000 +Linear,Logistic,0.95,0.957,6.4017676921854445,1.2808273819157843,0.953,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv index 9f5e5b1..798c40f 100644 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.786633570698548,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.9485,45.13128131893863,9.786633570698548,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2678647082222205,0.926,6.774304315459575,1000 -LGBM,Logistic,0.95,0.962,6.821911652197591,1.2678647082222205,0.961,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.5066569199658195,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.9885,8.85452711998085,1.5066569199658195,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.874,1.1425883251377777,0.2971374642918569,0.871,1.3505222299078565,1000 -Linear,Logistic,0.95,0.934,1.3614779635902856,0.2971374642918569,0.917,1.5496710496635275,1000 +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.78908641206742,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.9485,45.13128131893863,9.78908641206742,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2672832290627098,0.927,6.774304315459575,1000 +LGBM,Logistic,0.95,0.9635,6.821911652197591,1.2672832290627098,0.963,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.5064083965850152,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.989,8.85452711998085,1.5064083965850152,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.874,1.1425883251377777,0.294452547740757,0.871,1.3505222299078565,1000 +Linear,Logistic,0.95,0.9355,1.3614779635902856,0.294452547740757,0.92,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv index ed6a0ab..f6e453a 100644 --- a/results/irm/irm_apo_coverage_metadata.csv +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_apo_coverage.py,2025-02-04 15:18:58,5762.698203086853,3.12.8 +0.10.dev0,irm_apo_coverage.py,2025-02-05 08:56:54,5427.111275672913,3.12.8 From 26b5c6bf5228fae685b345dad0ee0e7816d87fd1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 09:18:47 +0000 Subject: [PATCH 40/67] Update results from script: scripts/irm/irm_atte_sensitivity.py --- results/irm/irm_atte_sensitivity_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv index 108a6db..e77f3a1 100644 --- a/results/irm/irm_atte_sensitivity_metadata.csv +++ b/results/irm/irm_atte_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_sensitivity.py,2025-02-04 15:37:15,6846.028452157974,3.12.8 +0.10.dev0,irm_atte_sensitivity.py,2025-02-05 09:18:46,6721.090203523636,3.12.8 From 7b925fb9e7a48b9bc23680ba480d5c2d069bf042 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 09:48:14 +0000 Subject: [PATCH 41/67] Update results from script: scripts/irm/irm_apo_sensitivity.py --- results/irm/irm_apo_sensitivity.csv | 18 +++++++++--------- results/irm/irm_apo_sensitivity_metadata.csv | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/results/irm/irm_apo_sensitivity.csv b/results/irm/irm_apo_sensitivity.csv index 62a9182..9e163f8 100644 --- a/results/irm/irm_apo_sensitivity.csv +++ b/results/irm/irm_apo_sensitivity.csv @@ -1,9 +1,9 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition -LGBM,LGBM,0.9,0.09795918367346938,0.26979767912769265,0.1821217753924097,1.0,1.0,0.09243594786866559,0.040494673522359054,0.034931083457801146,0.3811385489487934,245 -LGBM,LGBM,0.95,0.30612244897959184,0.3214837633807111,0.1821217753924097,1.0,1.0,0.09243594786866559,0.025920252504384952,0.034931083457801146,0.3811385489487934,245 -LGBM,Logistic Regr.,0.9,0.30612244897959184,0.25780889388055256,0.1481921276548859,1.0,1.0,0.07187715697817153,0.024546744165995487,0.06294768349694302,0.35849335183083353,245 -LGBM,Logistic Regr.,0.95,0.563265306122449,0.30719824464654294,0.1481921276548859,1.0,1.0,0.07187715697817153,0.012953227730808654,0.06294768349694302,0.35849335183083353,245 -Linear Reg.,LGBM,0.9,0.10612244897959183,0.2695747491766244,0.18019285630080534,1.0,1.0,0.09421113740884988,0.04089415485221311,0.03462367597552903,0.37388485920180015,245 -Linear Reg.,LGBM,0.95,0.2979591836734694,0.32121812595984284,0.18019285630080534,1.0,1.0,0.09421113740884988,0.026079753922994376,0.03462367597552903,0.37388485920180015,245 -Linear Reg.,Logistic Regr.,0.9,0.8857142857142857,0.2592552019315289,0.08773320288603256,1.0,1.0,0.044089226784792754,0.004228149001636824,0.11817382692543607,0.2931817832302881,245 -Linear Reg.,Logistic Regr.,0.95,0.9836734693877551,0.30892162698526066,0.08773320288603256,1.0,1.0,0.044089226784792754,0.0008049142525428561,0.11817382692543607,0.2931817832302881,245 +Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),CI Bound Length,repetition +LGBM,LGBM,0.9,0.07,0.2698259981639014,0.18382710833907953,1.0,1.0,0.0934618367601546,0.041505653337845755,0.03227697703508104,0.382441637586196,0.6083608232357008,100 +LGBM,LGBM,0.95,0.26,0.32151750759365294,0.18382710833907953,1.0,1.0,0.0934618367601546,0.026750555252366862,0.03227697703508104,0.382441637586196,0.66821374482021,100 +LGBM,Logistic Regr.,0.9,0.29,0.25790921955622786,0.15053371103507743,1.0,1.0,0.07299208919084316,0.02554140209640269,0.06088935933492399,0.3607952086437938,0.6215409883410642,100 +LGBM,Logistic Regr.,0.95,0.57,0.3073177900625157,0.15053371103507743,1.0,1.0,0.07299208919084316,0.01351184068966981,0.06088935933492399,0.3607952086437938,0.6785267962048729,100 +Linear Reg.,LGBM,0.9,0.08,0.26913194911070604,0.18173453246134896,1.0,1.0,0.09477056557669242,0.04135002511440721,0.03270793263350985,0.3758047001813371,0.6010521192150566,100 +Linear Reg.,LGBM,0.95,0.24,0.32069049713784215,0.18173453246134896,1.0,1.0,0.09477056557669242,0.026630284952792325,0.03270793263350985,0.3758047001813371,0.661409651461861,100 +Linear Reg.,Logistic Regr.,0.9,0.86,0.2593372579643025,0.09071141042929678,1.0,1.0,0.04555739916551309,0.004802271604119686,0.11535866633125197,0.29596134576109095,0.6134462459742914,100 +Linear Reg.,Logistic Regr.,0.95,0.97,0.30901940277899403,0.09071141042929678,1.0,1.0,0.04555739916551309,0.0010804388811594201,0.11535866633125197,0.29596134576109095,0.6707462246938475,100 diff --git a/results/irm/irm_apo_sensitivity_metadata.csv b/results/irm/irm_apo_sensitivity_metadata.csv index b50c486..1a5af4a 100644 --- a/results/irm/irm_apo_sensitivity_metadata.csv +++ b/results/irm/irm_apo_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Sensitivity Errors -0.10.dev0,irm_apo_sensitivity.py,2025-02-04 19:14:02,19864.97425556183,3.12.8,0 +0.10.dev0,irm_apo_sensitivity.py,2025-02-05 09:48:13,8501.513749599457,3.12.8,0 From 62dd147a26aaf195b45ad4f2c38951e3741c7c25 Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Wed, 5 Feb 2025 10:58:44 +0100 Subject: [PATCH 42/67] increase n_obs --- scripts/irm/irm_apo_sensitivity.py | 2 +- scripts/irm/irm_ate_sensitivity.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/irm/irm_apo_sensitivity.py b/scripts/irm/irm_apo_sensitivity.py index e61b1e2..e71596a 100644 --- a/scripts/irm/irm_apo_sensitivity.py +++ b/scripts/irm/irm_apo_sensitivity.py @@ -15,7 +15,7 @@ max_runtime = 5.5 * 3600 # 5.5 hours in seconds # DGP pars -n_obs = 5000 +n_obs = 20000 theta = 5.0 trimming_threshold = 0.05 diff --git a/scripts/irm/irm_ate_sensitivity.py b/scripts/irm/irm_ate_sensitivity.py index 7b48ae2..187285b 100644 --- a/scripts/irm/irm_ate_sensitivity.py +++ b/scripts/irm/irm_ate_sensitivity.py @@ -15,7 +15,7 @@ max_runtime = 5.5 * 3600 # 5.5 hours in seconds # DGP pars -n_obs = 5000 +n_obs = 20000 theta = 5.0 trimming_threshold = 0.05 From fdd4db7e279bd41b82aaf460f3b66d0769c1bbef Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 10:35:01 +0000 Subject: [PATCH 43/67] Update results from script: scripts/irm/irm_ate_sensitivity.py --- results/irm/irm_ate_sensitivity.csv | 16 ++++++++-------- results/irm/irm_ate_sensitivity_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_ate_sensitivity.csv b/results/irm/irm_ate_sensitivity.csv index 3f47f2c..9688fed 100644 --- a/results/irm/irm_ate_sensitivity.csv +++ b/results/irm/irm_ate_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),CI Bound Length,repetition -LGBM,LGBM,0.9,0.08,0.2670216845863586,0.17940963620485714,0.96,1.0,0.1241372280116238,0.05432610289837227,0.041818765832749474,0.32254502881613434,0.4945097406195433,100 -LGBM,LGBM,0.95,0.35,0.3181759618638197,0.17940963620485714,1.0,1.0,0.1241372280116238,0.03439763897795268,0.041818765832749474,0.32254502881613434,0.5535425909510789,100 -LGBM,Logistic Regr.,0.9,0.3,0.2581285280201102,0.1489250720013938,1.0,1.0,0.10039281695402444,0.034610324016033665,0.027819425637511053,0.29775753820387224,0.4988408323876356,100 -LGBM,Logistic Regr.,0.95,0.56,0.3075791122152416,0.1489250720013938,1.0,1.0,0.10039281695402444,0.017870308168433494,0.027819425637511053,0.29775753820387224,0.5558714046518355,100 -Linear Reg.,LGBM,0.9,0.09,0.2680454420037482,0.1789353408854105,0.97,1.0,0.12697023327157428,0.055500063642588984,0.04546847329030372,0.31838417542283604,0.4879323625959303,100 -Linear Reg.,LGBM,0.95,0.29,0.3193958440673857,0.1789353408854105,0.99,1.0,0.12697023327157428,0.03516077097784765,0.04546847329030372,0.31838417542283604,0.5471907936286992,100 -Linear Reg.,Logistic Regr.,0.9,0.88,0.2593925192155475,0.09058684260970791,1.0,1.0,0.0636454347862647,0.00654738335028408,0.05695175811582821,0.2354964421984655,0.4929868678265177,100 -Linear Reg.,Logistic Regr.,0.95,0.97,0.3090852506212617,0.09058684260970791,1.0,1.0,0.0636454347862647,0.0014769375799795156,0.05695175811582821,0.2354964421984655,0.5502941581407065,100 +LGBM,LGBM,0.9,0.0,0.1290670759744328,0.16700188836291868,0.97,1.0,0.11516972420151037,0.08199881099554834,0.02323389657437148,0.3116508618137395,0.38989882617146443,100 +LGBM,LGBM,0.95,0.0,0.1537929067698414,0.16700188836291868,1.0,1.0,0.11516972420151037,0.07236261547894074,0.02323389657437148,0.3116508618137395,0.41841777757435944,100 +LGBM,Logistic Regr.,0.9,0.0,0.12799554257877943,0.1521625357301867,1.0,1.0,0.10627830296996256,0.07276073501676483,0.014191719568706143,0.2957015551702169,0.38682853093364633,100 +LGBM,Logistic Regr.,0.95,0.0,0.15251609597689259,0.1521625357301867,1.0,1.0,0.10627830296996256,0.06302810491287761,0.014191719568706143,0.2957015551702169,0.41510640937704496,100 +Linear Reg.,LGBM,0.9,0.0,0.13046535997228081,0.16726605997683605,0.97,1.0,0.1139630025296028,0.08085153079077306,0.02209160244564357,0.31378717139235235,0.3947340873761868,100 +Linear Reg.,LGBM,0.95,0.0,0.15545906491974315,0.16726605997683605,1.0,1.0,0.1139630025296028,0.0712277815883182,0.02209160244564357,0.31378717139235235,0.4235623179451715,100 +Linear Reg.,Logistic Regr.,0.9,0.08,0.12933949008724985,0.08765035815698355,1.0,1.0,0.06183424333715742,0.026740494950661556,0.057894323058536486,0.2331950393725036,0.391886816241763,100 +Linear Reg.,Logistic Regr.,0.95,0.26,0.1541175082062569,0.08765035815698355,1.0,1.0,0.06183424333715742,0.016877272580948677,0.057894323058536486,0.2331950393725036,0.4204614937955592,100 diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv index 23018f0..f511b4e 100644 --- a/results/irm/irm_ate_sensitivity_metadata.csv +++ b/results/irm/irm_ate_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_sensitivity.py,2025-02-05 07:49:10,1345.4034423828125,3.12.8 +0.10.dev0,irm_ate_sensitivity.py,2025-02-05 10:35:00,2050.7990169525146,3.12.8 From 91dea36778f17f070f5212feed4d925b8b6467fb Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 10:34:20 +0000 Subject: [PATCH 44/67] Update results from script: scripts/irm/irm_gate_coverage.py --- results/irm/irm_gate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_gate_coverage_metadata.csv b/results/irm/irm_gate_coverage_metadata.csv index 6610d3a..8298bbb 100644 --- a/results/irm/irm_gate_coverage_metadata.csv +++ b/results/irm/irm_gate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_gate_coverage.py,2025-02-05 08:00:49,2046.5704367160797,3.12.8 +0.10.dev0,irm_gate_coverage.py,2025-02-05 10:34:19,2018.1910226345062,3.12.8 From aaf273ccd0ede4559ed06432cd0e313649c6e519 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 11:00:37 +0000 Subject: [PATCH 45/67] Update results from script: scripts/irm/irm_ate_coverage.py --- results/irm/irm_ate_coverage.csv | 12 ++++++------ results/irm/irm_ate_coverage_metadata.csv | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv index 6f7d66d..64efd75 100644 --- a/results/irm/irm_ate_coverage.csv +++ b/results/irm/irm_ate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536215,1000 Lasso,Logistic Regression,0.95,0.935,0.5573839547492131,0.12336884150536215,1000 -Lasso,Random Forest,0.9,0.9,0.6010940693795692,0.1456325105684905,1000 -Lasso,Random Forest,0.95,0.958,0.7162477608953431,0.1456325105684905,1000 -Random Forest,Logistic Regression,0.9,0.8,0.5174327575265436,0.15025537456897348,1000 -Random Forest,Logistic Regression,0.95,0.884,0.616559159159268,0.15025537456897348,1000 -Random Forest,Random Forest,0.9,0.901,0.6280042419978804,0.15067906184239996,1000 -Random Forest,Random Forest,0.95,0.953,0.748313209325182,0.15067906184239996,1000 +Lasso,Random Forest,0.9,0.909,0.6011862689821161,0.14561117830316445,1000 +Lasso,Random Forest,0.95,0.957,0.7163576234979603,0.14561117830316445,1000 +Random Forest,Logistic Regression,0.9,0.802,0.5171011511026118,0.14962901688095656,1000 +Random Forest,Logistic Regression,0.95,0.881,0.6161640257338379,0.14962901688095656,1000 +Random Forest,Random Forest,0.9,0.897,0.628287514879127,0.14794529970360057,1000 +Random Forest,Random Forest,0.95,0.95,0.7486507497822434,0.14794529970360057,1000 diff --git a/results/irm/irm_ate_coverage_metadata.csv b/results/irm/irm_ate_coverage_metadata.csv index 4e6c67c..47f9d21 100644 --- a/results/irm/irm_ate_coverage_metadata.csv +++ b/results/irm/irm_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_coverage.py,2025-02-05 08:26:24,3583.4047524929047,3.12.8 +0.10.dev0,irm_ate_coverage.py,2025-02-05 11:00:35,3592.42094039917,3.12.8 From a825e3ee5cdea9c38637c218faea87203663f657 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 11:00:37 +0000 Subject: [PATCH 46/67] Update results from script: scripts/irm/irm_atte_coverage.py --- results/irm/irm_atte_coverage.csv | 16 ++++++++-------- results/irm/irm_atte_coverage_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv index 83a5c9a..fabfeca 100644 --- a/results/irm/irm_atte_coverage.csv +++ b/results/irm/irm_atte_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.887,0.5331759172799809,0.13448727909967034,1000 -Lasso,Logistic Regression,0.95,0.942,0.6353182910443254,0.13448727909967034,1000 -Lasso,Random Forest,0.9,0.901,0.732502989221806,0.17916410379672257,1000 -Lasso,Random Forest,0.95,0.953,0.8728311467467906,0.17916410379672257,1000 -Random Forest,Logistic Regression,0.9,0.872,0.5516888939848164,0.14967076659539238,1000 -Random Forest,Logistic Regression,0.95,0.923,0.6573778633938455,0.14967076659539238,1000 -Random Forest,Random Forest,0.9,0.907,0.7398909146073201,0.18007616950951408,1000 -Random Forest,Random Forest,0.95,0.956,0.8816344028170062,0.18007616950951408,1000 +Lasso,Logistic Regression,0.9,0.888,0.5331759172799809,0.13457507541575686,1000 +Lasso,Logistic Regression,0.95,0.94,0.6353182910443254,0.13457507541575686,1000 +Lasso,Random Forest,0.9,0.899,0.7392127383996425,0.18393194111981076,1000 +Lasso,Random Forest,0.95,0.952,0.8808263060232,0.18393194111981076,1000 +Random Forest,Logistic Regression,0.9,0.87,0.5504363620848345,0.1495150196073516,1000 +Random Forest,Logistic Regression,0.95,0.921,0.6558853795805581,0.1495150196073516,1000 +Random Forest,Random Forest,0.9,0.906,0.7427543098020616,0.1818807384731993,1000 +Random Forest,Random Forest,0.95,0.947,0.8850463486359177,0.1818807384731993,1000 diff --git a/results/irm/irm_atte_coverage_metadata.csv b/results/irm/irm_atte_coverage_metadata.csv index 400efb2..f9815c4 100644 --- a/results/irm/irm_atte_coverage_metadata.csv +++ b/results/irm/irm_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_coverage.py,2025-02-05 08:26:28,3586.8515162467957,3.12.8 +0.10.dev0,irm_atte_coverage.py,2025-02-05 11:00:36,3586.9249584674835,3.12.8 From cabd0974be6d61b9f53e1783ffa7c890fdb6e2c8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 11:29:17 +0000 Subject: [PATCH 47/67] Update results from script: scripts/irm/irm_cate_coverage.py --- results/irm/irm_cate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_cate_coverage_metadata.csv b/results/irm/irm_cate_coverage_metadata.csv index fc0c8c2..435bf07 100644 --- a/results/irm/irm_cate_coverage_metadata.csv +++ b/results/irm/irm_cate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_cate_coverage.py,2025-02-05 08:54:23,5263.5848553180695,3.12.8 +0.10.dev0,irm_cate_coverage.py,2025-02-05 11:29:16,5305.505748510361,3.12.8 From 2836bf9e35aa083f3857e20cca5da9dc098c17e3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 11:36:40 +0000 Subject: [PATCH 48/67] Update results from script: scripts/irm/irm_apo_coverage.py --- results/irm/irm_apo_coverage_apo.csv | 48 +++++++++---------- results/irm/irm_apo_coverage_apos.csv | 16 +++---- .../irm/irm_apo_coverage_apos_contrast.csv | 16 +++---- results/irm/irm_apo_coverage_metadata.csv | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv index 1660c79..ac3c518 100644 --- a/results/irm/irm_apo_coverage_apo.csv +++ b/results/irm/irm_apo_coverage_apo.csv @@ -1,25 +1,25 @@ Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.911,8.657690136121921,2.076224364155097,1000 -LGBM,LGBM,0.0,0.95,0.965,10.316274091547035,2.076224364155097,1000 -LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.204502862976703,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.204502862976703,1000 -LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.589817842879643,1000 -LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.589817842879643,1000 -LGBM,Logistic,0.0,0.9,0.902,5.625897101886533,1.3373065397720765,1000 -LGBM,Logistic,0.0,0.95,0.954,6.7036698705295725,1.3373065397720765,1000 -LGBM,Logistic,1.0,0.9,0.923,7.423300143143785,1.6943068560206218,1000 -LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6943068560206218,1000 -LGBM,Logistic,2.0,0.9,0.92,7.321275660150268,1.6607553978369354,1000 -LGBM,Logistic,2.0,0.95,0.969,8.723838024042964,1.6607553978369354,1000 -Linear,LGBM,0.0,0.9,0.902,5.498257423071024,1.3081785987296133,1000 -Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.3081785987296133,1000 -Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.12789243928373,1000 -Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.12789243928373,1000 -Linear,LGBM,2.0,0.9,0.933,7.513644049429104,1.6350244914768306,1000 -Linear,LGBM,2.0,0.95,0.967,8.953059097926168,1.6350244914768306,1000 -Linear,Logistic,0.0,0.9,0.901,5.335670092717667,1.2867920134236264,1000 -Linear,Logistic,0.0,0.95,0.952,6.357843058957276,1.2867920134236264,1000 -Linear,Logistic,1.0,0.9,0.907,5.417512107920403,1.280123448265588,1000 -Linear,Logistic,1.0,0.95,0.955,6.455363835025866,1.280123448265588,1000 -Linear,Logistic,2.0,0.9,0.905,5.366403391397197,1.2819715669841822,1000 -Linear,Logistic,2.0,0.95,0.958,6.3944640430685675,1.2819715669841822,1000 +LGBM,LGBM,0.0,0.9,0.911,8.657690136121921,2.076180844739683,1000 +LGBM,LGBM,0.0,0.95,0.964,10.316274091547035,2.076180844739683,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.19612827552887,1000 +LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.19612827552887,1000 +LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.591072626302008,1000 +LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.591072626302008,1000 +LGBM,Logistic,0.0,0.9,0.902,5.625897101886533,1.3370808920668427,1000 +LGBM,Logistic,0.0,0.95,0.954,6.7036698705295725,1.3370808920668427,1000 +LGBM,Logistic,1.0,0.9,0.922,7.423300143143785,1.695105011307578,1000 +LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.695105011307578,1000 +LGBM,Logistic,2.0,0.9,0.919,7.321275660150268,1.6607072459373193,1000 +LGBM,Logistic,2.0,0.95,0.969,8.723838024042964,1.6607072459373193,1000 +Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.307939241944836,1000 +Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.307939241944836,1000 +Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.1272526733684773,1000 +Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.1272526733684773,1000 +Linear,LGBM,2.0,0.9,0.933,7.513644049429104,1.634906855530984,1000 +Linear,LGBM,2.0,0.95,0.967,8.953059097926168,1.634906855530984,1000 +Linear,Logistic,0.0,0.9,0.9,5.335670092717667,1.286530896931142,1000 +Linear,Logistic,0.0,0.95,0.952,6.357843058957276,1.286530896931142,1000 +Linear,Logistic,1.0,0.9,0.905,5.417512107920403,1.2801790487143856,1000 +Linear,Logistic,1.0,0.95,0.955,6.455363835025866,1.2801790487143856,1000 +Linear,Logistic,2.0,0.9,0.905,5.366403391397197,1.2816343439394218,1000 +Linear,Logistic,2.0,0.95,0.959,6.3944640430685675,1.2816343439394218,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv index 67da75c..d9e3474 100644 --- a/results/irm/irm_apo_coverage_apos.csv +++ b/results/irm/irm_apo_coverage_apos.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.1200033701167555,0.925,36.1943517986729,1000 -LGBM,LGBM,0.95,0.958,33.61454856442563,7.1200033701167555,0.973,40.868020474452145,1000 -LGBM,Logistic,0.9,0.9136666666666666,6.789316986971467,1.5717307211497997,0.926,8.394070597867524,1000 -LGBM,Logistic,0.95,0.9626666666666667,8.089970168806186,1.5717307211497997,0.96,9.592284619044477,1000 -Linear,LGBM,0.9,0.927,7.903418354805325,1.727110032102625,0.939,9.850445774370156,1000 -Linear,LGBM,0.95,0.9676666666666667,9.41750382912841,1.727110032102625,0.974,11.234392064837424,1000 -Linear,Logistic,0.9,0.904,5.372532806955153,1.2808273819157843,0.904,5.7964894175017925,1000 -Linear,Logistic,0.95,0.957,6.4017676921854445,1.2808273819157843,0.953,6.8182385960659,1000 +LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.118007184946538,0.925,36.1943517986729,1000 +LGBM,LGBM,0.95,0.958,33.61454856442563,7.118007184946538,0.973,40.868020474452145,1000 +LGBM,Logistic,0.9,0.914,6.789316986971467,1.5717062051967705,0.925,8.394070597867524,1000 +LGBM,Logistic,0.95,0.9626666666666667,8.089970168806186,1.5717062051967705,0.959,9.592284619044477,1000 +Linear,LGBM,0.9,0.927,7.903418354805325,1.726691917066493,0.939,9.850445774370156,1000 +Linear,LGBM,0.95,0.9676666666666667,9.41750382912841,1.726691917066493,0.974,11.234392064837424,1000 +Linear,Logistic,0.9,0.9036666666666666,5.372532806955153,1.2807512605469862,0.904,5.7964894175017925,1000 +Linear,Logistic,0.95,0.9566666666666667,6.4017676921854445,1.2807512605469862,0.953,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv index 798c40f..4c15ffb 100644 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.78908641206742,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.9485,45.13128131893863,9.78908641206742,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2672832290627098,0.927,6.774304315459575,1000 -LGBM,Logistic,0.95,0.9635,6.821911652197591,1.2672832290627098,0.963,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.5064083965850152,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.989,8.85452711998085,1.5064083965850152,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.874,1.1425883251377777,0.294452547740757,0.871,1.3505222299078565,1000 -Linear,Logistic,0.95,0.9355,1.3614779635902856,0.294452547740757,0.92,1.5496710496635275,1000 +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.786076826228053,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.9485,45.13128131893863,9.786076826228053,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.9275,5.725128733165455,1.267970031375751,0.926,6.774304315459575,1000 +LGBM,Logistic,0.95,0.962,6.821911652197591,1.267970031375751,0.961,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.5066916649143378,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.9885,8.85452711998085,1.5066916649143378,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.8735,1.1425883251377777,0.29750421186738246,0.87,1.3505222299078565,1000 +Linear,Logistic,0.95,0.934,1.3614779635902856,0.29750421186738246,0.916,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv index f6e453a..53759d8 100644 --- a/results/irm/irm_apo_coverage_metadata.csv +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_apo_coverage.py,2025-02-05 08:56:54,5427.111275672913,3.12.8 +0.10.dev0,irm_apo_coverage.py,2025-02-05 11:36:36,5752.12405705452,3.12.8 From 9652319042216e252ae57c97704ab418c6498193 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 11:51:32 +0000 Subject: [PATCH 49/67] Update results from script: scripts/irm/irm_atte_sensitivity.py --- results/irm/irm_atte_sensitivity_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv index e77f3a1..07d2b10 100644 --- a/results/irm/irm_atte_sensitivity_metadata.csv +++ b/results/irm/irm_atte_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_sensitivity.py,2025-02-05 09:18:46,6721.090203523636,3.12.8 +0.10.dev0,irm_atte_sensitivity.py,2025-02-05 11:51:31,6661.692620754242,3.12.8 From 23632ae5aa27e76f793d059afc30e4217ad9d3d3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 5 Feb 2025 13:22:14 +0000 Subject: [PATCH 50/67] Update results from script: scripts/irm/irm_apo_sensitivity.py --- results/irm/irm_apo_sensitivity.csv | 16 ++++++++-------- results/irm/irm_apo_sensitivity_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_apo_sensitivity.csv b/results/irm/irm_apo_sensitivity.csv index 9e163f8..7afbafa 100644 --- a/results/irm/irm_apo_sensitivity.csv +++ b/results/irm/irm_apo_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),CI Bound Length,repetition -LGBM,LGBM,0.9,0.07,0.2698259981639014,0.18382710833907953,1.0,1.0,0.0934618367601546,0.041505653337845755,0.03227697703508104,0.382441637586196,0.6083608232357008,100 -LGBM,LGBM,0.95,0.26,0.32151750759365294,0.18382710833907953,1.0,1.0,0.0934618367601546,0.026750555252366862,0.03227697703508104,0.382441637586196,0.66821374482021,100 -LGBM,Logistic Regr.,0.9,0.29,0.25790921955622786,0.15053371103507743,1.0,1.0,0.07299208919084316,0.02554140209640269,0.06088935933492399,0.3607952086437938,0.6215409883410642,100 -LGBM,Logistic Regr.,0.95,0.57,0.3073177900625157,0.15053371103507743,1.0,1.0,0.07299208919084316,0.01351184068966981,0.06088935933492399,0.3607952086437938,0.6785267962048729,100 -Linear Reg.,LGBM,0.9,0.08,0.26913194911070604,0.18173453246134896,1.0,1.0,0.09477056557669242,0.04135002511440721,0.03270793263350985,0.3758047001813371,0.6010521192150566,100 -Linear Reg.,LGBM,0.95,0.24,0.32069049713784215,0.18173453246134896,1.0,1.0,0.09477056557669242,0.026630284952792325,0.03270793263350985,0.3758047001813371,0.661409651461861,100 -Linear Reg.,Logistic Regr.,0.9,0.86,0.2593372579643025,0.09071141042929678,1.0,1.0,0.04555739916551309,0.004802271604119686,0.11535866633125197,0.29596134576109095,0.6134462459742914,100 -Linear Reg.,Logistic Regr.,0.95,0.97,0.30901940277899403,0.09071141042929678,1.0,1.0,0.04555739916551309,0.0010804388811594201,0.11535866633125197,0.29596134576109095,0.6707462246938475,100 +LGBM,LGBM,0.9,0.0,0.1292540820255691,0.1677439500916218,1.0,1.0,0.08334212448404087,0.059113043397617415,0.036836009396182916,0.3721331068951957,0.5095597104015827,100 +LGBM,LGBM,0.95,0.0,0.1540157382237242,0.1677439500916218,1.0,1.0,0.08334212448404087,0.05212249622605954,0.036836009396182916,0.3721331068951957,0.5381298360143767,100 +LGBM,Logistic Regr.,0.9,0.0,0.12798017567553704,0.15251467338033609,1.0,1.0,0.07659421125394784,0.05222417352921675,0.05042841323362346,0.35545775999429563,0.5056319268530703,100 +LGBM,Logistic Regr.,0.95,0.0,0.15249778518229332,0.15251467338033609,1.0,1.0,0.07659421125394784,0.04519737142215315,0.05042841323362346,0.35545775999429563,0.5339084620178494,100 +Linear Reg.,LGBM,0.9,0.0,0.13050148401216977,0.16740934350762515,1.0,1.0,0.08209259956920208,0.05796766242007333,0.039830831956169446,0.37464951897141974,0.5162334967376828,100 +Linear Reg.,LGBM,0.95,0.0,0.15550210936819642,0.16740934350762515,1.0,1.0,0.08209259956920208,0.05100237961568155,0.039830831956169446,0.37464951897141974,0.5450790996937619,100 +Linear Reg.,Logistic Regr.,0.9,0.08,0.1293446408364081,0.08762861257786415,1.0,1.0,0.044141171731517534,0.018998467144475298,0.11817849827800302,0.2934357234337313,0.5124233919151969,100 +Linear Reg.,Logistic Regr.,0.95,0.26,0.15412364570243178,0.08762861257786415,1.0,1.0,0.044141171731517534,0.011970031722965948,0.11817849827800302,0.2934357234337313,0.5410013909035332,100 diff --git a/results/irm/irm_apo_sensitivity_metadata.csv b/results/irm/irm_apo_sensitivity_metadata.csv index 1a5af4a..0901455 100644 --- a/results/irm/irm_apo_sensitivity_metadata.csv +++ b/results/irm/irm_apo_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Sensitivity Errors -0.10.dev0,irm_apo_sensitivity.py,2025-02-05 09:48:13,8501.513749599457,3.12.8,0 +0.10.dev0,irm_apo_sensitivity.py,2025-02-05 13:22:13,12080.8105738163,3.12.8,0 From 1df04ab7ffe9b5797273f05d4784469d43827b97 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 09:10:45 +0000 Subject: [PATCH 51/67] Update results from script: scripts/irm/irm_ate_sensitivity.py --- results/irm/irm_ate_sensitivity_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv index f511b4e..844a071 100644 --- a/results/irm/irm_ate_sensitivity_metadata.csv +++ b/results/irm/irm_ate_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_sensitivity.py,2025-02-05 10:35:00,2050.7990169525146,3.12.8 +0.10.dev0,irm_ate_sensitivity.py,2025-02-17 09:10:45,2014.9821014404297,3.12.9 From dea78155ba8a31acc39914a8a7f3359031c44ccd Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 09:11:45 +0000 Subject: [PATCH 52/67] Update results from script: scripts/irm/irm_gate_coverage.py --- results/irm/irm_gate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_gate_coverage_metadata.csv b/results/irm/irm_gate_coverage_metadata.csv index 8298bbb..eb7a49f 100644 --- a/results/irm/irm_gate_coverage_metadata.csv +++ b/results/irm/irm_gate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_gate_coverage.py,2025-02-05 10:34:19,2018.1910226345062,3.12.8 +0.10.dev0,irm_gate_coverage.py,2025-02-17 09:11:44,2069.9562945365906,3.12.9 From 80efd3fdb86aa7fb510378f032d3149c3116e66d Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 09:37:52 +0000 Subject: [PATCH 53/67] Update results from script: scripts/irm/irm_atte_coverage.py --- results/irm/irm_atte_coverage.csv | 16 ++++++++-------- results/irm/irm_atte_coverage_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv index fabfeca..2c8fe7a 100644 --- a/results/irm/irm_atte_coverage.csv +++ b/results/irm/irm_atte_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.888,0.5331759172799809,0.13457507541575686,1000 -Lasso,Logistic Regression,0.95,0.94,0.6353182910443254,0.13457507541575686,1000 -Lasso,Random Forest,0.9,0.899,0.7392127383996425,0.18393194111981076,1000 -Lasso,Random Forest,0.95,0.952,0.8808263060232,0.18393194111981076,1000 -Random Forest,Logistic Regression,0.9,0.87,0.5504363620848345,0.1495150196073516,1000 -Random Forest,Logistic Regression,0.95,0.921,0.6558853795805581,0.1495150196073516,1000 -Random Forest,Random Forest,0.9,0.906,0.7427543098020616,0.1818807384731993,1000 -Random Forest,Random Forest,0.95,0.947,0.8850463486359177,0.1818807384731993,1000 +Lasso,Logistic Regression,0.9,0.888,0.5331759172799809,0.13452354417969958,1000 +Lasso,Logistic Regression,0.95,0.941,0.6353182910443254,0.13452354417969958,1000 +Lasso,Random Forest,0.9,0.903,0.7299501387914156,0.17890617607036363,1000 +Lasso,Random Forest,0.95,0.951,0.8697892378379983,0.17890617607036363,1000 +Random Forest,Logistic Regression,0.9,0.873,0.5508681223214307,0.14919172056375843,1000 +Random Forest,Logistic Regression,0.95,0.92,0.6563998536345526,0.14919172056375843,1000 +Random Forest,Random Forest,0.9,0.902,0.7543996805167961,0.18403497407534655,1000 +Random Forest,Random Forest,0.95,0.954,0.8989226637155757,0.18403497407534655,1000 diff --git a/results/irm/irm_atte_coverage_metadata.csv b/results/irm/irm_atte_coverage_metadata.csv index f9815c4..95c4f1b 100644 --- a/results/irm/irm_atte_coverage_metadata.csv +++ b/results/irm/irm_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_coverage.py,2025-02-05 11:00:36,3586.9249584674835,3.12.8 +0.10.dev0,irm_atte_coverage.py,2025-02-17 09:37:51,3642.9659905433655,3.12.9 From 0b5647e97458b35c33819364f50a5724ea339491 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 09:37:00 +0000 Subject: [PATCH 54/67] Update results from script: scripts/irm/irm_ate_coverage.py --- results/irm/irm_ate_coverage.csv | 12 ++++++------ results/irm/irm_ate_coverage_metadata.csv | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv index 64efd75..999f92c 100644 --- a/results/irm/irm_ate_coverage.csv +++ b/results/irm/irm_ate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536215,1000 Lasso,Logistic Regression,0.95,0.935,0.5573839547492131,0.12336884150536215,1000 -Lasso,Random Forest,0.9,0.909,0.6011862689821161,0.14561117830316445,1000 -Lasso,Random Forest,0.95,0.957,0.7163576234979603,0.14561117830316445,1000 -Random Forest,Logistic Regression,0.9,0.802,0.5171011511026118,0.14962901688095656,1000 -Random Forest,Logistic Regression,0.95,0.881,0.6161640257338379,0.14962901688095656,1000 -Random Forest,Random Forest,0.9,0.897,0.628287514879127,0.14794529970360057,1000 -Random Forest,Random Forest,0.95,0.95,0.7486507497822434,0.14794529970360057,1000 +Lasso,Random Forest,0.9,0.918,0.6021088429949879,0.1471534184182516,1000 +Lasso,Random Forest,0.95,0.96,0.7174569382385991,0.1471534184182516,1000 +Random Forest,Logistic Regression,0.9,0.796,0.5169833951793216,0.15066786899795942,1000 +Random Forest,Logistic Regression,0.95,0.879,0.6160237108968011,0.15066786899795942,1000 +Random Forest,Random Forest,0.9,0.905,0.6288222586448227,0.1522621548395983,1000 +Random Forest,Random Forest,0.95,0.959,0.7492879362799036,0.1522621548395983,1000 diff --git a/results/irm/irm_ate_coverage_metadata.csv b/results/irm/irm_ate_coverage_metadata.csv index 47f9d21..983bb7b 100644 --- a/results/irm/irm_ate_coverage_metadata.csv +++ b/results/irm/irm_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_coverage.py,2025-02-05 11:00:35,3592.42094039917,3.12.8 +0.10.dev0,irm_ate_coverage.py,2025-02-17 09:36:59,3589.1906917095184,3.12.9 From 1ea1f2fa815dce622c7b8d9a146c3baabfeb7e69 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 10:04:34 +0000 Subject: [PATCH 55/67] Update results from script: scripts/irm/irm_cate_coverage.py --- results/irm/irm_cate_coverage_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_cate_coverage_metadata.csv b/results/irm/irm_cate_coverage_metadata.csv index 435bf07..1bff3f8 100644 --- a/results/irm/irm_cate_coverage_metadata.csv +++ b/results/irm/irm_cate_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_cate_coverage.py,2025-02-05 11:29:16,5305.505748510361,3.12.8 +0.10.dev0,irm_cate_coverage.py,2025-02-17 10:04:33,5245.124602556229,3.12.9 From 2f73587ac205a4c04c352fd464599d0b30218e6d Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 10:12:58 +0000 Subject: [PATCH 56/67] Update results from script: scripts/irm/irm_apo_coverage.py --- results/irm/irm_apo_coverage_apo.csv | 48 +++++++++---------- results/irm/irm_apo_coverage_apos.csv | 16 +++---- .../irm/irm_apo_coverage_apos_contrast.csv | 16 +++---- results/irm/irm_apo_coverage_metadata.csv | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv index ac3c518..ee5f3d1 100644 --- a/results/irm/irm_apo_coverage_apo.csv +++ b/results/irm/irm_apo_coverage_apo.csv @@ -1,25 +1,25 @@ Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.911,8.657690136121921,2.076180844739683,1000 -LGBM,LGBM,0.0,0.95,0.964,10.316274091547035,2.076180844739683,1000 -LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.19612827552887,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.19612827552887,1000 -LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.591072626302008,1000 -LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.591072626302008,1000 -LGBM,Logistic,0.0,0.9,0.902,5.625897101886533,1.3370808920668427,1000 -LGBM,Logistic,0.0,0.95,0.954,6.7036698705295725,1.3370808920668427,1000 -LGBM,Logistic,1.0,0.9,0.922,7.423300143143785,1.695105011307578,1000 -LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.695105011307578,1000 -LGBM,Logistic,2.0,0.9,0.919,7.321275660150268,1.6607072459373193,1000 -LGBM,Logistic,2.0,0.95,0.969,8.723838024042964,1.6607072459373193,1000 -Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.307939241944836,1000 -Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.307939241944836,1000 -Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.1272526733684773,1000 -Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.1272526733684773,1000 -Linear,LGBM,2.0,0.9,0.933,7.513644049429104,1.634906855530984,1000 -Linear,LGBM,2.0,0.95,0.967,8.953059097926168,1.634906855530984,1000 -Linear,Logistic,0.0,0.9,0.9,5.335670092717667,1.286530896931142,1000 -Linear,Logistic,0.0,0.95,0.952,6.357843058957276,1.286530896931142,1000 -Linear,Logistic,1.0,0.9,0.905,5.417512107920403,1.2801790487143856,1000 -Linear,Logistic,1.0,0.95,0.955,6.455363835025866,1.2801790487143856,1000 -Linear,Logistic,2.0,0.9,0.905,5.366403391397197,1.2816343439394218,1000 -Linear,Logistic,2.0,0.95,0.959,6.3944640430685675,1.2816343439394218,1000 +LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.075995364683431,1000 +LGBM,LGBM,0.0,0.95,0.963,10.316274091547035,2.075995364683431,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.193797216238146,1000 +LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.193797216238146,1000 +LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.599689794190313,1000 +LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.599689794190313,1000 +LGBM,Logistic,0.0,0.9,0.902,5.625897101886533,1.3358593836381196,1000 +LGBM,Logistic,0.0,0.95,0.956,6.7036698705295725,1.3358593836381196,1000 +LGBM,Logistic,1.0,0.9,0.92,7.423300143143785,1.6953709510387716,1000 +LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6953709510387716,1000 +LGBM,Logistic,2.0,0.9,0.918,7.321275660150268,1.660500602699947,1000 +LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.660500602699947,1000 +Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.3062893854489512,1000 +Linear,LGBM,0.0,0.95,0.948,6.551577812380716,1.3062893854489512,1000 +Linear,LGBM,1.0,0.9,0.95,10.700720020780512,2.127086970318636,1000 +Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.127086970318636,1000 +Linear,LGBM,2.0,0.9,0.932,7.513644049429104,1.634264519418015,1000 +Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.634264519418015,1000 +Linear,Logistic,0.0,0.9,0.902,5.335670092717667,1.284853808916775,1000 +Linear,Logistic,0.0,0.95,0.952,6.357843058957276,1.284853808916775,1000 +Linear,Logistic,1.0,0.9,0.905,5.417512107920403,1.2802654150082002,1000 +Linear,Logistic,1.0,0.95,0.954,6.455363835025866,1.2802654150082002,1000 +Linear,Logistic,2.0,0.9,0.907,5.366403391397197,1.2795787147265043,1000 +Linear,Logistic,2.0,0.95,0.96,6.3944640430685675,1.2795787147265043,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv index d9e3474..5f99bd7 100644 --- a/results/irm/irm_apo_coverage_apos.csv +++ b/results/irm/irm_apo_coverage_apos.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.118007184946538,0.925,36.1943517986729,1000 -LGBM,LGBM,0.95,0.958,33.61454856442563,7.118007184946538,0.973,40.868020474452145,1000 -LGBM,Logistic,0.9,0.914,6.789316986971467,1.5717062051967705,0.925,8.394070597867524,1000 -LGBM,Logistic,0.95,0.9626666666666667,8.089970168806186,1.5717062051967705,0.959,9.592284619044477,1000 -Linear,LGBM,0.9,0.927,7.903418354805325,1.726691917066493,0.939,9.850445774370156,1000 -Linear,LGBM,0.95,0.9676666666666667,9.41750382912841,1.726691917066493,0.974,11.234392064837424,1000 -Linear,Logistic,0.9,0.9036666666666666,5.372532806955153,1.2807512605469862,0.904,5.7964894175017925,1000 -Linear,Logistic,0.95,0.9566666666666667,6.4017676921854445,1.2807512605469862,0.953,6.8182385960659,1000 +LGBM,LGBM,0.9,0.9066666666666666,28.21021843292038,7.1204285014294015,0.924,36.1943517986729,1000 +LGBM,LGBM,0.95,0.9583333333333334,33.61454856442563,7.1204285014294015,0.974,40.868020474452145,1000 +LGBM,Logistic,0.9,0.917,6.789316986971467,1.5709695238934474,0.924,8.394070597867524,1000 +LGBM,Logistic,0.95,0.9633333333333334,8.089970168806186,1.5709695238934474,0.96,9.592284619044477,1000 +Linear,LGBM,0.9,0.926,7.903418354805325,1.7257798274842937,0.939,9.850445774370156,1000 +Linear,LGBM,0.95,0.9676666666666667,9.41750382912841,1.7257798274842937,0.974,11.234392064837424,1000 +Linear,Logistic,0.9,0.9043333333333333,5.372532806955153,1.2796203798007288,0.905,5.7964894175017925,1000 +Linear,Logistic,0.95,0.9573333333333334,6.4017676921854445,1.2796203798007288,0.953,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv index 4c15ffb..573b7ac 100644 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.786076826228053,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.9485,45.13128131893863,9.786076826228053,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.9275,5.725128733165455,1.267970031375751,0.926,6.774304315459575,1000 -LGBM,Logistic,0.95,0.962,6.821911652197591,1.267970031375751,0.961,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.5066916649143378,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.9885,8.85452711998085,1.5066916649143378,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.8735,1.1425883251377777,0.29750421186738246,0.87,1.3505222299078565,1000 -Linear,Logistic,0.95,0.934,1.3614779635902856,0.29750421186738246,0.916,1.5496710496635275,1000 +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.789191608123641,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.9485,45.13128131893863,9.789191608123641,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2672891385005909,0.927,6.774304315459575,1000 +LGBM,Logistic,0.95,0.9635,6.821911652197591,1.2672891385005909,0.963,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.506423849047913,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.989,8.85452711998085,1.506423849047913,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.873,1.1425883251377777,0.2945756816692818,0.87,1.3505222299078565,1000 +Linear,Logistic,0.95,0.935,1.3614779635902856,0.2945756816692818,0.92,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv index 53759d8..8eeb02f 100644 --- a/results/irm/irm_apo_coverage_metadata.csv +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_apo_coverage.py,2025-02-05 11:36:36,5752.12405705452,3.12.8 +0.10.dev0,irm_apo_coverage.py,2025-02-17 10:12:54,5756.271638393402,3.12.9 From cf0752ec7d3b883edcdfc06f3eb49ee981126049 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 10:32:48 +0000 Subject: [PATCH 57/67] Update results from script: scripts/irm/irm_atte_sensitivity.py --- results/irm/irm_atte_sensitivity_metadata.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv index 07d2b10..3d0e62f 100644 --- a/results/irm/irm_atte_sensitivity_metadata.csv +++ b/results/irm/irm_atte_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_sensitivity.py,2025-02-05 11:51:31,6661.692620754242,3.12.8 +0.10.dev0,irm_atte_sensitivity.py,2025-02-17 10:32:47,6933.687040805817,3.12.9 From 6678537bfbdd1933828989f6af5f3c972e604644 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 11:58:32 +0000 Subject: [PATCH 58/67] Update results from script: scripts/irm/irm_apo_sensitivity.py --- results/irm/irm_apo_sensitivity.csv | 16 ++++++++-------- results/irm/irm_apo_sensitivity_metadata.csv | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/results/irm/irm_apo_sensitivity.csv b/results/irm/irm_apo_sensitivity.csv index 7afbafa..adfa34e 100644 --- a/results/irm/irm_apo_sensitivity.csv +++ b/results/irm/irm_apo_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),CI Bound Length,repetition -LGBM,LGBM,0.9,0.0,0.1292540820255691,0.1677439500916218,1.0,1.0,0.08334212448404087,0.059113043397617415,0.036836009396182916,0.3721331068951957,0.5095597104015827,100 -LGBM,LGBM,0.95,0.0,0.1540157382237242,0.1677439500916218,1.0,1.0,0.08334212448404087,0.05212249622605954,0.036836009396182916,0.3721331068951957,0.5381298360143767,100 -LGBM,Logistic Regr.,0.9,0.0,0.12798017567553704,0.15251467338033609,1.0,1.0,0.07659421125394784,0.05222417352921675,0.05042841323362346,0.35545775999429563,0.5056319268530703,100 -LGBM,Logistic Regr.,0.95,0.0,0.15249778518229332,0.15251467338033609,1.0,1.0,0.07659421125394784,0.04519737142215315,0.05042841323362346,0.35545775999429563,0.5339084620178494,100 -Linear Reg.,LGBM,0.9,0.0,0.13050148401216977,0.16740934350762515,1.0,1.0,0.08209259956920208,0.05796766242007333,0.039830831956169446,0.37464951897141974,0.5162334967376828,100 -Linear Reg.,LGBM,0.95,0.0,0.15550210936819642,0.16740934350762515,1.0,1.0,0.08209259956920208,0.05100237961568155,0.039830831956169446,0.37464951897141974,0.5450790996937619,100 -Linear Reg.,Logistic Regr.,0.9,0.08,0.1293446408364081,0.08762861257786415,1.0,1.0,0.044141171731517534,0.018998467144475298,0.11817849827800302,0.2934357234337313,0.5124233919151969,100 -Linear Reg.,Logistic Regr.,0.95,0.26,0.15412364570243178,0.08762861257786415,1.0,1.0,0.044141171731517534,0.011970031722965948,0.11817849827800302,0.2934357234337313,0.5410013909035332,100 +LGBM,LGBM,0.9,0.0,0.1292540820255691,0.1677439500916218,0.94,1.0,0.11573872417116222,0.08252331475485856,0.024288344996646342,0.3122738965658806,0.3898116700153508,100 +LGBM,LGBM,0.95,0.0,0.1540157382237242,0.1677439500916218,0.99,1.0,0.11573872417116222,0.07287226227791722,0.024288344996646342,0.3122738965658806,0.418373398846942,100 +LGBM,Logistic Regr.,0.9,0.0,0.12798017567553704,0.15251467338033609,1.0,1.0,0.10652571495226121,0.07301481703198542,0.014645752061926958,0.29602520197592563,0.3867596062890008,100 +LGBM,Logistic Regr.,0.95,0.0,0.15249778518229332,0.15251467338033609,1.0,1.0,0.10652571495226121,0.06328412036942282,0.014645752061926958,0.29602520197592563,0.41503409907039024,100 +Linear Reg.,LGBM,0.9,0.0,0.13050148401216977,0.16740934350762515,0.97,1.0,0.11403669658454366,0.08093114285791209,0.022292080500950134,0.3139559124208978,0.3948171482587492,100 +Linear Reg.,LGBM,0.95,0.0,0.15550210936819642,0.16740934350762515,1.0,1.0,0.11403669658454366,0.07130818482813787,0.022292080500950134,0.3139559124208978,0.42365449174003855,100 +Linear Reg.,Logistic Regr.,0.9,0.08,0.1293446408364081,0.08762861257786415,1.0,1.0,0.06182185548164626,0.026723981525642357,0.057907195061308585,0.2331644202170369,0.39187310331025016,100 +Linear Reg.,Logistic Regr.,0.95,0.26,0.15412364570243178,0.08762861257786415,1.0,1.0,0.06182185548164626,0.016854287151815115,0.057907195061308585,0.2331644202170369,0.420448924509695,100 diff --git a/results/irm/irm_apo_sensitivity_metadata.csv b/results/irm/irm_apo_sensitivity_metadata.csv index 0901455..c11bbe8 100644 --- a/results/irm/irm_apo_sensitivity_metadata.csv +++ b/results/irm/irm_apo_sensitivity_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Sensitivity Errors -0.10.dev0,irm_apo_sensitivity.py,2025-02-05 13:22:13,12080.8105738163,3.12.8,0 +0.10.dev0,irm_apo_sensitivity.py,2025-02-17 11:58:31,12093.837664842606,3.12.9,0 From 5d36d0bf17a336e3641bc74aabfdb565050c4b3c Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Mon, 17 Feb 2025 13:36:05 +0100 Subject: [PATCH 59/67] set n_obs = 10000 for IRM & APO sensitivity, add n_obs and n_rep to metadata --- .github/workflows/apo_sim.yml | 2 +- scripts/did/did_cs_atte_coverage.py | 2 ++ scripts/irm/cvar_coverage.py | 2 ++ .../{irm_apo_sensitivity.py => irm_apo_ate_sensitivity.py} | 4 +++- scripts/irm/irm_apo_coverage.py | 2 ++ scripts/irm/irm_ate_coverage.py | 2 ++ scripts/irm/irm_ate_sensitivity.py | 4 +++- scripts/irm/irm_atte_coverage.py | 2 ++ scripts/irm/irm_atte_sensitivity.py | 6 ++++-- scripts/irm/irm_cate_coverage.py | 2 ++ scripts/irm/irm_gate_coverage.py | 2 ++ scripts/irm/lpq_coverage.py | 2 ++ scripts/irm/pq_coverage.py | 2 ++ scripts/irm/ssm_mar_ate_coverage.py | 2 ++ scripts/irm/ssm_nonignorable_ate_coverage.py | 2 ++ scripts/plm/pliv_late_coverage.py | 2 ++ scripts/plm/plr_ate_coverage.py | 2 ++ scripts/plm/plr_ate_sensitivity.py | 2 ++ scripts/plm/plr_cate_coverage.py | 2 ++ scripts/plm/plr_gate_coverage.py | 2 ++ scripts/rdd/rdd_sharp_coverage.py | 2 ++ 21 files changed, 45 insertions(+), 5 deletions(-) rename scripts/irm/{irm_apo_sensitivity.py => irm_apo_ate_sensitivity.py} (98%) diff --git a/.github/workflows/apo_sim.yml b/.github/workflows/apo_sim.yml index 5ffde19..fe5b12d 100644 --- a/.github/workflows/apo_sim.yml +++ b/.github/workflows/apo_sim.yml @@ -18,7 +18,7 @@ jobs: matrix: script: [ 'scripts/irm/irm_apo_coverage.py', - 'scripts/irm/irm_apo_sensitivity.py', + 'scripts/irm/irm_apo_ate_sensitivity.py', ] steps: diff --git a/scripts/did/did_cs_atte_coverage.py b/scripts/did/did_cs_atte_coverage.py index 3886ea8..e2d2796 100644 --- a/scripts/did/did_cs_atte_coverage.py +++ b/scripts/did/did_cs_atte_coverage.py @@ -125,6 +125,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/cvar_coverage.py b/scripts/irm/cvar_coverage.py index 4a166d9..4ab6e57 100644 --- a/scripts/irm/cvar_coverage.py +++ b/scripts/irm/cvar_coverage.py @@ -233,6 +233,8 @@ def dgp(n=200, p=5): 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/irm_apo_sensitivity.py b/scripts/irm/irm_apo_ate_sensitivity.py similarity index 98% rename from scripts/irm/irm_apo_sensitivity.py rename to scripts/irm/irm_apo_ate_sensitivity.py index e71596a..4abc7d3 100644 --- a/scripts/irm/irm_apo_sensitivity.py +++ b/scripts/irm/irm_apo_ate_sensitivity.py @@ -15,7 +15,7 @@ max_runtime = 5.5 * 3600 # 5.5 hours in seconds # DGP pars -n_obs = 20000 +n_obs = 10000 theta = 5.0 trimming_threshold = 0.05 @@ -177,6 +177,8 @@ 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], 'Sensitivity Errors': [sensitivity_err_count], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/irm_apo_coverage.py b/scripts/irm/irm_apo_coverage.py index 3f5f7ae..96f1ceb 100644 --- a/scripts/irm/irm_apo_coverage.py +++ b/scripts/irm/irm_apo_coverage.py @@ -218,6 +218,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/irm_ate_coverage.py b/scripts/irm/irm_ate_coverage.py index 82e7fa7..36a656d 100644 --- a/scripts/irm/irm_ate_coverage.py +++ b/scripts/irm/irm_ate_coverage.py @@ -105,6 +105,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/irm_ate_sensitivity.py b/scripts/irm/irm_ate_sensitivity.py index 187285b..31ddfe8 100644 --- a/scripts/irm/irm_ate_sensitivity.py +++ b/scripts/irm/irm_ate_sensitivity.py @@ -15,7 +15,7 @@ max_runtime = 5.5 * 3600 # 5.5 hours in seconds # DGP pars -n_obs = 20000 +n_obs = 10000 theta = 5.0 trimming_threshold = 0.05 @@ -155,6 +155,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/irm_atte_coverage.py b/scripts/irm/irm_atte_coverage.py index 8b84743..bc156c2 100644 --- a/scripts/irm/irm_atte_coverage.py +++ b/scripts/irm/irm_atte_coverage.py @@ -134,6 +134,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/irm_atte_sensitivity.py b/scripts/irm/irm_atte_sensitivity.py index aeda6b1..1412985 100644 --- a/scripts/irm/irm_atte_sensitivity.py +++ b/scripts/irm/irm_atte_sensitivity.py @@ -11,11 +11,11 @@ from doubleml.datasets import make_confounded_irm_data # Number of repetitions -n_rep = 500 +n_rep = 100 max_runtime = 5.5 * 3600 # 5.5 hours in seconds # DGP pars -n_obs = 5000 +n_obs = 10000 theta = 5.0 trimming_threshold = 0.05 @@ -152,6 +152,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/irm_cate_coverage.py b/scripts/irm/irm_cate_coverage.py index 9843de1..67d658e 100644 --- a/scripts/irm/irm_cate_coverage.py +++ b/scripts/irm/irm_cate_coverage.py @@ -120,6 +120,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/irm_gate_coverage.py b/scripts/irm/irm_gate_coverage.py index aafd915..3c09748 100644 --- a/scripts/irm/irm_gate_coverage.py +++ b/scripts/irm/irm_gate_coverage.py @@ -123,6 +123,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/lpq_coverage.py b/scripts/irm/lpq_coverage.py index ab1152a..a9bfb5a 100644 --- a/scripts/irm/lpq_coverage.py +++ b/scripts/irm/lpq_coverage.py @@ -250,6 +250,8 @@ def dgp(n=200, p=5): 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/pq_coverage.py b/scripts/irm/pq_coverage.py index 2a6a727..bcf9a79 100644 --- a/scripts/irm/pq_coverage.py +++ b/scripts/irm/pq_coverage.py @@ -229,6 +229,8 @@ def dgp(n=200, p=5): 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/ssm_mar_ate_coverage.py b/scripts/irm/ssm_mar_ate_coverage.py index 8aa0e56..a624732 100644 --- a/scripts/irm/ssm_mar_ate_coverage.py +++ b/scripts/irm/ssm_mar_ate_coverage.py @@ -117,6 +117,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/irm/ssm_nonignorable_ate_coverage.py b/scripts/irm/ssm_nonignorable_ate_coverage.py index ebbacfe..8435aa0 100644 --- a/scripts/irm/ssm_nonignorable_ate_coverage.py +++ b/scripts/irm/ssm_nonignorable_ate_coverage.py @@ -117,6 +117,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/plm/pliv_late_coverage.py b/scripts/plm/pliv_late_coverage.py index 6cc4329..9806d0e 100644 --- a/scripts/plm/pliv_late_coverage.py +++ b/scripts/plm/pliv_late_coverage.py @@ -129,6 +129,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/plm/plr_ate_coverage.py b/scripts/plm/plr_ate_coverage.py index ab715d7..2af49b5 100644 --- a/scripts/plm/plr_ate_coverage.py +++ b/scripts/plm/plr_ate_coverage.py @@ -121,6 +121,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/plm/plr_ate_sensitivity.py b/scripts/plm/plr_ate_sensitivity.py index 34e73dc..bc60bae 100644 --- a/scripts/plm/plr_ate_sensitivity.py +++ b/scripts/plm/plr_ate_sensitivity.py @@ -166,6 +166,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/plm/plr_cate_coverage.py b/scripts/plm/plr_cate_coverage.py index 5fee83d..d07aa53 100644 --- a/scripts/plm/plr_cate_coverage.py +++ b/scripts/plm/plr_cate_coverage.py @@ -120,6 +120,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/plm/plr_gate_coverage.py b/scripts/plm/plr_gate_coverage.py index 7c99669..ea41e84 100644 --- a/scripts/plm/plr_gate_coverage.py +++ b/scripts/plm/plr_gate_coverage.py @@ -123,6 +123,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) diff --git a/scripts/rdd/rdd_sharp_coverage.py b/scripts/rdd/rdd_sharp_coverage.py index bff96f9..f3cd6e4 100644 --- a/scripts/rdd/rdd_sharp_coverage.py +++ b/scripts/rdd/rdd_sharp_coverage.py @@ -155,6 +155,8 @@ 'Date': [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], 'Total Runtime (seconds)': [total_runtime], 'Python Version': [f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"], + 'Number of observations': [n_obs], + 'Number of repetitions': [n_rep], }) print(metadata) From 97efa1d15e87e85b45d202f3052045f8600ac674 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 13:05:02 +0000 Subject: [PATCH 60/67] Update results from script: scripts/irm/irm_atte_sensitivity.py --- results/irm/irm_atte_sensitivity.csv | 16 ++++++++-------- results/irm/irm_atte_sensitivity_metadata.csv | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/results/irm/irm_atte_sensitivity.csv b/results/irm/irm_atte_sensitivity.csv index 901b16f..38c5529 100644 --- a/results/irm/irm_atte_sensitivity.csv +++ b/results/irm/irm_atte_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition -LGBM,LGBM,0.9,0.702,0.348892741328716,0.13535473129404849,0.95,1.0,0.10509293446782587,0.024288450266824614,0.0649958270073569,0.25876253126104387,500 -LGBM,LGBM,0.95,0.826,0.41573134306126747,0.13535473129404849,0.982,1.0,0.10509293446782587,0.012452066983782135,0.0649958270073569,0.25876253126104387,500 -LGBM,Logistic Regr.,0.9,0.714,0.34666910502599596,0.13078975736827733,0.964,1.0,0.09817018528214369,0.022246998237972562,0.06545442330246619,0.2589976808508975,500 -LGBM,Logistic Regr.,0.95,0.834,0.41308171698108886,0.13078975736827733,0.984,1.0,0.09817018528214369,0.010949342084431752,0.06545442330246619,0.2589976808508975,500 -Linear Reg.,LGBM,0.9,0.754,0.3496967006881292,0.12455057551341779,0.962,1.0,0.09867724125956986,0.02021759355041513,0.06504946816195573,0.24393419014571052,500 -Linear Reg.,LGBM,0.95,0.858,0.416689319724762,0.12455057551341779,0.986,1.0,0.09867724125956986,0.009856683129418061,0.06504946816195573,0.24393419014571052,500 -Linear Reg.,Logistic Regr.,0.9,0.948,0.3502540540945954,0.07444772768321124,0.996,1.0,0.05840145836627319,0.0041811437412796835,0.09544484272838329,0.17545346180009289,500 -Linear Reg.,Logistic Regr.,0.95,0.976,0.41735344727108903,0.07444772768321124,0.998,1.0,0.05840145836627319,0.001573924919578166,0.09544484272838329,0.17545346180009289,500 +LGBM,LGBM,0.9,0.49,0.24396025943517974,0.12078793981792306,0.99,1.0,0.09312792462754033,0.028226037071058148,0.04146009435287488,0.25069761854571676,100 +LGBM,LGBM,0.95,0.67,0.2906965789036176,0.12078793981792306,1.0,1.0,0.09312792462754033,0.01585795047607936,0.04146009435287488,0.25069761854571676,100 +LGBM,Logistic Regr.,0.9,0.53,0.24426158440218287,0.11892095837667906,0.97,1.0,0.0924028336595813,0.027272723350173122,0.042150350065681345,0.2477418180760181,100 +LGBM,Logistic Regr.,0.95,0.66,0.2910556297475905,0.11892095837667906,1.0,1.0,0.0924028336595813,0.015364138052493198,0.042150350065681345,0.2477418180760181,100 +Linear Reg.,LGBM,0.9,0.51,0.24463965072027827,0.11756694550094914,0.98,1.0,0.0905239932672192,0.02606692134127496,0.041919687829977396,0.24763902427345857,100 +Linear Reg.,LGBM,0.95,0.71,0.2915061235514726,0.11756694550094914,1.0,1.0,0.0905239932672192,0.014359552326725858,0.041919687829977396,0.24763902427345857,100 +Linear Reg.,Logistic Regr.,0.9,0.95,0.24728188741382112,0.05084320840039999,1.0,1.0,0.04043535771019453,0.0022454610340575335,0.09338366625402501,0.1656214414367636,100 +Linear Reg.,Logistic Regr.,0.95,0.99,0.29465454276222763,0.05084320840039999,1.0,1.0,0.04043535771019453,0.00047406259110364837,0.09338366625402501,0.1656214414367636,100 diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv index 3d0e62f..2d2a013 100644 --- a/results/irm/irm_atte_sensitivity_metadata.csv +++ b/results/irm/irm_atte_sensitivity_metadata.csv @@ -1,2 +1,2 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_sensitivity.py,2025-02-17 10:32:47,6933.687040805817,3.12.9 +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions +0.10.dev0,irm_atte_sensitivity.py,2025-02-17 13:05:01,1552.7866415977478,3.12.9,10000,100 From 74b8b123a8b23db1175d77921d4cc26d31f33ecf Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 13:05:23 +0000 Subject: [PATCH 61/67] Update results from script: scripts/irm/irm_ate_sensitivity.py --- results/irm/irm_ate_sensitivity.csv | 16 ++++++++-------- results/irm/irm_ate_sensitivity_metadata.csv | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/results/irm/irm_ate_sensitivity.csv b/results/irm/irm_ate_sensitivity.csv index 9688fed..b81d88b 100644 --- a/results/irm/irm_ate_sensitivity.csv +++ b/results/irm/irm_ate_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),CI Bound Length,repetition -LGBM,LGBM,0.9,0.0,0.1290670759744328,0.16700188836291868,0.97,1.0,0.11516972420151037,0.08199881099554834,0.02323389657437148,0.3116508618137395,0.38989882617146443,100 -LGBM,LGBM,0.95,0.0,0.1537929067698414,0.16700188836291868,1.0,1.0,0.11516972420151037,0.07236261547894074,0.02323389657437148,0.3116508618137395,0.41841777757435944,100 -LGBM,Logistic Regr.,0.9,0.0,0.12799554257877943,0.1521625357301867,1.0,1.0,0.10627830296996256,0.07276073501676483,0.014191719568706143,0.2957015551702169,0.38682853093364633,100 -LGBM,Logistic Regr.,0.95,0.0,0.15251609597689259,0.1521625357301867,1.0,1.0,0.10627830296996256,0.06302810491287761,0.014191719568706143,0.2957015551702169,0.41510640937704496,100 -Linear Reg.,LGBM,0.9,0.0,0.13046535997228081,0.16726605997683605,0.97,1.0,0.1139630025296028,0.08085153079077306,0.02209160244564357,0.31378717139235235,0.3947340873761868,100 -Linear Reg.,LGBM,0.95,0.0,0.15545906491974315,0.16726605997683605,1.0,1.0,0.1139630025296028,0.0712277815883182,0.02209160244564357,0.31378717139235235,0.4235623179451715,100 -Linear Reg.,Logistic Regr.,0.9,0.08,0.12933949008724985,0.08765035815698355,1.0,1.0,0.06183424333715742,0.026740494950661556,0.057894323058536486,0.2331950393725036,0.391886816241763,100 -Linear Reg.,Logistic Regr.,0.95,0.26,0.1541175082062569,0.08765035815698355,1.0,1.0,0.06183424333715742,0.016877272580948677,0.057894323058536486,0.2331950393725036,0.4204614937955592,100 +LGBM,LGBM,0.9,0.0,0.18456010957050942,0.17337243340910458,0.95,1.0,0.11953784478980024,0.07183819359114625,0.03136325666565055,0.31756830147937315,0.43226987036337094,100 +LGBM,LGBM,0.95,0.0,0.21991693474354154,0.17337243340910458,0.99,1.0,0.11953784478980024,0.057824558257341376,0.03136325666565055,0.31756830147937315,0.47305732219900365,100 +LGBM,Logistic Regr.,0.9,0.0,0.1812877961255176,0.15119732710067918,0.98,1.0,0.10434720873886104,0.05696666617037721,0.021677485760078018,0.29642303876248005,0.4317358050796223,100 +LGBM,Logistic Regr.,0.95,0.04,0.2160177327761319,0.15119732710067918,1.0,1.0,0.10434720873886104,0.04307608641610582,0.021677485760078018,0.29642303876248005,0.47178796413164226,100 +Linear Reg.,LGBM,0.9,0.0,0.18605425587835156,0.17399977827987403,0.94,1.0,0.11990868556653686,0.07190610415701137,0.03212149599746574,0.3182707711756607,0.4335852019350203,100 +Linear Reg.,LGBM,0.95,0.0,0.22169731988117328,0.17399977827987403,1.0,1.0,0.11990868556653686,0.057787828308944675,0.03212149599746574,0.3182707711756607,0.4747029383060918,100 +Linear Reg.,Logistic Regr.,0.9,0.57,0.1828627595282207,0.08930473133509086,1.0,1.0,0.06281551261599926,0.015943919677551065,0.05677070380042624,0.234795381047018,0.4334912988005637,100 +Linear Reg.,Logistic Regr.,0.95,0.72,0.21789441742191876,0.08930473133509086,1.0,1.0,0.06281551261599926,0.007945274454461557,0.05677070380042624,0.234795381047018,0.4738909034178378,100 diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv index 844a071..4860b2a 100644 --- a/results/irm/irm_ate_sensitivity_metadata.csv +++ b/results/irm/irm_ate_sensitivity_metadata.csv @@ -1,2 +1,2 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_sensitivity.py,2025-02-17 09:10:45,2014.9821014404297,3.12.9 +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions +0.10.dev0,irm_ate_sensitivity.py,2025-02-17 13:05:22,1572.6342079639435,3.12.9,10000,100 From 83d15e89c5565da3a4c765f98b0320c1881c5657 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 13:12:37 +0000 Subject: [PATCH 62/67] Update results from script: scripts/irm/irm_gate_coverage.py --- results/irm/irm_gate_coverage_metadata.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/results/irm/irm_gate_coverage_metadata.csv b/results/irm/irm_gate_coverage_metadata.csv index eb7a49f..5273a30 100644 --- a/results/irm/irm_gate_coverage_metadata.csv +++ b/results/irm/irm_gate_coverage_metadata.csv @@ -1,2 +1,2 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_gate_coverage.py,2025-02-17 09:11:44,2069.9562945365906,3.12.9 +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions +0.10.dev0,irm_gate_coverage.py,2025-02-17 13:12:35,2007.2094383239746,3.12.9,500,1000 From 6e1cee681c3cb177ed240bb5432d9e9413f066b6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 13:39:13 +0000 Subject: [PATCH 63/67] Update results from script: scripts/irm/irm_atte_coverage.py --- results/irm/irm_atte_coverage.csv | 16 ++++++++-------- results/irm/irm_atte_coverage_metadata.csv | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv index 2c8fe7a..c319959 100644 --- a/results/irm/irm_atte_coverage.csv +++ b/results/irm/irm_atte_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.888,0.5331759172799809,0.13452354417969958,1000 -Lasso,Logistic Regression,0.95,0.941,0.6353182910443254,0.13452354417969958,1000 -Lasso,Random Forest,0.9,0.903,0.7299501387914156,0.17890617607036363,1000 -Lasso,Random Forest,0.95,0.951,0.8697892378379983,0.17890617607036363,1000 -Random Forest,Logistic Regression,0.9,0.873,0.5508681223214307,0.14919172056375843,1000 -Random Forest,Logistic Regression,0.95,0.92,0.6563998536345526,0.14919172056375843,1000 -Random Forest,Random Forest,0.9,0.902,0.7543996805167961,0.18403497407534655,1000 -Random Forest,Random Forest,0.95,0.954,0.8989226637155757,0.18403497407534655,1000 +Lasso,Logistic Regression,0.9,0.889,0.5331759172799809,0.1351855584332284,1000 +Lasso,Logistic Regression,0.95,0.937,0.6353182910443254,0.1351855584332284,1000 +Lasso,Random Forest,0.9,0.9,0.7254806184755598,0.18158167210289639,1000 +Lasso,Random Forest,0.95,0.955,0.864463475895592,0.18158167210289639,1000 +Random Forest,Logistic Regression,0.9,0.869,0.5498973024315036,0.14982335722803167,1000 +Random Forest,Logistic Regression,0.95,0.919,0.6552430503855857,0.14982335722803167,1000 +Random Forest,Random Forest,0.9,0.9,0.754141932378241,0.18774885787201864,1000 +Random Forest,Random Forest,0.95,0.944,0.8986155378653647,0.18774885787201864,1000 diff --git a/results/irm/irm_atte_coverage_metadata.csv b/results/irm/irm_atte_coverage_metadata.csv index 95c4f1b..d250b9b 100644 --- a/results/irm/irm_atte_coverage_metadata.csv +++ b/results/irm/irm_atte_coverage_metadata.csv @@ -1,2 +1,2 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_coverage.py,2025-02-17 09:37:51,3642.9659905433655,3.12.9 +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions +0.10.dev0,irm_atte_coverage.py,2025-02-17 13:39:11,3599.8705773353577,3.12.9,500,1000 From 4e4b6e19a258cbf0430bed32359c0587af3e2c92 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 13:39:00 +0000 Subject: [PATCH 64/67] Update results from script: scripts/irm/irm_ate_coverage.py --- results/irm/irm_ate_coverage.csv | 12 ++++++------ results/irm/irm_ate_coverage_metadata.csv | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv index 999f92c..205cc83 100644 --- a/results/irm/irm_ate_coverage.csv +++ b/results/irm/irm_ate_coverage.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,repetition Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536215,1000 Lasso,Logistic Regression,0.95,0.935,0.5573839547492131,0.12336884150536215,1000 -Lasso,Random Forest,0.9,0.918,0.6021088429949879,0.1471534184182516,1000 -Lasso,Random Forest,0.95,0.96,0.7174569382385991,0.1471534184182516,1000 -Random Forest,Logistic Regression,0.9,0.796,0.5169833951793216,0.15066786899795942,1000 -Random Forest,Logistic Regression,0.95,0.879,0.6160237108968011,0.15066786899795942,1000 -Random Forest,Random Forest,0.9,0.905,0.6288222586448227,0.1522621548395983,1000 -Random Forest,Random Forest,0.95,0.959,0.7492879362799036,0.1522621548395983,1000 +Lasso,Random Forest,0.9,0.918,0.6075675690766795,0.14883418949014868,1000 +Lasso,Random Forest,0.95,0.96,0.7239614115523821,0.14883418949014868,1000 +Random Forest,Logistic Regression,0.9,0.794,0.5179181614571521,0.15026458264746098,1000 +Random Forest,Logistic Regression,0.95,0.886,0.6171375536172053,0.15026458264746098,1000 +Random Forest,Random Forest,0.9,0.887,0.6267427414652039,0.1523909615423744,1000 +Random Forest,Random Forest,0.95,0.951,0.7468100387268922,0.1523909615423744,1000 diff --git a/results/irm/irm_ate_coverage_metadata.csv b/results/irm/irm_ate_coverage_metadata.csv index 983bb7b..ebc0666 100644 --- a/results/irm/irm_ate_coverage_metadata.csv +++ b/results/irm/irm_ate_coverage_metadata.csv @@ -1,2 +1,2 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_coverage.py,2025-02-17 09:36:59,3589.1906917095184,3.12.9 +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions +0.10.dev0,irm_ate_coverage.py,2025-02-17 13:38:59,3587.198892593384,3.12.9,500,1000 From 82e104b23085d3f5af1ef90088eb49f23712b57b Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 14:09:03 +0000 Subject: [PATCH 65/67] Update results from script: scripts/irm/irm_cate_coverage.py --- results/irm/irm_cate_coverage_metadata.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/results/irm/irm_cate_coverage_metadata.csv b/results/irm/irm_cate_coverage_metadata.csv index 1bff3f8..ad0ca97 100644 --- a/results/irm/irm_cate_coverage_metadata.csv +++ b/results/irm/irm_cate_coverage_metadata.csv @@ -1,2 +1,2 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_cate_coverage.py,2025-02-17 10:04:33,5245.124602556229,3.12.9 +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions +0.10.dev0,irm_cate_coverage.py,2025-02-17 14:09:01,5386.899612903595,3.12.9,2000,1000 From ef41dfd6d71b48ac73daddc143859f31bb5ed1d6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 14:15:52 +0000 Subject: [PATCH 66/67] Update results from script: scripts/irm/irm_apo_coverage.py --- results/irm/irm_apo_coverage_apo.csv | 48 +++++++++---------- results/irm/irm_apo_coverage_apos.csv | 16 +++---- .../irm/irm_apo_coverage_apos_contrast.csv | 16 +++---- results/irm/irm_apo_coverage_metadata.csv | 4 +- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv index ee5f3d1..8ee65bb 100644 --- a/results/irm/irm_apo_coverage_apo.csv +++ b/results/irm/irm_apo_coverage_apo.csv @@ -1,25 +1,25 @@ Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.075995364683431,1000 -LGBM,LGBM,0.0,0.95,0.963,10.316274091547035,2.075995364683431,1000 -LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.193797216238146,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.193797216238146,1000 -LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.599689794190313,1000 -LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.599689794190313,1000 -LGBM,Logistic,0.0,0.9,0.902,5.625897101886533,1.3358593836381196,1000 -LGBM,Logistic,0.0,0.95,0.956,6.7036698705295725,1.3358593836381196,1000 -LGBM,Logistic,1.0,0.9,0.92,7.423300143143785,1.6953709510387716,1000 -LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6953709510387716,1000 -LGBM,Logistic,2.0,0.9,0.918,7.321275660150268,1.660500602699947,1000 -LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.660500602699947,1000 -Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.3062893854489512,1000 -Linear,LGBM,0.0,0.95,0.948,6.551577812380716,1.3062893854489512,1000 -Linear,LGBM,1.0,0.9,0.95,10.700720020780512,2.127086970318636,1000 -Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.127086970318636,1000 -Linear,LGBM,2.0,0.9,0.932,7.513644049429104,1.634264519418015,1000 -Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.634264519418015,1000 -Linear,Logistic,0.0,0.9,0.902,5.335670092717667,1.284853808916775,1000 -Linear,Logistic,0.0,0.95,0.952,6.357843058957276,1.284853808916775,1000 -Linear,Logistic,1.0,0.9,0.905,5.417512107920403,1.2802654150082002,1000 -Linear,Logistic,1.0,0.95,0.954,6.455363835025866,1.2802654150082002,1000 -Linear,Logistic,2.0,0.9,0.907,5.366403391397197,1.2795787147265043,1000 -Linear,Logistic,2.0,0.95,0.96,6.3944640430685675,1.2795787147265043,1000 +LGBM,LGBM,0.0,0.9,0.91,8.657690136121921,2.077166476271961,1000 +LGBM,LGBM,0.0,0.95,0.966,10.316274091547035,2.077166476271961,1000 +LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.216990722545221,1000 +LGBM,LGBM,1.0,0.95,0.968,45.55789754237906,9.216990722545221,1000 +LGBM,LGBM,2.0,0.9,0.894,37.49194764946096,9.572829064615373,1000 +LGBM,LGBM,2.0,0.95,0.953,44.67441108385784,9.572829064615373,1000 +LGBM,Logistic,0.0,0.9,0.904,5.625897101886533,1.341832646808728,1000 +LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.341832646808728,1000 +LGBM,Logistic,1.0,0.9,0.924,7.423300143143785,1.6934000724558478,1000 +LGBM,Logistic,1.0,0.95,0.969,8.84540769378873,1.6934000724558478,1000 +LGBM,Logistic,2.0,0.9,0.92,7.321275660150268,1.6623515097372739,1000 +LGBM,Logistic,2.0,0.95,0.965,8.723838024042964,1.6623515097372739,1000 +Linear,LGBM,0.0,0.9,0.9,5.498257423071024,1.3124516734608647,1000 +Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.3124516734608647,1000 +Linear,LGBM,1.0,0.9,0.948,10.700720020780512,2.1292560240454224,1000 +Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.1292560240454224,1000 +Linear,LGBM,2.0,0.9,0.934,7.513644049429104,1.6375362733403969,1000 +Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.6375362733403969,1000 +Linear,Logistic,0.0,0.9,0.905,5.335670092717667,1.2915169940302822,1000 +Linear,Logistic,0.0,0.95,0.955,6.357843058957276,1.2915169940302822,1000 +Linear,Logistic,1.0,0.9,0.908,5.417512107920403,1.2804808010829234,1000 +Linear,Logistic,1.0,0.95,0.957,6.455363835025866,1.2804808010829234,1000 +Linear,Logistic,2.0,0.9,0.906,5.366403391397197,1.2873523314776736,1000 +Linear,Logistic,2.0,0.95,0.957,6.3944640430685675,1.2873523314776736,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv index 5f99bd7..9ba88ab 100644 --- a/results/irm/irm_apo_coverage_apos.csv +++ b/results/irm/irm_apo_coverage_apos.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9066666666666666,28.21021843292038,7.1204285014294015,0.924,36.1943517986729,1000 -LGBM,LGBM,0.95,0.9583333333333334,33.61454856442563,7.1204285014294015,0.974,40.868020474452145,1000 -LGBM,Logistic,0.9,0.917,6.789316986971467,1.5709695238934474,0.924,8.394070597867524,1000 -LGBM,Logistic,0.95,0.9633333333333334,8.089970168806186,1.5709695238934474,0.96,9.592284619044477,1000 -Linear,LGBM,0.9,0.926,7.903418354805325,1.7257798274842937,0.939,9.850445774370156,1000 -Linear,LGBM,0.95,0.9676666666666667,9.41750382912841,1.7257798274842937,0.974,11.234392064837424,1000 -Linear,Logistic,0.9,0.9043333333333333,5.372532806955153,1.2796203798007288,0.905,5.7964894175017925,1000 -Linear,Logistic,0.95,0.9573333333333334,6.4017676921854445,1.2796203798007288,0.953,6.8182385960659,1000 +LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.117854990409516,0.926,36.1943517986729,1000 +LGBM,LGBM,0.95,0.958,33.61454856442563,7.117854990409516,0.973,40.868020474452145,1000 +LGBM,Logistic,0.9,0.9126666666666666,6.789316986971467,1.5740024109154114,0.922,8.394070597867524,1000 +LGBM,Logistic,0.95,0.962,8.089970168806186,1.5740024109154114,0.959,9.592284619044477,1000 +Linear,LGBM,0.9,0.927,7.903418354805325,1.730081847384891,0.936,9.850445774370156,1000 +Linear,LGBM,0.95,0.968,9.41750382912841,1.730081847384891,0.974,11.234392064837424,1000 +Linear,Logistic,0.9,0.9053333333333333,5.372532806955153,1.2840405983352503,0.901,5.7964894175017925,1000 +Linear,Logistic,0.95,0.9556666666666667,6.4017676921854445,1.2840405983352503,0.952,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv index 573b7ac..0c2215c 100644 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ b/results/irm/irm_apo_coverage_apos_contrast.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.789191608123641,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.9485,45.13128131893863,9.789191608123641,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2672891385005909,0.927,6.774304315459575,1000 -LGBM,Logistic,0.95,0.9635,6.821911652197591,1.2672891385005909,0.963,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.506423849047913,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.989,8.85452711998085,1.506423849047913,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.873,1.1425883251377777,0.2945756816692818,0.87,1.3505222299078565,1000 -Linear,Logistic,0.95,0.935,1.3614779635902856,0.2945756816692818,0.92,1.5496710496635275,1000 +LGBM,LGBM,0.9,0.8885,37.87536523730769,9.786311172069826,0.898,44.828925781533165,1000 +LGBM,LGBM,0.95,0.9485,45.13128131893863,9.786311172069826,0.965,51.42387403222799,1000 +LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2679687960356285,0.926,6.774304315459575,1000 +LGBM,Logistic,0.95,0.962,6.821911652197591,1.2679687960356285,0.961,7.7654094416927215,1000 +Linear,LGBM,0.9,0.958,7.430953406859927,1.506712453445455,0.975,8.798628447594016,1000 +Linear,LGBM,0.95,0.9885,8.85452711998085,1.506712453445455,0.992,10.090116294778499,1000 +Linear,Logistic,0.9,0.873,1.1425883251377777,0.2976271580660105,0.871,1.3505222299078565,1000 +Linear,Logistic,0.95,0.933,1.3614779635902856,0.2976271580660105,0.916,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv index 8eeb02f..4681100 100644 --- a/results/irm/irm_apo_coverage_metadata.csv +++ b/results/irm/irm_apo_coverage_metadata.csv @@ -1,2 +1,2 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_apo_coverage.py,2025-02-17 10:12:54,5756.271638393402,3.12.9 +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions +0.10.dev0,irm_apo_coverage.py,2025-02-17 14:15:48,5805.248526096344,3.12.9,500,1000 From 68c486c816a24660e6af021cefa4ca7900de833f Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Feb 2025 15:18:23 +0000 Subject: [PATCH 67/67] Update results from script: scripts/irm/irm_apo_ate_sensitivity.py --- results/irm/irm_apo_sensitivity.csv | 16 ++++++++-------- results/irm/irm_apo_sensitivity_metadata.csv | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/results/irm/irm_apo_sensitivity.csv b/results/irm/irm_apo_sensitivity.csv index adfa34e..ef046c9 100644 --- a/results/irm/irm_apo_sensitivity.csv +++ b/results/irm/irm_apo_sensitivity.csv @@ -1,9 +1,9 @@ Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),CI Bound Length,repetition -LGBM,LGBM,0.9,0.0,0.1292540820255691,0.1677439500916218,0.94,1.0,0.11573872417116222,0.08252331475485856,0.024288344996646342,0.3122738965658806,0.3898116700153508,100 -LGBM,LGBM,0.95,0.0,0.1540157382237242,0.1677439500916218,0.99,1.0,0.11573872417116222,0.07287226227791722,0.024288344996646342,0.3122738965658806,0.418373398846942,100 -LGBM,Logistic Regr.,0.9,0.0,0.12798017567553704,0.15251467338033609,1.0,1.0,0.10652571495226121,0.07301481703198542,0.014645752061926958,0.29602520197592563,0.3867596062890008,100 -LGBM,Logistic Regr.,0.95,0.0,0.15249778518229332,0.15251467338033609,1.0,1.0,0.10652571495226121,0.06328412036942282,0.014645752061926958,0.29602520197592563,0.41503409907039024,100 -Linear Reg.,LGBM,0.9,0.0,0.13050148401216977,0.16740934350762515,0.97,1.0,0.11403669658454366,0.08093114285791209,0.022292080500950134,0.3139559124208978,0.3948171482587492,100 -Linear Reg.,LGBM,0.95,0.0,0.15550210936819642,0.16740934350762515,1.0,1.0,0.11403669658454366,0.07130818482813787,0.022292080500950134,0.3139559124208978,0.42365449174003855,100 -Linear Reg.,Logistic Regr.,0.9,0.08,0.1293446408364081,0.08762861257786415,1.0,1.0,0.06182185548164626,0.026723981525642357,0.057907195061308585,0.2331644202170369,0.39187310331025016,100 -Linear Reg.,Logistic Regr.,0.95,0.26,0.15412364570243178,0.08762861257786415,1.0,1.0,0.06182185548164626,0.016854287151815115,0.057907195061308585,0.2331644202170369,0.420448924509695,100 +LGBM,LGBM,0.9,0.0,0.1850901948571434,0.1728687505886857,0.94,1.0,0.11948035202564793,0.07154674763559875,0.031810817504062554,0.31670282436556935,0.431995754637111,100 +LGBM,LGBM,0.95,0.0,0.2205485703210263,0.1728687505886857,0.98,1.0,0.11948035202564793,0.05745432815748635,0.031810817504062554,0.31670282436556935,0.4729106257831797,100 +LGBM,Logistic Regr.,0.9,0.0,0.18131193168905674,0.15121724930885022,0.99,1.0,0.10434537281186201,0.05696357062448621,0.021088484865101825,0.29646127759826496,0.43179145449456513,100 +LGBM,Logistic Regr.,0.95,0.03,0.2160464920739249,0.15121724930885022,1.0,1.0,0.10434537281186201,0.04307260925250725,0.021088484865101825,0.29646127759826496,0.47184900436363103,100 +Linear Reg.,LGBM,0.9,0.0,0.18663164735761398,0.17472687461915876,0.92,1.0,0.12056131194284912,0.07240650957100214,0.03257050661484845,0.31875766896448493,0.4335837394031654,100 +Linear Reg.,LGBM,0.95,0.0,0.2223853242639293,0.17472687461915876,0.99,1.0,0.12056131194284912,0.05822924694927436,0.03257050661484845,0.31875766896448493,0.47483724706967173,100 +Linear Reg.,Logistic Regr.,0.9,0.56,0.1828483122747645,0.08933864685804803,1.0,1.0,0.06284434571148402,0.015959822013915787,0.05670070892745642,0.23481634587221883,0.43345409695347575,100 +Linear Reg.,Logistic Regr.,0.95,0.71,0.21787720245762926,0.08933864685804803,1.0,1.0,0.06284434571148402,0.007943564313924893,0.05670070892745642,0.23481634587221883,0.4738504980497505,100 diff --git a/results/irm/irm_apo_sensitivity_metadata.csv b/results/irm/irm_apo_sensitivity_metadata.csv index c11bbe8..6900b31 100644 --- a/results/irm/irm_apo_sensitivity_metadata.csv +++ b/results/irm/irm_apo_sensitivity_metadata.csv @@ -1,2 +1,2 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Sensitivity Errors -0.10.dev0,irm_apo_sensitivity.py,2025-02-17 11:58:31,12093.837664842606,3.12.9,0 +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Sensitivity Errors,Number of observations,Number of repetitions +0.10.dev0,irm_apo_sensitivity.py,2025-02-17 15:18:22,9570.017815113068,3.12.9,0,10000,100