From 381e5b29d5c2290f09327b951ac9fd83a27fc77e Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Thu, 5 Jun 2025 14:02:59 +0200 Subject: [PATCH 01/35] remove old scripts --- scripts/irm/cvar_coverage.py | 304 ----------------- scripts/irm/iivm_late_coverage.py | 143 -------- scripts/irm/irm_apo_coverage.py | 295 ----------------- scripts/irm/irm_ate_coverage.py | 139 -------- scripts/irm/irm_ate_sensitivity_old.py | 198 ----------- scripts/irm/irm_atte_coverage.py | 182 ---------- scripts/irm/irm_atte_sensitivity_old.py | 198 ----------- scripts/irm/irm_cate_coverage.py | 161 --------- scripts/irm/irm_gate_coverage.py | 167 ---------- scripts/irm/lpq_coverage.py | 328 ------------------- scripts/irm/pq_coverage.py | 299 ----------------- scripts/irm/ssm_mar_ate_coverage.py | 148 --------- scripts/irm/ssm_nonignorable_ate_coverage.py | 148 --------- scripts/plm/pliv_late_coverage.py | 178 ---------- scripts/plm/plr_ate_coverage.py | 160 --------- scripts/plm/plr_ate_sensitivity_old.py | 240 -------------- scripts/plm/plr_cate_coverage.py | 161 --------- scripts/plm/plr_gate_coverage.py | 167 ---------- scripts/rdd/rdd_fuzzy_coverage.py | 241 -------------- scripts/rdd/rdd_sharp_coverage.py | 202 ------------ 20 files changed, 4059 deletions(-) delete mode 100644 scripts/irm/cvar_coverage.py delete mode 100644 scripts/irm/iivm_late_coverage.py delete mode 100644 scripts/irm/irm_apo_coverage.py delete mode 100644 scripts/irm/irm_ate_coverage.py delete mode 100644 scripts/irm/irm_ate_sensitivity_old.py delete mode 100644 scripts/irm/irm_atte_coverage.py delete mode 100644 scripts/irm/irm_atte_sensitivity_old.py delete mode 100644 scripts/irm/irm_cate_coverage.py delete mode 100644 scripts/irm/irm_gate_coverage.py delete mode 100644 scripts/irm/lpq_coverage.py delete mode 100644 scripts/irm/pq_coverage.py delete mode 100644 scripts/irm/ssm_mar_ate_coverage.py delete mode 100644 scripts/irm/ssm_nonignorable_ate_coverage.py delete mode 100644 scripts/plm/pliv_late_coverage.py delete mode 100644 scripts/plm/plr_ate_coverage.py delete mode 100644 scripts/plm/plr_ate_sensitivity_old.py delete mode 100644 scripts/plm/plr_cate_coverage.py delete mode 100644 scripts/plm/plr_gate_coverage.py delete mode 100644 scripts/rdd/rdd_fuzzy_coverage.py delete mode 100644 scripts/rdd/rdd_sharp_coverage.py diff --git a/scripts/irm/cvar_coverage.py b/scripts/irm/cvar_coverage.py deleted file mode 100644 index 10b4e71..0000000 --- a/scripts/irm/cvar_coverage.py +++ /dev/null @@ -1,304 +0,0 @@ -import numpy as np -import pandas as pd -import multiprocessing -from datetime import datetime -import time -import sys - -from sklearn.linear_model import LogisticRegressionCV, LinearRegression -from lightgbm import LGBMClassifier, LGBMRegressor - -import doubleml as dml - -# set up parallelization -n_cores = multiprocessing.cpu_count() -print(f"Number of Cores: {n_cores}") -cores_used = n_cores - 1 - -# Number of repetitions -n_rep = 100 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 5000 -tau_vec = np.arange(0.2, 0.85, 0.05) -p = 5 - - -# define loc-scale model -def f_loc(D, X): - loc = ( - 0.5 * D - + 2 * D * X[:, 4] - + 2.0 * (X[:, 1] > 0.1) - - 1.7 * (X[:, 0] * X[:, 2] > 0) - - 3 * X[:, 3] - ) - return loc - - -def f_scale(D, X): - scale = np.sqrt(0.5 * D + 0.3 * D * X[:, 1] + 2) - return scale - - -def dgp(n=200, p=5): - X = np.random.uniform(-1, 1, size=[n, p]) - D = ((X[:, 1] - X[:, 3] + 1.5 * (X[:, 0] > 0) + np.random.normal(size=n)) > 0) * 1.0 - epsilon = np.random.normal(size=n) - - Y = f_loc(D, X) + f_scale(D, X) * epsilon - return Y, X, D, epsilon - - -# Estimate true and QTE with counterfactuals on large sample -n_true = int(10e6) - -_, X_true, _, epsilon_true = dgp(n=n_true, p=p) -D1 = np.ones(n_true) -D0 = np.zeros(n_true) - -Y1 = f_loc(D1, X_true) + f_scale(D1, X_true) * epsilon_true -Y0 = f_loc(D0, X_true) + f_scale(D0, X_true) * epsilon_true - -Y1_quant = np.quantile(Y1, q=tau_vec) -Y0_quant = np.quantile(Y0, q=tau_vec) -Y1_cvar = [Y1[Y1 >= quant].mean() for quant in Y1_quant] -Y0_cvar = [Y0[Y0 >= quant].mean() for quant in Y0_quant] -CVAR = np.array(Y1_cvar) - np.array(Y0_cvar) - -print(f"Conditional Value at Risk Y(0): {Y0_cvar}") -print(f"Conditional Value at Risk Y(1): {Y1_cvar}") -print(f"Conditional Value at Risk Effect: {CVAR}") - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - Y, X, D, _ = dgp(n=n_obs, p=p) - data = dml.DoubleMLData.from_arrays(X, Y, D) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Linear", LinearRegression()), - ( - "LGBM", - LGBMRegressor( - n_estimators=300, learning_rate=0.05, num_leaves=10, verbose=-1 - ), - ), - ], - "learner_m": [ - ("Logistic Regression", LogisticRegressionCV()), - ( - "LGBM", - LGBMClassifier( - n_estimators=300, learning_rate=0.05, num_leaves=10, verbose=-1 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed_qte = pd.DataFrame() -df_results_detailed_pq0 = pd.DataFrame() -df_results_detailed_pq1 = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep + 1}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = datasets[i_rep] - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_qte = dml.DoubleMLQTE( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - score="CVaR", - quantiles=tau_vec, - ) - dml_qte.fit(n_jobs_models=cores_used) - effects = dml_qte.coef - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_qte.confint(level=level) - coverage = np.mean( - (confint.iloc[:, 0] < CVAR) & (CVAR < confint.iloc[:, 1]) - ) - ci_length = np.mean(confint.iloc[:, 1] - confint.iloc[:, 0]) - - dml_qte.bootstrap(n_rep_boot=2000) - confint_uniform = dml_qte.confint(level=level, joint=True) - coverage_uniform = all( - (confint_uniform.iloc[:, 0] < CVAR) - & (CVAR < confint_uniform.iloc[:, 1]) - ) - ci_length_uniform = np.mean( - confint_uniform.iloc[:, 1] - confint_uniform.iloc[:, 0] - ) - df_results_detailed_qte = pd.concat( - ( - df_results_detailed_qte, - pd.DataFrame( - { - "Coverage": coverage, - "CI Length": ci_length, - "Bias": np.mean(abs(effects - CVAR)), - "Uniform Coverage": coverage_uniform, - "Uniform CI Length": ci_length_uniform, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - # evaluate each model - coverage_0 = np.zeros(len(tau_vec)) - coverage_1 = np.zeros(len(tau_vec)) - - ci_length_0 = np.zeros(len(tau_vec)) - ci_length_1 = np.zeros(len(tau_vec)) - - bias_0 = np.zeros(len(tau_vec)) - bias_1 = np.zeros(len(tau_vec)) - for tau_idx, tau in enumerate(tau_vec): - model_0 = dml_qte.modellist_0[tau_idx] - model_1 = dml_qte.modellist_1[tau_idx] - - confint_0 = model_0.confint(level=level) - confint_1 = model_1.confint(level=level) - - coverage_0[tau_idx] = (confint_0.iloc[0, 0] < Y0_cvar[tau_idx]) & ( - Y0_cvar[tau_idx] < confint_0.iloc[0, 1] - ) - coverage_1[tau_idx] = (confint_1.iloc[0, 0] < Y1_cvar[tau_idx]) & ( - Y1_cvar[tau_idx] < confint_1.iloc[0, 1] - ) - - ci_length_0[tau_idx] = confint_0.iloc[0, 1] - confint_0.iloc[0, 0] - ci_length_1[tau_idx] = confint_1.iloc[0, 1] - confint_1.iloc[0, 0] - - bias_0[tau_idx] = abs(model_0.coef[0] - Y0_cvar[tau_idx]) - bias_1[tau_idx] = abs(model_1.coef[0] - Y1_cvar[tau_idx]) - - df_results_detailed_pq0 = pd.concat( - ( - df_results_detailed_pq0, - pd.DataFrame( - { - "Coverage": np.mean(coverage_0), - "CI Length": np.mean(ci_length_0), - "Bias": np.mean(bias_0), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - df_results_detailed_pq1 = pd.concat( - ( - df_results_detailed_pq1, - pd.DataFrame( - { - "Coverage": np.mean(coverage_1), - "CI Length": np.mean(ci_length_1), - "Bias": np.mean(bias_1), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results_qte = ( - df_results_detailed_qte.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Uniform Coverage": "mean", - "Uniform CI Length": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results_qte) - -df_results_pq0 = ( - df_results_detailed_pq0.groupby(["Learner g", "Learner m", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results_pq0) - -df_results_pq1 = ( - df_results_detailed_pq1.groupby(["Learner g", "Learner m", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results_pq1) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "cvar_coverage.py" -path = "results/irm/cvar_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results_qte.to_csv(f"{path}_qte.csv", index=False) -df_results_pq0.to_csv(f"{path}_pq0.csv", index=False) -df_results_pq1.to_csv(f"{path}_pq1.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/iivm_late_coverage.py b/scripts/irm/iivm_late_coverage.py deleted file mode 100644 index 52a2fd9..0000000 --- a/scripts/irm/iivm_late_coverage.py +++ /dev/null @@ -1,143 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier -from sklearn.linear_model import LassoCV, LogisticRegressionCV - -import doubleml as dml -from doubleml.datasets import make_iivm_data - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -theta = 0.5 -n_obs = 500 -dim_x = 20 -alpha_x = 1.0 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - data = make_iivm_data( - theta=theta, n_obs=n_obs, dim_x=dim_x, alpha_x=alpha_x, return_type="DataFrame" - ) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Lasso", LassoCV()), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "learner_m": [ - ("Logistic Regression", LogisticRegressionCV()), - ( - "Random Forest", - RandomForestClassifier( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData(datasets[i_rep], "y", "d", z_cols="z") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_iivm = dml.DoubleMLIIVM( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - ml_r=ml_m, - ) - dml_iivm.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_iivm.confint(level=level) - coverage = (confint.iloc[0, 0] < theta) & (theta < confint.iloc[0, 1]) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - confint.iloc[0, 0], - "Bias": abs(dml_iivm.coef[0] - theta), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "iivm_late_coverage.py" -path = "results/irm/iivm_late_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/irm_apo_coverage.py b/scripts/irm/irm_apo_coverage.py deleted file mode 100644 index 692e9e9..0000000 --- a/scripts/irm/irm_apo_coverage.py +++ /dev/null @@ -1,295 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from lightgbm import LGBMRegressor, LGBMClassifier -from sklearn.linear_model import LinearRegression, LogisticRegression - -import doubleml as dml -from doubleml.datasets import make_irm_data_discrete_treatments - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 500 -n_levels = 2 - -# generate the APOs true values -data_apo_large = make_irm_data_discrete_treatments( - n_obs=int(1e6), n_levels=n_levels, linear=True -) -y0 = data_apo_large["oracle_values"]["y0"] -ite = data_apo_large["oracle_values"]["ite"] -d = data_apo_large["d"] - -average_ites = np.full(n_levels + 1, np.nan) -apos = np.full(n_levels + 1, np.nan) -for i in range(n_levels + 1): - average_ites[i] = np.mean(ite[d == i]) * (i > 0) - apos[i] = np.mean(y0) + average_ites[i] - -ates = np.full(n_levels, np.nan) -for i in range(n_levels): - ates[i] = apos[i + 1] - apos[0] - -print(f"Levels and their counts:\n{np.unique(d, return_counts=True)}") -print(f"True APOs: {apos}") -print(f"True ATEs: {ates}") - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - data_apo = make_irm_data_discrete_treatments( - n_obs=n_obs, n_levels=n_levels, linear=True - ) - df_apo = pd.DataFrame( - np.column_stack((data_apo["y"], data_apo["d"], data_apo["x"])), - columns=["y", "d"] + ["x" + str(i) for i in range(data_apo["x"].shape[1])], - ) - datasets.append(df_apo) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [("Linear", LinearRegression()), ("LGBM", LGBMRegressor(verbose=-1))], - "learner_m": [ - ("Logistic", LogisticRegression()), - ("LGBM", LGBMClassifier(verbose=-1)), - ], - "treatment_levels": [0.0, 1.0, 2.0], - "level": [0.95, 0.90], - "trimming_threshold": 0.01, -} - -# set up the results dataframe -df_results_detailed_apo = pd.DataFrame() -df_results_detailed_apos = pd.DataFrame() -df_results_detailed_apos_constrast = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData(datasets[i_rep], "y", "d") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - for treatment_idx, treatment_level in enumerate( - hyperparam_dict["treatment_levels"] - ): - dml_apo = dml.DoubleMLAPO( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - treatment_level=treatment_level, - trimming_threshold=hyperparam_dict["trimming_threshold"], - ) - dml_apo.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_apo.confint(level=level) - coverage = (confint.iloc[0, 0] < apos[treatment_idx]) & ( - apos[treatment_idx] < confint.iloc[0, 1] - ) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - df_results_detailed_apo = pd.concat( - ( - df_results_detailed_apo, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - - confint.iloc[0, 0], - "Bias": abs(dml_apo.coef[0] - apos[treatment_idx]), - "Treatment Level": treatment_level, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - # calculate the APOs - dml_apos = dml.DoubleMLAPOS( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - treatment_levels=hyperparam_dict["treatment_levels"], - trimming_threshold=hyperparam_dict["trimming_threshold"], - ) - dml_apos.fit(n_jobs_cv=5) - effects = dml_apos.coef - - causal_contrast_model = dml_apos.causal_contrast(reference_levels=0) - est_ates = causal_contrast_model.thetas - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_apos.confint(level=level) - coverage = np.mean( - (confint.iloc[:, 0] < apos) & (apos < confint.iloc[:, 1]) - ) - ci_length = np.mean(confint.iloc[:, 1] - confint.iloc[:, 0]) - - dml_apos.bootstrap(n_rep_boot=2000) - confint_uniform = dml_apos.confint(level=level, joint=True) - coverage_uniform = all( - (confint_uniform.iloc[:, 0] < apos) - & (apos < confint_uniform.iloc[:, 1]) - ) - ci_length_uniform = np.mean( - confint_uniform.iloc[:, 1] - confint_uniform.iloc[:, 0] - ) - df_results_detailed_apos = pd.concat( - ( - df_results_detailed_apos, - pd.DataFrame( - { - "Coverage": coverage, - "CI Length": ci_length, - "Bias": np.mean(abs(effects - apos)), - "Uniform Coverage": coverage_uniform, - "Uniform CI Length": ci_length_uniform, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - # calculate the ATEs - confint_contrast = causal_contrast_model.confint(level=level) - coverage_contrast = np.mean( - (confint_contrast.iloc[:, 0] < ates) - & (ates < confint_contrast.iloc[:, 1]) - ) - ci_length_contrast = np.mean( - confint_contrast.iloc[:, 1] - confint_contrast.iloc[:, 0] - ) - - causal_contrast_model.bootstrap(n_rep_boot=2000) - confint_contrast_uniform = causal_contrast_model.confint( - level=level, joint=True - ) - coverage_contrast_uniform = all( - (confint_contrast_uniform.iloc[:, 0] < ates) - & (ates < confint_contrast_uniform.iloc[:, 1]) - ) - ci_length_contrast_uniform = np.mean( - confint_contrast_uniform.iloc[:, 1] - - confint_contrast_uniform.iloc[:, 0] - ) - df_results_detailed_apos_constrast = pd.concat( - ( - df_results_detailed_apos_constrast, - pd.DataFrame( - { - "Coverage": coverage_contrast, - "CI Length": ci_length_contrast, - "Bias": np.mean(abs(est_ates - ates)), - "Uniform Coverage": coverage_contrast_uniform, - "Uniform CI Length": ci_length_contrast_uniform, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results_apo = ( - df_results_detailed_apo.groupby( - ["Learner g", "Learner m", "Treatment Level", "level"] - ) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results_apo) - -df_results_apos = ( - df_results_detailed_apos.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Uniform Coverage": "mean", - "Uniform CI Length": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results_apos) - -df_results_apos_contrast = ( - df_results_detailed_apos_constrast.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Uniform Coverage": "mean", - "Uniform CI Length": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results_apos_contrast) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "irm_apo_coverage.py" -path = "results/irm/irm_apo_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results_apo.to_csv(f"{path}_apo.csv", index=False) -df_results_apos.to_csv(f"{path}_apos.csv", index=False) -df_results_apos_contrast.to_csv(f"{path}_apos_contrast.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/irm_ate_coverage.py b/scripts/irm/irm_ate_coverage.py deleted file mode 100644 index 318bded..0000000 --- a/scripts/irm/irm_ate_coverage.py +++ /dev/null @@ -1,139 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier -from sklearn.linear_model import LassoCV, LogisticRegressionCV - -import doubleml as dml -from doubleml.datasets import make_irm_data - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -theta = 0.5 -n_obs = 500 -dim_x = 20 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - data = make_irm_data(theta=theta, n_obs=n_obs, dim_x=dim_x, return_type="DataFrame") - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Lasso", LassoCV()), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "learner_m": [ - ("Logistic Regression", LogisticRegressionCV()), - ( - "Random Forest", - RandomForestClassifier( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData(datasets[i_rep], "y", "d") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_irm = dml.DoubleMLIRM( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - ) - dml_irm.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_irm.confint(level=level) - coverage = (confint.iloc[0, 0] < theta) & (theta < confint.iloc[0, 1]) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - confint.iloc[0, 0], - "Bias": abs(dml_irm.coef[0] - theta), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "irm_ate_coverage.py" -path = "results/irm/irm_ate_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/irm_ate_sensitivity_old.py b/scripts/irm/irm_ate_sensitivity_old.py deleted file mode 100644 index b53600c..0000000 --- a/scripts/irm/irm_ate_sensitivity_old.py +++ /dev/null @@ -1,198 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from sklearn.linear_model import LinearRegression, LogisticRegression -from lightgbm import LGBMRegressor, LGBMClassifier - -import doubleml as dml -from doubleml.datasets import make_confounded_irm_data - -# Number of repetitions -n_rep = 500 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 5000 -theta = 5.0 -trimming_threshold = 0.05 - -dgp_pars = { - "gamma_a": 0.198, - "beta_a": 0.582, - "theta": theta, - "var_epsilon_y": 1.0, - "trimming_threshold": trimming_threshold, - "linear": False, -} - -# test inputs -np.random.seed(42) -dgp_dict = make_confounded_irm_data(n_obs=int(1e6), **dgp_pars) - -oracle_dict = dgp_dict["oracle_values"] -rho = oracle_dict["rho_ate"] -cf_y = oracle_dict["cf_y"] -cf_d = oracle_dict["cf_d_ate"] - -print(f"Confounding factor for Y: {cf_y}") -print(f"Confounding factor for D: {cf_d}") -print(f"Rho: {rho}") - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - dgp_dict = make_confounded_irm_data(n_obs=n_obs, **dgp_pars) - datasets.append(dgp_dict) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Linear Reg.", LinearRegression()), - ( - "LGBM", - LGBMRegressor( - n_estimators=500, learning_rate=0.01, min_child_samples=10, verbose=-1 - ), - ), - ], - "learner_m": [ - ("Logistic Regr.", LogisticRegression()), - ( - "LGBM", - LGBMClassifier( - n_estimators=500, learning_rate=0.01, min_child_samples=10, verbose=-1 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - dgp_dict = datasets[i_rep] - - x_cols = [f"X{i + 1}" for i in np.arange(dgp_dict["x"].shape[1])] - df = pd.DataFrame( - np.column_stack((dgp_dict["x"], dgp_dict["y"], dgp_dict["d"])), - columns=x_cols + ["y", "d"], - ) - obj_dml_data = dml.DoubleMLData(df, "y", "d") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_irm = dml.DoubleMLIRM( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - trimming_threshold=trimming_threshold, - ) - dml_irm.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - estimate = dml_irm.coef[0] - confint = dml_irm.confint(level=level) - coverage = (confint.iloc[0, 0] < theta) & (theta < confint.iloc[0, 1]) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - # test sensitivity parameters - dml_irm.sensitivity_analysis( - cf_y=cf_y, cf_d=cf_d, rho=rho, level=level, null_hypothesis=theta - ) - cover_lower = theta >= dml_irm.sensitivity_params["ci"]["lower"] - cover_upper = theta <= dml_irm.sensitivity_params["ci"]["upper"] - rv = dml_irm.sensitivity_params["rv"] - rva = dml_irm.sensitivity_params["rva"] - bias_lower = abs(theta - dml_irm.sensitivity_params["theta"]["lower"]) - bias_upper = abs(theta - dml_irm.sensitivity_params["theta"]["upper"]) - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - confint.iloc[0, 0], - "Bias": abs(estimate - theta), - "Coverage (Lower)": cover_lower.astype(int), - "Coverage (Upper)": cover_upper.astype(int), - "RV": rv, - "RVa": rva, - "Bias (Lower)": bias_lower, - "Bias (Upper)": bias_upper, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Coverage (Lower)": "mean", - "Coverage (Upper)": "mean", - "RV": "mean", - "RVa": "mean", - "Bias (Lower)": "mean", - "Bias (Upper)": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "irm_ate_sensitivity.py" -path = "results/irm/irm_ate_sensitivity" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/irm_atte_coverage.py b/scripts/irm/irm_atte_coverage.py deleted file mode 100644 index 53d38bc..0000000 --- a/scripts/irm/irm_atte_coverage.py +++ /dev/null @@ -1,182 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier -from sklearn.linear_model import LassoCV, LogisticRegressionCV - -import doubleml as dml -from doubleml.datasets import make_irm_data -from scipy.linalg import toeplitz - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -theta = 0.5 -n_obs = 500 -dim_x = 20 - -# We can simulate the ATTE from the function via MC-samples -n_obs_atte = 50000 - -# manual make irm data with default params -R2_d = 0.5 -R2_y = 0.5 - -v = np.random.uniform( - size=[ - n_obs_atte, - ] -) -zeta = np.random.standard_normal( - size=[ - n_obs_atte, - ] -) - -cov_mat = toeplitz([np.power(0.5, k) for k in range(dim_x)]) -x = np.random.multivariate_normal( - np.zeros(dim_x), - cov_mat, - size=[ - n_obs_atte, - ], -) - -beta = [1 / (k**2) for k in range(1, dim_x + 1)] -b_sigma_b = np.dot(np.dot(cov_mat, beta), beta) -c_y = np.sqrt(R2_y / ((1 - R2_y) * b_sigma_b)) -c_d = np.sqrt(np.pi**2 / 3.0 * R2_d / ((1 - R2_d) * b_sigma_b)) - -xx = np.exp(np.dot(x, np.multiply(beta, c_d))) -d = 1.0 * ((xx / (1 + xx)) > v) - -y = d * theta + d * np.dot(x, np.multiply(beta, c_y)) + zeta -y0 = zeta -y1 = theta + np.dot(x, np.multiply(beta, c_y)) + zeta - -ATTE = np.mean(y1[d == 1] - y0[d == 1]) -print(ATTE) - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - data = make_irm_data(theta=theta, n_obs=n_obs, dim_x=dim_x, return_type="DataFrame") - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Lasso", LassoCV()), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "learner_m": [ - ("Logistic Regression", LogisticRegressionCV()), - ( - "Random Forest", - RandomForestClassifier( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData(datasets[i_rep], "y", "d") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_irm = dml.DoubleMLIRM( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - score="ATTE", - ) - dml_irm.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_irm.confint(level=level) - coverage = (confint.iloc[0, 0] < ATTE) & (ATTE < confint.iloc[0, 1]) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - confint.iloc[0, 0], - "Bias": abs(dml_irm.coef[0] - ATTE), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results) -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "irm_atte_coverage.py" -path = "results/irm/irm_atte_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/irm_atte_sensitivity_old.py b/scripts/irm/irm_atte_sensitivity_old.py deleted file mode 100644 index 10b5f0d..0000000 --- a/scripts/irm/irm_atte_sensitivity_old.py +++ /dev/null @@ -1,198 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from sklearn.linear_model import LinearRegression, LogisticRegression -from lightgbm import LGBMRegressor, LGBMClassifier - -import doubleml as dml -from doubleml.datasets import make_confounded_irm_data - -# Number of repetitions -n_rep = 500 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 5000 -theta = 5.0 -trimming_threshold = 0.05 - -dgp_pars = { - "gamma_a": 0.151, - "beta_a": 0.580, - "theta": theta, - "var_epsilon_y": 1.0, - "trimming_threshold": trimming_threshold, - "linear": False, -} - -# test inputs -np.random.seed(42) -dgp_dict = make_confounded_irm_data(n_obs=int(1e6), **dgp_pars) - -oracle_dict = dgp_dict["oracle_values"] -rho = oracle_dict["rho_atte"] -cf_y = oracle_dict["cf_y"] -cf_d = oracle_dict["cf_d_atte"] - -print(f"Confounding factor for Y: {cf_y}") -print(f"Confounding factor for D: {cf_d}") -print(f"Rho: {rho}") - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - dgp_dict = make_confounded_irm_data(n_obs=n_obs, **dgp_pars) - datasets.append(dgp_dict) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Linear Reg.", LinearRegression()), - ( - "LGBM", - LGBMRegressor( - n_estimators=500, learning_rate=0.01, min_child_samples=10, verbose=-1 - ), - ), - ], - "learner_m": [ - ("Logistic Regr.", LogisticRegression()), - ( - "LGBM", - LGBMClassifier( - n_estimators=500, learning_rate=0.01, min_child_samples=10, verbose=-1 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - dgp_dict = datasets[i_rep] - - x_cols = [f"X{i + 1}" for i in np.arange(dgp_dict["x"].shape[1])] - df = pd.DataFrame( - np.column_stack((dgp_dict["x"], dgp_dict["y"], dgp_dict["d"])), - columns=x_cols + ["y", "d"], - ) - obj_dml_data = dml.DoubleMLData(df, "y", "d") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_irm = dml.DoubleMLIRM( - obj_dml_data=obj_dml_data, - score="ATTE", - ml_g=ml_g, - ml_m=ml_m, - trimming_threshold=trimming_threshold, - ) - dml_irm.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - estimate = dml_irm.coef[0] - confint = dml_irm.confint(level=level) - coverage = (confint.iloc[0, 0] < theta) & (theta < confint.iloc[0, 1]) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - # test sensitivity parameters - dml_irm.sensitivity_analysis( - cf_y=cf_y, cf_d=cf_d, rho=rho, level=level, null_hypothesis=theta - ) - cover_lower = theta >= dml_irm.sensitivity_params["ci"]["lower"] - cover_upper = theta <= dml_irm.sensitivity_params["ci"]["upper"] - rv = dml_irm.sensitivity_params["rv"] - rva = dml_irm.sensitivity_params["rva"] - bias_lower = abs(theta - dml_irm.sensitivity_params["theta"]["lower"]) - bias_upper = abs(theta - dml_irm.sensitivity_params["theta"]["upper"]) - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - confint.iloc[0, 0], - "Bias": abs(estimate - theta), - "Coverage (Lower)": cover_lower.astype(int), - "Coverage (Upper)": cover_upper.astype(int), - "RV": rv, - "RVa": rva, - "Bias (Lower)": bias_lower, - "Bias (Upper)": bias_upper, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Coverage (Lower)": "mean", - "Coverage (Upper)": "mean", - "RV": "mean", - "RVa": "mean", - "Bias (Lower)": "mean", - "Bias (Upper)": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results) -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "irm_atte_sensitivity.py" -path = "results/irm/irm_atte_sensitivity" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/irm_cate_coverage.py b/scripts/irm/irm_cate_coverage.py deleted file mode 100644 index 85e5dfc..0000000 --- a/scripts/irm/irm_cate_coverage.py +++ /dev/null @@ -1,161 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys -import patsy - -from lightgbm import LGBMRegressor, LGBMClassifier -from sklearn.linear_model import LassoCV, LogisticRegressionCV - -import doubleml as dml -from doubleml.datasets import make_heterogeneous_data - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 2000 -p = 10 -support_size = 5 -n_x = 1 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - data = make_heterogeneous_data( - n_obs=n_obs, p=p, support_size=support_size, n_x=n_x, binary_treatment=True - ) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Lasso", LassoCV()), - ("LGBM", LGBMRegressor(n_estimators=200, learning_rate=0.05, verbose=-1)), - ], - "learner_m": [ - ("Logistic Regression", LogisticRegressionCV()), - ("LGBM", LGBMClassifier(n_estimators=200, learning_rate=0.05, verbose=-1)), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - data = datasets[i_rep]["data"] - design_matrix = patsy.dmatrix("bs(x, df=5, degree=2)", {"x": data["X_0"]}) - spline_basis = pd.DataFrame(design_matrix) - - true_effects = datasets[i_rep]["effects"] - - obj_dml_data = dml.DoubleMLData(data, "y", "d") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_irm = dml.DoubleMLIRM( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - ) - dml_irm.fit(n_jobs_cv=5) - cate = dml_irm.cate(spline_basis) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = cate.confint(basis=spline_basis, level=level) - effects = confint["effect"] - coverage = (confint.iloc[:, 0] < true_effects) & ( - true_effects < confint.iloc[:, 2] - ) - ci_length = confint.iloc[:, 2] - confint.iloc[:, 0] - confint_uniform = cate.confint( - basis=spline_basis, level=0.95, joint=True, n_rep_boot=2000 - ) - coverage_uniform = all( - (confint_uniform.iloc[:, 0] < true_effects) - & (true_effects < confint_uniform.iloc[:, 2]) - ) - ci_length_uniform = ( - confint_uniform.iloc[:, 2] - confint_uniform.iloc[:, 0] - ) - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.mean(), - "CI Length": ci_length.mean(), - "Bias": abs(effects - true_effects).mean(), - "Uniform Coverage": coverage_uniform, - "Uniform CI Length": ci_length_uniform.mean(), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Uniform Coverage": "mean", - "Uniform CI Length": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "irm_cate_coverage.py" -path = "results/irm/irm_cate_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/irm_gate_coverage.py b/scripts/irm/irm_gate_coverage.py deleted file mode 100644 index 4588172..0000000 --- a/scripts/irm/irm_gate_coverage.py +++ /dev/null @@ -1,167 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from lightgbm import LGBMRegressor, LGBMClassifier -from sklearn.linear_model import LassoCV, LogisticRegressionCV - -import doubleml as dml -from doubleml.datasets import make_heterogeneous_data - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 500 -p = 10 -support_size = 5 -n_x = 1 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - data = make_heterogeneous_data( - n_obs=n_obs, p=p, support_size=support_size, n_x=n_x, binary_treatment=True - ) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Lasso", LassoCV()), - ("LGBM", LGBMRegressor(n_estimators=200, learning_rate=0.05, verbose=-1)), - ], - "learner_m": [ - ("Logistic Regression", LogisticRegressionCV()), - ("LGBM", LGBMClassifier(n_estimators=200, learning_rate=0.05, verbose=-1)), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - data = datasets[i_rep]["data"] - ite = datasets[i_rep]["effects"] - - groups = pd.DataFrame( - np.column_stack( - ( - data["X_0"] <= 0.3, - (data["X_0"] > 0.3) & (data["X_0"] <= 0.7), - data["X_0"] > 0.7, - ) - ), - columns=["Group 1", "Group 2", "Group 3"], - ) - true_effects = [ite[groups[group]].mean() for group in groups.columns] - - obj_dml_data = dml.DoubleMLData(data, "y", "d") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_irm = dml.DoubleMLIRM( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - ) - dml_irm.fit(n_jobs_cv=5) - gate = dml_irm.gate(groups=groups) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = gate.confint(level=level) - effects = confint["effect"] - coverage = (confint.iloc[:, 0] < true_effects) & ( - true_effects < confint.iloc[:, 2] - ) - ci_length = confint.iloc[:, 2] - confint.iloc[:, 0] - confint_uniform = gate.confint(level=0.95, joint=True, n_rep_boot=2000) - coverage_uniform = all( - (confint_uniform.iloc[:, 0] < true_effects) - & (true_effects < confint_uniform.iloc[:, 2]) - ) - ci_length_uniform = ( - confint_uniform.iloc[:, 2] - confint_uniform.iloc[:, 0] - ) - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.mean(), - "CI Length": ci_length.mean(), - "Bias": abs(effects - true_effects).mean(), - "Uniform Coverage": coverage_uniform, - "Uniform CI Length": ci_length_uniform.mean(), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Uniform Coverage": "mean", - "Uniform CI Length": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "irm_gate_coverage.py" -path = "results/irm/irm_gate_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/lpq_coverage.py b/scripts/irm/lpq_coverage.py deleted file mode 100644 index 796053c..0000000 --- a/scripts/irm/lpq_coverage.py +++ /dev/null @@ -1,328 +0,0 @@ -import numpy as np -import pandas as pd -import multiprocessing -from datetime import datetime -import time -import sys - -from sklearn.linear_model import LogisticRegressionCV -from lightgbm import LGBMClassifier - -import doubleml as dml - -# set up parallelization -n_cores = multiprocessing.cpu_count() -print(f"Number of Cores: {n_cores}") -cores_used = n_cores - 1 - -# Number of repetitions -n_rep = 100 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 5000 -tau_vec = np.arange(0.3, 0.75, 0.05) -p = 5 - - -# define loc-scale model -def f_loc(D, X, X_conf): - loc = ( - 0.5 * D - + 2 * D * X[:, 4] - + 2.0 * (X[:, 1] > 0.1) - - 1.7 * (X[:, 0] * X[:, 2] > 0) - - 3 * X[:, 3] - - 2 * X_conf[:, 0] - ) - return loc - - -def f_scale(D, X, X_conf): - scale = np.sqrt(0.5 * D + 3 * D * X[:, 0] + 0.4 * X_conf[:, 0] + 2) - return scale - - -def generate_treatment(Z, X, X_conf): - eta = np.random.normal(size=len(Z)) - d = ((0.5 * Z - 0.3 * X[:, 0] + 0.7 * X_conf[:, 0] + eta) > 0) * 1.0 - return d - - -def dgp(n=200, p=5): - X = np.random.uniform(0, 1, size=[n, p]) - X_conf = np.random.uniform(-1, 1, size=[n, 1]) - Z = np.random.binomial(1, p=0.5, size=n) - D = generate_treatment(Z, X, X_conf) - epsilon = np.random.normal(size=n) - - Y = f_loc(D, X, X_conf) + f_scale(D, X, X_conf) * epsilon - - return Y, X, D, Z - - -# Estimate true LPQ and LQTE with counterfactuals on large sample - -n_true = int(10e6) - -X_true = np.random.uniform(0, 1, size=[n_true, p]) -X_conf_true = np.random.uniform(-1, 1, size=[n_true, 1]) -Z_true = np.random.binomial(1, p=0.5, size=n_true) -eta_true = np.random.normal(size=n_true) -D1_true = generate_treatment(np.ones_like(Z_true), X_true, X_conf_true) -D0_true = generate_treatment(np.zeros_like(Z_true), X_true, X_conf_true) -epsilon_true = np.random.normal(size=n_true) - -compliers = (D1_true == 1) * (D0_true == 0) -print(f"Compliance probability: {str(compliers.mean())}") -n_compliers = compliers.sum() -Y1 = ( - f_loc(np.ones(n_compliers), X_true[compliers, :], X_conf_true[compliers, :]) - + f_scale(np.ones(n_compliers), X_true[compliers, :], X_conf_true[compliers, :]) - * epsilon_true[compliers] -) -Y0 = ( - f_loc(np.zeros(n_compliers), X_true[compliers, :], X_conf_true[compliers, :]) - + f_scale(np.zeros(n_compliers), X_true[compliers, :], X_conf_true[compliers, :]) - * epsilon_true[compliers] -) - -Y0_quant = np.quantile(Y0, q=tau_vec) -Y1_quant = np.quantile(Y1, q=tau_vec) -print(f"Local Potential Quantile Y(0): {Y0_quant}") -print(f"Local Potential Quantile Y(1): {Y1_quant}") -LQTE = Y1_quant - Y0_quant -print(f"Local Quantile Treatment Effect: {LQTE}") - - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - Y, X, D, Z = dgp(n=n_obs, p=p) - data = dml.DoubleMLData.from_arrays(X, Y, D, Z) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Logistic Regression", LogisticRegressionCV()), - ( - "LGBM", - LGBMClassifier( - n_estimators=300, learning_rate=0.05, num_leaves=10, verbose=-1 - ), - ), - ], - "learner_m": [ - ("Logistic Regression", LogisticRegressionCV()), - ( - "LGBM", - LGBMClassifier( - n_estimators=300, learning_rate=0.05, num_leaves=10, verbose=-1 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed_qte = pd.DataFrame() -df_results_detailed_pq0 = pd.DataFrame() -df_results_detailed_pq1 = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = datasets[i_rep] - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_qte = dml.DoubleMLQTE( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - score="LPQ", - quantiles=tau_vec, - ) - dml_qte.fit(n_jobs_models=cores_used) - effects = dml_qte.coef - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_qte.confint(level=level) - coverage = np.mean( - (confint.iloc[:, 0] < LQTE) & (LQTE < confint.iloc[:, 1]) - ) - ci_length = np.mean(confint.iloc[:, 1] - confint.iloc[:, 0]) - - dml_qte.bootstrap(n_rep_boot=2000) - confint_uniform = dml_qte.confint(level=level, joint=True) - coverage_uniform = all( - (confint_uniform.iloc[:, 0] < LQTE) - & (LQTE < confint_uniform.iloc[:, 1]) - ) - ci_length_uniform = np.mean( - confint_uniform.iloc[:, 1] - confint_uniform.iloc[:, 0] - ) - df_results_detailed_qte = pd.concat( - ( - df_results_detailed_qte, - pd.DataFrame( - { - "Coverage": coverage, - "CI Length": ci_length, - "Bias": np.mean(abs(effects - LQTE)), - "Uniform Coverage": coverage_uniform, - "Uniform CI Length": ci_length_uniform, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - # evaluate each model - coverage_0 = np.zeros(len(tau_vec)) - coverage_1 = np.zeros(len(tau_vec)) - - ci_length_0 = np.zeros(len(tau_vec)) - ci_length_1 = np.zeros(len(tau_vec)) - - bias_0 = np.zeros(len(tau_vec)) - bias_1 = np.zeros(len(tau_vec)) - for tau_idx, tau in enumerate(tau_vec): - model_0 = dml_qte.modellist_0[tau_idx] - model_1 = dml_qte.modellist_1[tau_idx] - - confint_0 = model_0.confint(level=level) - confint_1 = model_1.confint(level=level) - - coverage_0[tau_idx] = (confint_0.iloc[0, 0] < Y0_quant[tau_idx]) & ( - Y0_quant[tau_idx] < confint_0.iloc[0, 1] - ) - coverage_1[tau_idx] = (confint_1.iloc[0, 0] < Y1_quant[tau_idx]) & ( - Y1_quant[tau_idx] < confint_1.iloc[0, 1] - ) - - ci_length_0[tau_idx] = confint_0.iloc[0, 1] - confint_0.iloc[0, 0] - ci_length_1[tau_idx] = confint_1.iloc[0, 1] - confint_1.iloc[0, 0] - - bias_0[tau_idx] = abs(model_0.coef[0] - Y0_quant[tau_idx]) - bias_1[tau_idx] = abs(model_1.coef[0] - Y1_quant[tau_idx]) - - df_results_detailed_pq0 = pd.concat( - ( - df_results_detailed_pq0, - pd.DataFrame( - { - "Coverage": np.mean(coverage_0), - "CI Length": np.mean(ci_length_0), - "Bias": np.mean(bias_0), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - df_results_detailed_pq1 = pd.concat( - ( - df_results_detailed_pq1, - pd.DataFrame( - { - "Coverage": np.mean(coverage_1), - "CI Length": np.mean(ci_length_1), - "Bias": np.mean(bias_1), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results_qte = ( - df_results_detailed_qte.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Uniform Coverage": "mean", - "Uniform CI Length": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results_qte) - -df_results_pq0 = ( - df_results_detailed_pq0.groupby(["Learner g", "Learner m", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results_pq0) - -df_results_pq1 = ( - df_results_detailed_pq1.groupby(["Learner g", "Learner m", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results_pq1) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "lpq_coverage.py" -path = "results/irm/lpq_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results_qte.to_csv(f"{path}_lqte.csv", index=False) -df_results_pq0.to_csv(f"{path}_lpq0.csv", index=False) -df_results_pq1.to_csv(f"{path}_lpq1.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/pq_coverage.py b/scripts/irm/pq_coverage.py deleted file mode 100644 index 480bef9..0000000 --- a/scripts/irm/pq_coverage.py +++ /dev/null @@ -1,299 +0,0 @@ -import numpy as np -import pandas as pd -import multiprocessing -from datetime import datetime -import time -import sys - -from sklearn.linear_model import LogisticRegressionCV -from lightgbm import LGBMClassifier - -import doubleml as dml - -# set up parallelization -n_cores = multiprocessing.cpu_count() -print(f"Number of Cores: {n_cores}") -cores_used = n_cores - 1 - -# Number of repetitions -n_rep = 100 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 5000 -tau_vec = np.arange(0.2, 0.85, 0.05) -p = 5 - - -# define loc-scale model -def f_loc(D, X): - loc = ( - 0.5 * D - + 2 * D * X[:, 4] - + 2.0 * (X[:, 1] > 0.1) - - 1.7 * (X[:, 0] * X[:, 2] > 0) - - 3 * X[:, 3] - ) - return loc - - -def f_scale(D, X): - scale = np.sqrt(0.5 * D + 0.3 * D * X[:, 1] + 2) - return scale - - -def dgp(n=200, p=5): - X = np.random.uniform(-1, 1, size=[n, p]) - D = ((X[:, 1] - X[:, 3] + 1.5 * (X[:, 0] > 0) + np.random.normal(size=n)) > 0) * 1.0 - epsilon = np.random.normal(size=n) - - Y = f_loc(D, X) + f_scale(D, X) * epsilon - return Y, X, D, epsilon - - -# Estimate true PQ and QTE with counterfactuals on large sample -n_true = int(10e6) - -_, X_true, _, epsilon_true = dgp(n=n_true, p=p) -D1 = np.ones(n_true) -D0 = np.zeros(n_true) - -Y1 = f_loc(D1, X_true) + f_scale(D1, X_true) * epsilon_true -Y0 = f_loc(D0, X_true) + f_scale(D0, X_true) * epsilon_true - -Y1_quant = np.quantile(Y1, q=tau_vec) -Y0_quant = np.quantile(Y0, q=tau_vec) -QTE = Y1_quant - Y0_quant - - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - Y, X, D, _ = dgp(n=n_obs, p=p) - data = dml.DoubleMLData.from_arrays(X, Y, D) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Logistic Regression", LogisticRegressionCV()), - ( - "LGBM", - LGBMClassifier( - n_estimators=300, learning_rate=0.05, num_leaves=10, verbose=-1 - ), - ), - ], - "learner_m": [ - ("Logistic Regression", LogisticRegressionCV()), - ( - "LGBM", - LGBMClassifier( - n_estimators=300, learning_rate=0.05, num_leaves=10, verbose=-1 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed_qte = pd.DataFrame() -df_results_detailed_pq0 = pd.DataFrame() -df_results_detailed_pq1 = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = datasets[i_rep] - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_qte = dml.DoubleMLQTE( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - score="PQ", - quantiles=tau_vec, - ) - dml_qte.fit(n_jobs_models=cores_used) - effects = dml_qte.coef - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_qte.confint(level=level) - coverage = np.mean( - (confint.iloc[:, 0] < QTE) & (QTE < confint.iloc[:, 1]) - ) - ci_length = np.mean(confint.iloc[:, 1] - confint.iloc[:, 0]) - - dml_qte.bootstrap(n_rep_boot=2000) - confint_uniform = dml_qte.confint(level=level, joint=True) - coverage_uniform = all( - (confint_uniform.iloc[:, 0] < QTE) - & (QTE < confint_uniform.iloc[:, 1]) - ) - ci_length_uniform = np.mean( - confint_uniform.iloc[:, 1] - confint_uniform.iloc[:, 0] - ) - df_results_detailed_qte = pd.concat( - ( - df_results_detailed_qte, - pd.DataFrame( - { - "Coverage": coverage, - "CI Length": ci_length, - "Bias": np.mean(abs(effects - QTE)), - "Uniform Coverage": coverage_uniform, - "Uniform CI Length": ci_length_uniform, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - # evaluate each model - coverage_0 = np.zeros(len(tau_vec)) - coverage_1 = np.zeros(len(tau_vec)) - - ci_length_0 = np.zeros(len(tau_vec)) - ci_length_1 = np.zeros(len(tau_vec)) - - bias_0 = np.zeros(len(tau_vec)) - bias_1 = np.zeros(len(tau_vec)) - for tau_idx, tau in enumerate(tau_vec): - model_0 = dml_qte.modellist_0[tau_idx] - model_1 = dml_qte.modellist_1[tau_idx] - - confint_0 = model_0.confint(level=level) - confint_1 = model_1.confint(level=level) - - coverage_0[tau_idx] = (confint_0.iloc[0, 0] < Y0_quant[tau_idx]) & ( - Y0_quant[tau_idx] < confint_0.iloc[0, 1] - ) - coverage_1[tau_idx] = (confint_1.iloc[0, 0] < Y1_quant[tau_idx]) & ( - Y1_quant[tau_idx] < confint_1.iloc[0, 1] - ) - - ci_length_0[tau_idx] = confint_0.iloc[0, 1] - confint_0.iloc[0, 0] - ci_length_1[tau_idx] = confint_1.iloc[0, 1] - confint_1.iloc[0, 0] - - bias_0[tau_idx] = abs(model_0.coef[0] - Y0_quant[tau_idx]) - bias_1[tau_idx] = abs(model_1.coef[0] - Y1_quant[tau_idx]) - - df_results_detailed_pq0 = pd.concat( - ( - df_results_detailed_pq0, - pd.DataFrame( - { - "Coverage": np.mean(coverage_0), - "CI Length": np.mean(ci_length_0), - "Bias": np.mean(bias_0), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - df_results_detailed_pq1 = pd.concat( - ( - df_results_detailed_pq1, - pd.DataFrame( - { - "Coverage": np.mean(coverage_1), - "CI Length": np.mean(ci_length_1), - "Bias": np.mean(bias_1), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results_qte = ( - df_results_detailed_qte.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Uniform Coverage": "mean", - "Uniform CI Length": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results_qte) - -df_results_pq0 = ( - df_results_detailed_pq0.groupby(["Learner g", "Learner m", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results_pq0) - -df_results_pq1 = ( - df_results_detailed_pq1.groupby(["Learner g", "Learner m", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results_pq1) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "pq_coverage.py" -path = "results/irm/pq_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results_qte.to_csv(f"{path}_qte.csv", index=False) -df_results_pq0.to_csv(f"{path}_pq0.csv", index=False) -df_results_pq1.to_csv(f"{path}_pq1.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/ssm_mar_ate_coverage.py b/scripts/irm/ssm_mar_ate_coverage.py deleted file mode 100644 index ac073d2..0000000 --- a/scripts/irm/ssm_mar_ate_coverage.py +++ /dev/null @@ -1,148 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from lightgbm import LGBMRegressor, LGBMClassifier -from sklearn.linear_model import LassoCV, LogisticRegressionCV - -import doubleml as dml -from doubleml.datasets import make_ssm_data - - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -theta = 1.0 -n_obs = 500 -dim_x = 20 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) - -datasets = [] -for i in range(n_rep): - data = make_ssm_data( - theta=theta, n_obs=n_obs, dim_x=dim_x, mar=True, return_type="DataFrame" - ) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "score": ["missing-at-random"], - "learner_g": [("Lasso", LassoCV()), ("LGBM", LGBMRegressor(verbose=-1))], - "learner_m": [ - ("Logistic", LogisticRegressionCV()), - ("LGBM", LGBMClassifier(verbose=-1)), - ], - "learner_pi": [ - ("Logistic", LogisticRegressionCV()), - ("LGBM", LGBMClassifier(verbose=-1)), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData(datasets[i_rep], "y", "d", s_col="s") - - for score_idx, score in enumerate(hyperparam_dict["score"]): - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - for learner_pi_idx, (learner_pi_name, ml_pi) in enumerate( - hyperparam_dict["learner_pi"] - ): - - dml_ssm = dml.DoubleMLSSM( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - ml_pi=ml_pi, - score=score, - ) - dml_ssm.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_ssm.confint(level=level) - coverage = (confint.iloc[0, 0] < theta) & ( - theta < confint.iloc[0, 1] - ) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - - confint.iloc[0, 0], - "Bias": abs(dml_ssm.coef[0] - theta), - "score": score, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "Learner pi": learner_pi_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby( - ["Learner g", "Learner m", "Learner pi", "score", "level"] - ) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "ssm_mar_ate_coverage.py" -path = "results/irm/ssm_mar_ate_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/irm/ssm_nonignorable_ate_coverage.py b/scripts/irm/ssm_nonignorable_ate_coverage.py deleted file mode 100644 index fa57b3e..0000000 --- a/scripts/irm/ssm_nonignorable_ate_coverage.py +++ /dev/null @@ -1,148 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from lightgbm import LGBMRegressor, LGBMClassifier -from sklearn.linear_model import LassoCV, LogisticRegressionCV - -import doubleml as dml -from doubleml.datasets import make_ssm_data - - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -theta = 1.0 -n_obs = 500 -dim_x = 20 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) - -datasets = [] -for i in range(n_rep): - data = make_ssm_data( - theta=theta, n_obs=n_obs, dim_x=dim_x, mar=False, return_type="DataFrame" - ) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "score": ["nonignorable"], - "learner_g": [("Lasso", LassoCV()), ("LGBM", LGBMRegressor(verbose=-1))], - "learner_m": [ - ("Logistic", LogisticRegressionCV()), - ("LGBM", LGBMClassifier(verbose=-1)), - ], - "learner_pi": [ - ("Logistic", LogisticRegressionCV()), - ("LGBM", LGBMClassifier(verbose=-1)), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData(datasets[i_rep], "y", "d", z_cols="z", s_col="s") - - for score_idx, score in enumerate(hyperparam_dict["score"]): - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - for learner_pi_idx, (learner_pi_name, ml_pi) in enumerate( - hyperparam_dict["learner_pi"] - ): - - dml_ssm = dml.DoubleMLSSM( - obj_dml_data=obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - ml_pi=ml_pi, - score=score, - ) - dml_ssm.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_ssm.confint(level=level) - coverage = (confint.iloc[0, 0] < theta) & ( - theta < confint.iloc[0, 1] - ) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - - confint.iloc[0, 0], - "Bias": abs(dml_ssm.coef[0] - theta), - "score": score, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "Learner pi": learner_pi_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby( - ["Learner g", "Learner m", "Learner pi", "score", "level"] - ) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "ssm_nonignorable_ate_coverage.py" -path = "results/irm/ssm_nonignorable_ate_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/plm/pliv_late_coverage.py b/scripts/plm/pliv_late_coverage.py deleted file mode 100644 index ee25015..0000000 --- a/scripts/plm/pliv_late_coverage.py +++ /dev/null @@ -1,178 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from sklearn.ensemble import RandomForestRegressor -from sklearn.linear_model import LassoCV - -import doubleml as dml -from doubleml.datasets import make_pliv_CHS2015 - - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -theta = 0.5 -n_obs = 500 -dim_x = 20 -dim_z = 1 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) - -datasets = [] -for i in range(n_rep): - data = make_pliv_CHS2015( - alpha=theta, n_obs=n_obs, dim_x=dim_x, dim_z=dim_z, return_type="DataFrame" - ) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "score": ["partialling out", "IV-type"], - "learner_g": [ - ("Lasso", LassoCV()), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "learner_m": [ - ("Lasso", LassoCV()), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "learner_r": [ - ("Lasso", LassoCV()), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData(datasets[i_rep], "y", "d", z_cols="Z1") - - for score_idx, score in enumerate(hyperparam_dict["score"]): - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - for learner_r_idx, (learner_r_name, ml_r) in enumerate( - hyperparam_dict["learner_r"] - ): - if score == "IV-type": - # Set machine learning methods for g & m - dml_pliv = dml.DoubleMLPLIV( - obj_dml_data=obj_dml_data, - ml_l=ml_g, - ml_m=ml_m, - ml_g=ml_g, - ml_r=ml_r, - score="IV-type", - ) - else: - # Set machine learning methods for g & m - dml_pliv = dml.DoubleMLPLIV( - obj_dml_data=obj_dml_data, - ml_l=ml_g, - ml_m=ml_m, - ml_r=ml_r, - score=score, - ) - dml_pliv.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_pliv.confint(level=level) - coverage = (confint.iloc[0, 0] < theta) & ( - theta < confint.iloc[0, 1] - ) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - - confint.iloc[0, 0], - "Bias": abs(dml_pliv.coef[0] - theta), - "score": score, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "Learner r": learner_r_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby( - ["Learner g", "Learner m", "Learner r", "score", "level"] - ) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "pliv_late_coverage.py" -path = "results/plm/pliv_late_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/plm/plr_ate_coverage.py b/scripts/plm/plr_ate_coverage.py deleted file mode 100644 index f0dec11..0000000 --- a/scripts/plm/plr_ate_coverage.py +++ /dev/null @@ -1,160 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from sklearn.ensemble import RandomForestRegressor -from sklearn.linear_model import LassoCV - -import doubleml as dml -from doubleml.datasets import make_plr_CCDDHNR2018 - - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -theta = 0.5 -n_obs = 500 -dim_x = 20 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) - -datasets = [] -for i in range(n_rep): - data = make_plr_CCDDHNR2018( - alpha=theta, n_obs=n_obs, dim_x=dim_x, return_type="DataFrame" - ) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "score": ["partialling out", "IV-type"], - "learner_g": [ - ("Lasso", LassoCV()), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "learner_m": [ - ("Lasso", LassoCV()), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData(datasets[i_rep], "y", "d") - - for score_idx, score in enumerate(hyperparam_dict["score"]): - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - if score == "IV-type": - # Set machine learning methods for g & m - dml_plr = dml.DoubleMLPLR( - obj_dml_data=obj_dml_data, - ml_l=ml_g, - ml_m=ml_m, - ml_g=ml_g, - score="IV-type", - ) - else: - # Set machine learning methods for g & m - dml_plr = dml.DoubleMLPLR( - obj_dml_data=obj_dml_data, - ml_l=ml_g, - ml_m=ml_m, - score=score, - ) - dml_plr.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = dml_plr.confint(level=level) - coverage = (confint.iloc[0, 0] < theta) & ( - theta < confint.iloc[0, 1] - ) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - - confint.iloc[0, 0], - "Bias": abs(dml_plr.coef[0] - theta), - "score": score, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "score", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "plr_ate_coverage.py" -path = "results/plm/plr_ate_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/plm/plr_ate_sensitivity_old.py b/scripts/plm/plr_ate_sensitivity_old.py deleted file mode 100644 index 7557435..0000000 --- a/scripts/plm/plr_ate_sensitivity_old.py +++ /dev/null @@ -1,240 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from sklearn.ensemble import RandomForestRegressor -from lightgbm import LGBMRegressor - -import doubleml as dml -from doubleml.datasets import make_confounded_plr_data - - -# Number of repetitions -n_rep = 500 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 1000 -cf_y = 0.1 -cf_d = 0.1 -theta = 5.0 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) - -# test inputs -dgp_dict = make_confounded_plr_data(n_obs=int(1e6), cf_y=cf_y, cf_d=cf_d) -oracle_dict = dgp_dict["oracle_values"] - -cf_y_test = np.mean( - np.square(oracle_dict["g_long"] - oracle_dict["g_short"]) -) / np.mean(np.square(dgp_dict["y"] - oracle_dict["g_short"])) -print(f"Input cf_y:{cf_y} \nCalculated cf_y: {round(cf_y_test, 5)}") - -rr_long = (dgp_dict["d"] - oracle_dict["m_long"]) / np.mean( - np.square(dgp_dict["d"] - oracle_dict["m_long"]) -) -rr_short = (dgp_dict["d"] - oracle_dict["m_short"]) / np.mean( - np.square(dgp_dict["d"] - oracle_dict["m_short"]) -) -C2_D = (np.mean(np.square(rr_long)) - np.mean(np.square(rr_short))) / np.mean( - np.square(rr_short) -) -cf_d_test = C2_D / (1 + C2_D) -print(f"Input cf_d:{cf_d}\nCalculated cf_d: {round(cf_d_test, 5)}") - -# compute the value for rho -rho = np.corrcoef( - (oracle_dict["g_long"] - oracle_dict["g_short"]), (rr_long - rr_short) -)[0, 1] -print(f"Correlation rho: {round(rho, 5)}") - -datasets = [] -for i in range(n_rep): - data = make_confounded_plr_data(n_obs=n_obs, cf_y=cf_y, cf_d=cf_d, theta=theta) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "score": ["partialling out", "IV-type"], - "learner_g": [ - ( - "LGBM", - LGBMRegressor( - n_estimators=500, learning_rate=0.05, min_child_samples=5, verbose=-1 - ), - ), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "learner_m": [ - ( - "LGBM", - LGBMRegressor( - n_estimators=500, learning_rate=0.05, min_child_samples=2, verbose=-1 - ), - ), - ( - "Random Forest", - RandomForestRegressor( - n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2 - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep + 1}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - dgp_dict = datasets[i_rep] - x_cols = [f"X{i + 1}" for i in np.arange(dgp_dict["x"].shape[1])] - df = pd.DataFrame( - np.column_stack((dgp_dict["x"], dgp_dict["y"], dgp_dict["d"])), - columns=x_cols + ["y", "d"], - ) - obj_dml_data = dml.DoubleMLData(df, "y", "d") - - for score_idx, score in enumerate(hyperparam_dict["score"]): - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - if score == "IV-type": - # Set machine learning methods for g & m - dml_plr = dml.DoubleMLPLR( - obj_dml_data=obj_dml_data, - ml_l=ml_g, - ml_m=ml_m, - ml_g=ml_g, - score="IV-type", - ) - else: - # Set machine learning methods for g & m - dml_plr = dml.DoubleMLPLR( - obj_dml_data=obj_dml_data, - ml_l=ml_g, - ml_m=ml_m, - score=score, - ) - dml_plr.fit(n_jobs_cv=5) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - - estimate = dml_plr.coef[0] - confint = dml_plr.confint(level=level) - coverage = (confint.iloc[0, 0] < theta) & ( - theta < confint.iloc[0, 1] - ) - ci_length = confint.iloc[0, 1] - confint.iloc[0, 0] - - # test sensitivity parameters - dml_plr.sensitivity_analysis( - cf_y=cf_y, - cf_d=cf_d, - rho=rho, - level=level, - null_hypothesis=theta, - ) - cover_lower = theta >= dml_plr.sensitivity_params["ci"]["lower"] - cover_upper = theta <= dml_plr.sensitivity_params["ci"]["upper"] - rv = dml_plr.sensitivity_params["rv"] - rva = dml_plr.sensitivity_params["rva"] - bias_lower = abs( - theta - dml_plr.sensitivity_params["theta"]["lower"] - ) - bias_upper = abs( - theta - dml_plr.sensitivity_params["theta"]["upper"] - ) - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": confint.iloc[0, 1] - - confint.iloc[0, 0], - "Bias": abs(estimate - theta), - "Coverage (Lower)": cover_lower.astype(int), - "Coverage (Upper)": cover_upper.astype(int), - "RV": rv, - "RVa": rva, - "Bias (Lower)": bias_lower, - "Bias (Upper)": bias_upper, - "score": score, - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "score", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Coverage (Lower)": "mean", - "Coverage (Upper)": "mean", - "RV": "mean", - "RVa": "mean", - "Bias (Lower)": "mean", - "Bias (Upper)": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "plr_ate_sensitivity.py" -path = "results/plm/plr_ate_sensitivity" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/plm/plr_cate_coverage.py b/scripts/plm/plr_cate_coverage.py deleted file mode 100644 index 0009040..0000000 --- a/scripts/plm/plr_cate_coverage.py +++ /dev/null @@ -1,161 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys -import patsy - -from lightgbm import LGBMRegressor -from sklearn.linear_model import LassoCV - -import doubleml as dml -from doubleml.datasets import make_heterogeneous_data - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 2000 -p = 10 -support_size = 5 -n_x = 1 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - data = make_heterogeneous_data( - n_obs=n_obs, p=p, support_size=support_size, n_x=n_x, binary_treatment=False - ) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Lasso", LassoCV()), - ("LGBM", LGBMRegressor(n_estimators=200, learning_rate=0.05, verbose=-1)), - ], - "learner_m": [ - ("Lasso", LassoCV()), - ("LGBM", LGBMRegressor(n_estimators=200, learning_rate=0.05, verbose=-1)), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - data = datasets[i_rep]["data"] - design_matrix = patsy.dmatrix("bs(x, df=5, degree=2)", {"x": data["X_0"]}) - spline_basis = pd.DataFrame(design_matrix) - - true_effects = datasets[i_rep]["effects"] - - obj_dml_data = dml.DoubleMLData(data, "y", "d") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_plr = dml.DoubleMLPLR( - obj_dml_data=obj_dml_data, - ml_l=ml_g, - ml_m=ml_m, - ) - dml_plr.fit(n_jobs_cv=5) - cate = dml_plr.cate(spline_basis) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = cate.confint(basis=spline_basis, level=level) - effects = confint["effect"] - coverage = (confint.iloc[:, 0] < true_effects) & ( - true_effects < confint.iloc[:, 2] - ) - ci_length = confint.iloc[:, 2] - confint.iloc[:, 0] - confint_uniform = cate.confint( - basis=spline_basis, level=0.95, joint=True, n_rep_boot=2000 - ) - coverage_uniform = all( - (confint_uniform.iloc[:, 0] < true_effects) - & (true_effects < confint_uniform.iloc[:, 2]) - ) - ci_length_uniform = ( - confint_uniform.iloc[:, 2] - confint_uniform.iloc[:, 0] - ) - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.mean(), - "CI Length": ci_length.mean(), - "Bias": abs(effects - true_effects).mean(), - "Uniform Coverage": coverage_uniform, - "Uniform CI Length": ci_length_uniform.mean(), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Uniform Coverage": "mean", - "Uniform CI Length": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "plr_cate_coverage.py" -path = "results/plm/plr_cate_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/plm/plr_gate_coverage.py b/scripts/plm/plr_gate_coverage.py deleted file mode 100644 index 6709ccd..0000000 --- a/scripts/plm/plr_gate_coverage.py +++ /dev/null @@ -1,167 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from lightgbm import LGBMRegressor -from sklearn.linear_model import LassoCV - -import doubleml as dml -from doubleml.datasets import make_heterogeneous_data - -# Number of repetitions -n_rep = 1000 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 500 -p = 10 -support_size = 5 -n_x = 1 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) -datasets = [] -for i in range(n_rep): - data = make_heterogeneous_data( - n_obs=n_obs, p=p, support_size=support_size, n_x=n_x, binary_treatment=False - ) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "learner_g": [ - ("Lasso", LassoCV()), - ("LGBM", LGBMRegressor(n_estimators=200, learning_rate=0.05, verbose=-1)), - ], - "learner_m": [ - ("Lasso", LassoCV()), - ("LGBM", LGBMRegressor(n_estimators=200, learning_rate=0.05, verbose=-1)), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - # define the DoubleML data object - data = datasets[i_rep]["data"] - ite = datasets[i_rep]["effects"] - - groups = pd.DataFrame( - np.column_stack( - ( - data["X_0"] <= 0.3, - (data["X_0"] > 0.3) & (data["X_0"] <= 0.7), - data["X_0"] > 0.7, - ) - ), - columns=["Group 1", "Group 2", "Group 3"], - ) - true_effects = [ite[groups[group]].mean() for group in groups.columns] - - obj_dml_data = dml.DoubleMLData(data, "y", "d") - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - # Set machine learning methods for g & m - dml_plr = dml.DoubleMLPLR( - obj_dml_data=obj_dml_data, - ml_l=ml_g, - ml_m=ml_m, - ) - dml_plr.fit(n_jobs_cv=5) - gate = dml_plr.gate(groups=groups) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = gate.confint(level=level) - effects = confint["effect"] - coverage = (confint.iloc[:, 0] < true_effects) & ( - true_effects < confint.iloc[:, 2] - ) - ci_length = confint.iloc[:, 2] - confint.iloc[:, 0] - confint_uniform = gate.confint(level=0.95, joint=True, n_rep_boot=2000) - coverage_uniform = all( - (confint_uniform.iloc[:, 0] < true_effects) - & (true_effects < confint_uniform.iloc[:, 2]) - ) - ci_length_uniform = ( - confint_uniform.iloc[:, 2] - confint_uniform.iloc[:, 0] - ) - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.mean(), - "CI Length": ci_length.mean(), - "Bias": abs(effects - true_effects).mean(), - "Uniform Coverage": coverage_uniform, - "Uniform CI Length": ci_length_uniform.mean(), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Learner g", "Learner m", "level"]) - .agg( - { - "Coverage": "mean", - "CI Length": "mean", - "Bias": "mean", - "Uniform Coverage": "mean", - "Uniform CI Length": "mean", - "repetition": "count", - } - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "plr_gate_coverage.py" -path = "results/plm/plr_gate_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/rdd/rdd_fuzzy_coverage.py b/scripts/rdd/rdd_fuzzy_coverage.py deleted file mode 100644 index 20b49ff..0000000 --- a/scripts/rdd/rdd_fuzzy_coverage.py +++ /dev/null @@ -1,241 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from lightgbm import LGBMRegressor, LGBMClassifier -from sklearn.ensemble import StackingRegressor, StackingClassifier -from sklearn.linear_model import LinearRegression, Ridge, LogisticRegression -from rdrobust import rdrobust - -import doubleml as dml -from doubleml.rdd import RDFlex -from doubleml.rdd.datasets import make_simple_rdd_data -from doubleml.utils import GlobalRegressor, GlobalClassifier - -from statsmodels.nonparametric.kernel_regression import KernelReg - - -# Number of repetitions -n_rep = 500 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 2000 -cutoff = 0 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) - -datasets = [] -for i in range(n_rep): - data = make_simple_rdd_data(n_obs=n_obs, fuzzy=True, cutoff=cutoff) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "fs_specification": ["cutoff", "cutoff and score", "interacted cutoff and score"], - "learner_g": [ - ("Linear", LinearRegression()), - ( - "LGBM", - LGBMRegressor(n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1), - ), - ("Global linear", GlobalRegressor(LinearRegression())), - ( - "Stacked", - StackingRegressor( - estimators=[ - ("lr", LinearRegression()), - ( - "lgbm", - LGBMRegressor( - n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1 - ), - ), - ("glr", GlobalRegressor(LinearRegression())), - ], - final_estimator=Ridge(), - ), - ), - ], - "learner_m": [ - ("Linear", LogisticRegression()), - ( - "LGBM", - LGBMClassifier( - n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1 - ), - ), - ("Global linear", GlobalClassifier(LogisticRegression())), - ( - "Stacked", - StackingClassifier( - estimators=[ - ("lr", LogisticRegression()), - ( - "lgbm", - LGBMClassifier( - n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1 - ), - ), - ("glr", GlobalClassifier(LogisticRegression())), - ], - final_estimator=LogisticRegression(), - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - data = datasets[i_rep] - # get oracle value - score = data["score"] - complier_mask = ((data["D"] == 0) & (data["score"] < cutoff)) | ( - (data["D"] == 1) & (data["score"] > cutoff) - ) - - ite = data["oracle_values"]["Y1"] - data["oracle_values"]["Y0"] - kernel_reg = KernelReg( - endog=ite[complier_mask], exog=score[complier_mask], var_type="c", reg_type="ll" - ) - effect_at_cutoff, _ = kernel_reg.fit(np.array([cutoff])) - oracle_effect = effect_at_cutoff[0] - - Y = data["Y"] - Z = data["X"].reshape(n_obs, -1) - D = data["D"] - - # baseline - for level_idx, level in enumerate(hyperparam_dict["level"]): - res = rdrobust(y=Y, x=score, fuzzy=D, covs=Z, c=cutoff, level=level * 100) - coef = res.coef.loc["Robust", "Coeff"] - ci_lower = res.ci.loc["Robust", "CI Lower"] - ci_upper = res.ci.loc["Robust", "CI Upper"] - - coverage = (ci_lower < oracle_effect) & (oracle_effect < ci_upper) - ci_length = ci_upper - ci_lower - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": ci_length, - "Bias": abs(coef - oracle_effect), - "Learner g": "linear", - "Learner m": "linear", - "Method": "rdrobust", - "fs specification": "cutoff", - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData.from_arrays(y=Y, d=D, x=Z, s=score) - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for learner_m_idx, (learner_m_name, ml_m) in enumerate( - hyperparam_dict["learner_m"] - ): - for fs_specification_idx, fs_specification in enumerate( - hyperparam_dict["fs_specification"] - ): - rdflex_model = RDFlex( - obj_dml_data, - ml_g=ml_g, - ml_m=ml_m, - n_folds=5, - n_rep=1, - cutoff=cutoff, - fuzzy=True, - fs_specification=fs_specification, - ) - rdflex_model.fit(n_iterations=2) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = rdflex_model.confint(level=level) - coverage = (confint.iloc[2, 0] < oracle_effect) & ( - oracle_effect < confint.iloc[2, 1] - ) - ci_length = confint.iloc[2, 1] - confint.iloc[2, 0] - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": ci_length, - "Bias": abs(rdflex_model.coef[2] - oracle_effect), - "Learner g": learner_g_name, - "Learner m": learner_m_name, - "Method": "rdflex", - "fs specification": fs_specification, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby( - ["Method", "fs specification", "Learner g", "Learner m", "level"] - ) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "rdd_fuzzy_coverage.py" -path = "results/rdd/rdd_fuzzy_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) diff --git a/scripts/rdd/rdd_sharp_coverage.py b/scripts/rdd/rdd_sharp_coverage.py deleted file mode 100644 index 1f33a5e..0000000 --- a/scripts/rdd/rdd_sharp_coverage.py +++ /dev/null @@ -1,202 +0,0 @@ -import numpy as np -import pandas as pd -from datetime import datetime -import time -import sys - -from lightgbm import LGBMRegressor -from sklearn.ensemble import StackingRegressor -from sklearn.linear_model import LinearRegression, Ridge -from rdrobust import rdrobust - -import doubleml as dml -from doubleml.rdd import RDFlex -from doubleml.rdd.datasets import make_simple_rdd_data -from doubleml.utils import GlobalRegressor - -from statsmodels.nonparametric.kernel_regression import KernelReg - - -# Number of repetitions -n_rep = 500 -max_runtime = 5.5 * 3600 # 5.5 hours in seconds - -# DGP pars -n_obs = 1000 -cutoff = 0 - -# to get the best possible comparison between different learners (and settings) we first simulate all datasets -np.random.seed(42) - -datasets = [] -for i in range(n_rep): - data = make_simple_rdd_data(n_obs=n_obs, fuzzy=False, cutoff=cutoff) - datasets.append(data) - -# set up hyperparameters -hyperparam_dict = { - "fs_specification": ["cutoff", "cutoff and score", "interacted cutoff and score"], - "learner_g": [ - ("Linear", LinearRegression()), - ( - "LGBM", - LGBMRegressor(n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1), - ), - ("Global linear", GlobalRegressor(LinearRegression())), - ( - "Stacked", - StackingRegressor( - estimators=[ - ("lr", LinearRegression()), - ( - "lgbm", - LGBMRegressor( - n_estimators=100, max_depth=5, learning_rate=0.1, verbose=-1 - ), - ), - ("glr", GlobalRegressor(LinearRegression())), - ], - final_estimator=Ridge(), - ), - ), - ], - "level": [0.95, 0.90], -} - -# set up the results dataframe -df_results_detailed = pd.DataFrame() - -# start simulation -np.random.seed(42) -start_time = time.time() - -for i_rep in range(n_rep): - print(f"Repetition: {i_rep}/{n_rep}", end="\r") - - # Check the elapsed time - elapsed_time = time.time() - start_time - if elapsed_time > max_runtime: - print("Maximum runtime exceeded. Stopping the simulation.") - break - - data = datasets[i_rep] - # get oracle value - score = data["score"] - ite = data["oracle_values"]["Y1"] - data["oracle_values"]["Y0"] - - kernel_reg = KernelReg(endog=ite, exog=score, var_type="c", reg_type="ll") - effect_at_cutoff, _ = kernel_reg.fit(np.array([cutoff])) - oracle_effect = effect_at_cutoff[0] - - Y = data["Y"] - Z = data["X"].reshape(n_obs, -1) - D = data["D"] - - # baseline - for level_idx, level in enumerate(hyperparam_dict["level"]): - res = rdrobust(y=Y, x=score, covs=Z, c=cutoff, level=level * 100) - coef = res.coef.loc["Robust", "Coeff"] - ci_lower = res.ci.loc["Robust", "CI Lower"] - ci_upper = res.ci.loc["Robust", "CI Upper"] - - coverage = (ci_lower < oracle_effect) & (oracle_effect < ci_upper) - ci_length = ci_upper - ci_lower - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": ci_length, - "Bias": abs(coef - oracle_effect), - "Learner g": "linear", - "Method": "rdrobust", - "fs specification": "cutoff", - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - - # define the DoubleML data object - obj_dml_data = dml.DoubleMLData.from_arrays(y=Y, d=D, x=Z, s=score) - - for learner_g_idx, (learner_g_name, ml_g) in enumerate( - hyperparam_dict["learner_g"] - ): - for fs_specification_idx, fs_specification in enumerate( - hyperparam_dict["fs_specification"] - ): - rdflex_model = RDFlex( - obj_dml_data, - ml_g=ml_g, - n_folds=5, - n_rep=1, - cutoff=cutoff, - fuzzy=False, - fs_specification=fs_specification, - ) - rdflex_model.fit(n_iterations=2) - - for level_idx, level in enumerate(hyperparam_dict["level"]): - confint = rdflex_model.confint(level=level) - coverage = (confint.iloc[2, 0] < oracle_effect) & ( - oracle_effect < confint.iloc[2, 1] - ) - ci_length = confint.iloc[2, 1] - confint.iloc[2, 0] - - df_results_detailed = pd.concat( - ( - df_results_detailed, - pd.DataFrame( - { - "Coverage": coverage.astype(int), - "CI Length": ci_length, - "Bias": abs(rdflex_model.coef[2] - oracle_effect), - "Learner g": learner_g_name, - "Method": "rdflex", - "fs specification": fs_specification, - "level": level, - "repetition": i_rep, - }, - index=[0], - ), - ), - ignore_index=True, - ) - -df_results = ( - df_results_detailed.groupby(["Method", "fs specification", "Learner g", "level"]) - .agg( - {"Coverage": "mean", "CI Length": "mean", "Bias": "mean", "repetition": "count"} - ) - .reset_index() -) -print(df_results) - -end_time = time.time() -total_runtime = end_time - start_time - -# save results -script_name = "rdd_sharp_coverage.py" -path = "results/rdd/rdd_sharp_coverage" - -metadata = pd.DataFrame( - { - "DoubleML Version": [dml.__version__], - "Script": [script_name], - "Date": [datetime.now().strftime("%Y-%m-%d %H:%M:%S")], - "Total Runtime (seconds)": [total_runtime], - "Python Version": [ - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ], - } -) -print(metadata) - -df_results.to_csv(f"{path}.csv", index=False) -metadata.to_csv(f"{path}_metadata.csv", index=False) From d891567c902ca4f1babe5e8555b7e28eb1903757 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Thu, 5 Jun 2025 14:05:50 +0200 Subject: [PATCH 02/35] remove results --- results/did/did_cs_atte_coverage.csv | 49 ------------ results/did/did_cs_atte_coverage_metadata.csv | 2 - results/did/did_multi_detailed.csv | 49 ------------ results/did/did_multi_eventstudy.csv | 49 ------------ results/did/did_multi_group.csv | 49 ------------ results/did/did_multi_metadata.csv | 2 - results/did/did_multi_time.csv | 49 ------------ results/did/did_pa_atte_coverage.csv | 49 ------------ results/did/did_pa_atte_coverage_metadata.csv | 2 - results/did/did_pa_multi_config.yml | 43 ----------- results/did/did_pa_multi_coverage.csv | 25 ------- .../did/did_pa_multi_coverage_metadata.csv | 2 - results/irm/apo_config.yml | 49 ------------ results/irm/apo_coverage.csv | 25 ------- results/irm/apo_metadata.csv | 2 - results/irm/apos_causal_contrast.csv | 9 --- results/irm/apos_config.yml | 49 ------------ results/irm/apos_coverage.csv | 9 --- results/irm/apos_metadata.csv | 2 - results/irm/cvar_Y0_coverage.csv | 9 --- results/irm/cvar_Y1_coverage.csv | 9 --- results/irm/cvar_config.yml | 65 ---------------- results/irm/cvar_coverage_metadata.csv | 2 - results/irm/cvar_coverage_pq0.csv | 9 --- results/irm/cvar_coverage_pq1.csv | 9 --- results/irm/cvar_coverage_qte.csv | 9 --- results/irm/cvar_effect_coverage.csv | 9 --- results/irm/cvar_metadata.csv | 2 - results/irm/iivm_late_coverage.csv | 9 --- results/irm/iivm_late_coverage_metadata.csv | 2 - results/irm/irm_apo_coverage_apo.csv | 25 ------- results/irm/irm_apo_coverage_apos.csv | 9 --- .../irm/irm_apo_coverage_apos_contrast.csv | 9 --- results/irm/irm_apo_coverage_metadata.csv | 2 - results/irm/irm_ate_config.yml | 61 --------------- results/irm/irm_ate_coverage.csv | 15 ---- results/irm/irm_ate_coverage_metadata.csv | 2 - results/irm/irm_ate_metadata.csv | 2 - results/irm/irm_ate_sensitivity_config.yml | 53 ------------- results/irm/irm_ate_sensitivity_coverage.csv | 9 --- results/irm/irm_ate_sensitivity_metadata.csv | 2 - results/irm/irm_atte_config.yml | 61 --------------- results/irm/irm_atte_coverage.csv | 15 ---- results/irm/irm_atte_coverage_metadata.csv | 2 - results/irm/irm_atte_metadata.csv | 2 - results/irm/irm_atte_sensitivity_config.yml | 53 ------------- results/irm/irm_atte_sensitivity_coverage.csv | 9 --- results/irm/irm_atte_sensitivity_metadata.csv | 2 - results/irm/irm_cate_config.yml | 63 ---------------- results/irm/irm_cate_coverage.csv | 15 ---- results/irm/irm_cate_coverage_metadata.csv | 2 - results/irm/irm_cate_metadata.csv | 2 - results/irm/irm_gate_config.yml | 63 ---------------- results/irm/irm_gate_coverage.csv | 15 ---- results/irm/irm_gate_coverage_metadata.csv | 2 - results/irm/irm_gate_metadata.csv | 2 - results/irm/lpq_Y0_coverage.csv | 9 --- results/irm/lpq_Y1_coverage.csv | 9 --- results/irm/lpq_config.yml | 48 ------------ results/irm/lpq_coverage_lpq0.csv | 9 --- results/irm/lpq_coverage_lpq1.csv | 9 --- results/irm/lpq_coverage_lqte.csv | 9 --- results/irm/lpq_coverage_metadata.csv | 2 - results/irm/lpq_effect_coverage.csv | 9 --- results/irm/lpq_metadata.csv | 2 - results/irm/pq_Y0_coverage.csv | 9 --- results/irm/pq_Y1_coverage.csv | 9 --- results/irm/pq_config.yml | 50 ------------- results/irm/pq_coverage_metadata.csv | 2 - results/irm/pq_coverage_pq0.csv | 9 --- results/irm/pq_coverage_pq1.csv | 9 --- results/irm/pq_coverage_qte.csv | 9 --- results/irm/pq_effect_coverage.csv | 9 --- results/irm/pq_metadata.csv | 2 - results/irm/ssm_mar_ate_coverage.csv | 17 ----- results/irm/ssm_mar_ate_coverage_metadata.csv | 2 - results/irm/ssm_nonignorable_ate_coverage.csv | 17 ----- ...ssm_nonignorable_ate_coverage_metadata.csv | 2 - results/plm/pliv_late_config.yml | 57 -------------- results/plm/pliv_late_coverage.csv | 33 --------- results/plm/pliv_late_coverage_metadata.csv | 2 - results/plm/pliv_late_metadata.csv | 2 - results/plm/plr_ate_config.yml | 50 ------------- results/plm/plr_ate_coverage.csv | 29 -------- results/plm/plr_ate_coverage_metadata.csv | 2 - results/plm/plr_ate_metadata.csv | 2 - results/plm/plr_ate_sensitivity_config.yml | 49 ------------ results/plm/plr_ate_sensitivity_coverage.csv | 29 -------- results/plm/plr_ate_sensitivity_metadata.csv | 2 - results/plm/plr_cate_config.yml | 52 ------------- results/plm/plr_cate_coverage.csv | 29 -------- results/plm/plr_cate_coverage_metadata.csv | 2 - results/plm/plr_cate_metadata.csv | 2 - results/plm/plr_gate_config.yml | 52 ------------- results/plm/plr_gate_coverage.csv | 29 -------- results/plm/plr_gate_coverage_metadata.csv | 2 - results/plm/plr_gate_metadata.csv | 2 - results/rdd/rdd_fuzzy_config.yml | 63 ---------------- results/rdd/rdd_fuzzy_coverage.csv | 27 ------- results/rdd/rdd_fuzzy_coverage_metadata.csv | 2 - results/rdd/rdd_fuzzy_metadata.csv | 2 - results/rdd/rdd_sharp_config.yml | 41 ---------- results/rdd/rdd_sharp_coverage.csv | 27 ------- results/rdd/rdd_sharp_coverage_metadata.csv | 2 - results/rdd/rdd_sharp_metadata.csv | 2 - results/ssm/ssm_mar_ate_config.yml | 74 ------------------- results/ssm/ssm_mar_ate_coverage.csv | 19 ----- results/ssm/ssm_mar_ate_metadata.csv | 2 - results/ssm/ssm_nonig_ate_config.yml | 74 ------------------- results/ssm/ssm_nonig_ate_coverage.csv | 19 ----- results/ssm/ssm_nonig_ate_metadata.csv | 2 - 111 files changed, 2181 deletions(-) delete mode 100644 results/did/did_cs_atte_coverage.csv delete mode 100644 results/did/did_cs_atte_coverage_metadata.csv delete mode 100644 results/did/did_multi_detailed.csv delete mode 100644 results/did/did_multi_eventstudy.csv delete mode 100644 results/did/did_multi_group.csv delete mode 100644 results/did/did_multi_metadata.csv delete mode 100644 results/did/did_multi_time.csv delete mode 100644 results/did/did_pa_atte_coverage.csv delete mode 100644 results/did/did_pa_atte_coverage_metadata.csv delete mode 100644 results/did/did_pa_multi_config.yml delete mode 100644 results/did/did_pa_multi_coverage.csv delete mode 100644 results/did/did_pa_multi_coverage_metadata.csv delete mode 100644 results/irm/apo_config.yml delete mode 100644 results/irm/apo_coverage.csv delete mode 100644 results/irm/apo_metadata.csv delete mode 100644 results/irm/apos_causal_contrast.csv delete mode 100644 results/irm/apos_config.yml delete mode 100644 results/irm/apos_coverage.csv delete mode 100644 results/irm/apos_metadata.csv delete mode 100644 results/irm/cvar_Y0_coverage.csv delete mode 100644 results/irm/cvar_Y1_coverage.csv delete mode 100644 results/irm/cvar_config.yml delete mode 100644 results/irm/cvar_coverage_metadata.csv delete mode 100644 results/irm/cvar_coverage_pq0.csv delete mode 100644 results/irm/cvar_coverage_pq1.csv delete mode 100644 results/irm/cvar_coverage_qte.csv delete mode 100644 results/irm/cvar_effect_coverage.csv delete mode 100644 results/irm/cvar_metadata.csv delete mode 100644 results/irm/iivm_late_coverage.csv delete mode 100644 results/irm/iivm_late_coverage_metadata.csv delete mode 100644 results/irm/irm_apo_coverage_apo.csv delete mode 100644 results/irm/irm_apo_coverage_apos.csv delete mode 100644 results/irm/irm_apo_coverage_apos_contrast.csv delete mode 100644 results/irm/irm_apo_coverage_metadata.csv delete mode 100644 results/irm/irm_ate_config.yml delete mode 100644 results/irm/irm_ate_coverage.csv delete mode 100644 results/irm/irm_ate_coverage_metadata.csv delete mode 100644 results/irm/irm_ate_metadata.csv delete mode 100644 results/irm/irm_ate_sensitivity_config.yml delete mode 100644 results/irm/irm_ate_sensitivity_coverage.csv delete mode 100644 results/irm/irm_ate_sensitivity_metadata.csv delete mode 100644 results/irm/irm_atte_config.yml delete mode 100644 results/irm/irm_atte_coverage.csv delete mode 100644 results/irm/irm_atte_coverage_metadata.csv delete mode 100644 results/irm/irm_atte_metadata.csv delete mode 100644 results/irm/irm_atte_sensitivity_config.yml delete mode 100644 results/irm/irm_atte_sensitivity_coverage.csv delete mode 100644 results/irm/irm_atte_sensitivity_metadata.csv delete mode 100644 results/irm/irm_cate_config.yml delete mode 100644 results/irm/irm_cate_coverage.csv delete mode 100644 results/irm/irm_cate_coverage_metadata.csv delete mode 100644 results/irm/irm_cate_metadata.csv delete mode 100644 results/irm/irm_gate_config.yml delete mode 100644 results/irm/irm_gate_coverage.csv delete mode 100644 results/irm/irm_gate_coverage_metadata.csv delete mode 100644 results/irm/irm_gate_metadata.csv delete mode 100644 results/irm/lpq_Y0_coverage.csv delete mode 100644 results/irm/lpq_Y1_coverage.csv delete mode 100644 results/irm/lpq_config.yml delete mode 100644 results/irm/lpq_coverage_lpq0.csv delete mode 100644 results/irm/lpq_coverage_lpq1.csv delete mode 100644 results/irm/lpq_coverage_lqte.csv delete mode 100644 results/irm/lpq_coverage_metadata.csv delete mode 100644 results/irm/lpq_effect_coverage.csv delete mode 100644 results/irm/lpq_metadata.csv delete mode 100644 results/irm/pq_Y0_coverage.csv delete mode 100644 results/irm/pq_Y1_coverage.csv delete mode 100644 results/irm/pq_config.yml delete mode 100644 results/irm/pq_coverage_metadata.csv delete mode 100644 results/irm/pq_coverage_pq0.csv delete mode 100644 results/irm/pq_coverage_pq1.csv delete mode 100644 results/irm/pq_coverage_qte.csv delete mode 100644 results/irm/pq_effect_coverage.csv delete mode 100644 results/irm/pq_metadata.csv delete mode 100644 results/irm/ssm_mar_ate_coverage.csv delete mode 100644 results/irm/ssm_mar_ate_coverage_metadata.csv delete mode 100644 results/irm/ssm_nonignorable_ate_coverage.csv delete mode 100644 results/irm/ssm_nonignorable_ate_coverage_metadata.csv delete mode 100644 results/plm/pliv_late_config.yml delete mode 100644 results/plm/pliv_late_coverage.csv delete mode 100644 results/plm/pliv_late_coverage_metadata.csv delete mode 100644 results/plm/pliv_late_metadata.csv delete mode 100644 results/plm/plr_ate_config.yml delete mode 100644 results/plm/plr_ate_coverage.csv delete mode 100644 results/plm/plr_ate_coverage_metadata.csv delete mode 100644 results/plm/plr_ate_metadata.csv delete mode 100644 results/plm/plr_ate_sensitivity_config.yml delete mode 100644 results/plm/plr_ate_sensitivity_coverage.csv delete mode 100644 results/plm/plr_ate_sensitivity_metadata.csv delete mode 100644 results/plm/plr_cate_config.yml delete mode 100644 results/plm/plr_cate_coverage.csv delete mode 100644 results/plm/plr_cate_coverage_metadata.csv delete mode 100644 results/plm/plr_cate_metadata.csv delete mode 100644 results/plm/plr_gate_config.yml delete mode 100644 results/plm/plr_gate_coverage.csv delete mode 100644 results/plm/plr_gate_coverage_metadata.csv delete mode 100644 results/plm/plr_gate_metadata.csv delete mode 100644 results/rdd/rdd_fuzzy_config.yml delete mode 100644 results/rdd/rdd_fuzzy_coverage.csv delete mode 100644 results/rdd/rdd_fuzzy_coverage_metadata.csv delete mode 100644 results/rdd/rdd_fuzzy_metadata.csv delete mode 100644 results/rdd/rdd_sharp_config.yml delete mode 100644 results/rdd/rdd_sharp_coverage.csv delete mode 100644 results/rdd/rdd_sharp_coverage_metadata.csv delete mode 100644 results/rdd/rdd_sharp_metadata.csv delete mode 100644 results/ssm/ssm_mar_ate_config.yml delete mode 100644 results/ssm/ssm_mar_ate_coverage.csv delete mode 100644 results/ssm/ssm_mar_ate_metadata.csv delete mode 100644 results/ssm/ssm_nonig_ate_config.yml delete mode 100644 results/ssm/ssm_nonig_ate_coverage.csv delete mode 100644 results/ssm/ssm_nonig_ate_metadata.csv diff --git a/results/did/did_cs_atte_coverage.csv b/results/did/did_cs_atte_coverage.csv deleted file mode 100644 index 53cf347..0000000 --- a/results/did/did_cs_atte_coverage.csv +++ /dev/null @@ -1,49 +0,0 @@ -Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,experimental,False,1,0.9,0.714,10.421426989395727,3.905582429463871,1000 -LGBM,LGBM,experimental,False,1,0.95,0.796,12.417896177537456,3.905582429463871,1000 -LGBM,LGBM,experimental,False,2,0.9,0.747,11.153655901082034,3.716762206006709,1000 -LGBM,LGBM,experimental,False,2,0.95,0.83,13.290400740757454,3.716762206006709,1000 -LGBM,LGBM,experimental,False,3,0.9,0.826,10.129500599367143,2.9806910089316485,1000 -LGBM,LGBM,experimental,False,3,0.95,0.9,12.070044428775317,2.9806910089316485,1000 -LGBM,LGBM,experimental,False,4,0.9,0.709,10.248410164509226,3.911148948219506,1000 -LGBM,LGBM,experimental,False,4,0.95,0.788,12.211733914865182,3.911148948219506,1000 -LGBM,LGBM,experimental,False,5,0.9,0.897,11.953436462004694,2.904688869350282,1000 -LGBM,LGBM,experimental,False,5,0.95,0.95,14.243398058730905,2.904688869350282,1000 -LGBM,LGBM,experimental,False,6,0.9,0.901,10.409876930645252,2.475898589693061,1000 -LGBM,LGBM,experimental,False,6,0.95,0.951,12.40413343366814,2.475898589693061,1000 -LGBM,LGBM,experimental,True,1,0.9,0.695,10.441642571924747,3.98549935766534,1000 -LGBM,LGBM,experimental,True,1,0.95,0.774,12.441984529859,3.98549935766534,1000 -LGBM,LGBM,experimental,True,2,0.9,0.769,11.14737947150305,3.7228962496196263,1000 -LGBM,LGBM,experimental,True,2,0.95,0.832,13.282921913629773,3.7228962496196263,1000 -LGBM,LGBM,experimental,True,3,0.9,0.822,10.139719900553855,2.993567832780566,1000 -LGBM,LGBM,experimental,True,3,0.95,0.896,12.082221477203783,2.993567832780566,1000 -LGBM,LGBM,experimental,True,4,0.9,0.707,10.258140130646604,3.9509918046955974,1000 -LGBM,LGBM,experimental,True,4,0.95,0.782,12.223327884618827,3.9509918046955974,1000 -LGBM,LGBM,experimental,True,5,0.9,0.894,11.981860540543671,2.9439981898378322,1000 -LGBM,LGBM,experimental,True,5,0.95,0.949,14.277267437329282,2.9439981898378322,1000 -LGBM,LGBM,experimental,True,6,0.9,0.894,10.42424549288115,2.562430198965583,1000 -LGBM,LGBM,experimental,True,6,0.95,0.955,12.421254631585413,2.562430198965583,1000 -LGBM,LGBM,observational,False,1,0.9,0.94,50.01837238134115,11.670635965681225,1000 -LGBM,LGBM,observational,False,1,0.95,0.973,59.600566777747616,11.670635965681225,1000 -LGBM,LGBM,observational,False,2,0.9,0.929,59.19235508827008,13.470175038952636,1000 -LGBM,LGBM,observational,False,2,0.95,0.977,70.53204141217991,13.470175038952636,1000 -LGBM,LGBM,observational,False,3,0.9,0.945,56.62260255116421,12.634763113659828,1000 -LGBM,LGBM,observational,False,3,0.95,0.989,67.46999240102099,12.634763113659828,1000 -LGBM,LGBM,observational,False,4,0.9,0.945,70.02798665966547,16.708878014378698,1000 -LGBM,LGBM,observational,False,4,0.95,0.982,83.44349279101235,16.708878014378698,1000 -LGBM,LGBM,observational,False,5,0.9,0.932,32.68395008367948,7.535531362351606,1000 -LGBM,LGBM,observational,False,5,0.95,0.973,38.945328621880215,7.535531362351606,1000 -LGBM,LGBM,observational,False,6,0.9,0.922,31.254676611393744,7.328062886784694,1000 -LGBM,LGBM,observational,False,6,0.95,0.96,37.24224423562365,7.328062886784694,1000 -LGBM,LGBM,observational,True,1,0.9,0.903,17.911052050251026,4.470376853620159,1000 -LGBM,LGBM,observational,True,1,0.95,0.954,21.342334885309523,4.470376853620159,1000 -LGBM,LGBM,observational,True,2,0.9,0.928,20.466840035852762,4.861276719991755,1000 -LGBM,LGBM,observational,True,2,0.95,0.965,24.387744107030723,4.861276719991755,1000 -LGBM,LGBM,observational,True,3,0.9,0.916,20.087760624155962,4.7945055222830755,1000 -LGBM,LGBM,observational,True,3,0.95,0.958,23.93604312766554,4.7945055222830755,1000 -LGBM,LGBM,observational,True,4,0.9,0.913,23.82669034521118,5.634290832938362,1000 -LGBM,LGBM,observational,True,4,0.95,0.956,28.391252681829,5.634290832938362,1000 -LGBM,LGBM,observational,True,5,0.9,0.89,16.373740873746755,4.117691170125128,1000 -LGBM,LGBM,observational,True,5,0.95,0.943,19.51051563427767,4.117691170125128,1000 -LGBM,LGBM,observational,True,6,0.9,0.891,14.987930087831806,3.7336563054022593,1000 -LGBM,LGBM,observational,True,6,0.95,0.957,17.85922023310908,3.7336563054022593,1000 diff --git a/results/did/did_cs_atte_coverage_metadata.csv b/results/did/did_cs_atte_coverage_metadata.csv deleted file mode 100644 index c055b9e..0000000 --- a/results/did/did_cs_atte_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.11.dev0,did_cs_atte_coverage.py,2025-06-02 15:42:26,12669.156663179398,3.12.3 diff --git a/results/did/did_multi_detailed.csv b/results/did/did_multi_detailed.csv deleted file mode 100644 index ab7ab8b..0000000 --- a/results/did/did_multi_detailed.csv +++ /dev/null @@ -1,49 +0,0 @@ -Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.4005833333333333,0.668417462452362,0.4514209317520677,0.069,1.001536224942987,1000 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.49541666666666667,0.7964685316542967,0.4514209317520677,0.119,1.1096223780406622,1000 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.525,0.5836476641555988,0.3357039587486384,0.192,0.8984549944516823,1000 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.6135,0.6954590868526271,0.3357039587486384,0.266,0.9889489428407441,1000 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8913333333333334,0.5798205422267835,0.1436401623087172,0.896,0.8921080477762784,1000 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9469166666666666,0.6908987898012575,0.1436401623087172,0.95,0.9823087136174479,1000 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.403,0.668486737121192,0.4516172220554326,0.064,1.0020517044274269,1000 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4925,0.7965510775135288,0.4516172220554326,0.118,1.110697612751064,1000 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.5244166666666666,0.5834979623893575,0.33563072303718966,0.202,0.8980797806262651,1000 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.6114166666666666,0.6952807061958639,0.33563072303718966,0.275,0.9886661012047989,1000 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8903333333333334,0.5798432460053057,0.1452858251598464,0.888,0.8930518639150258,1000 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9449166666666666,0.690925843021974,0.1452858251598464,0.942,0.9832578157026843,1000 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9115,2.7279550989921355,0.7081663387461643,0.951,4.239219420515267,1000 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9635833333333333,3.250558990696578,0.7081663387461643,0.988,4.6533388042213675,1000 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9074166666666666,3.5225937797559883,0.9757809691235549,0.97,5.426540882685481,1000 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9669166666666666,4.19742938055986,0.9757809691235549,0.992,5.969933156329405,1000 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9271666666666666,2.1769181957500305,0.5048068220768145,0.971,3.397149166606052,1000 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9709166666666667,2.5939580221905385,0.5048068220768145,0.996,3.7281189530691132,1000 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.90375,1.1192914856600868,0.27862527337620824,0.928,1.7467988922140234,1000 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9589166666666666,1.3337180671583386,0.27862527337620824,0.967,1.914692647031146,1000 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9199166666666666,1.4197310386522546,0.32827809352307774,0.935,2.19395277603094,1000 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9664166666666666,1.6917138752639644,0.32827809352307774,0.968,2.4119391624873256,1000 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9005,1.0294284373000027,0.2502841114417097,0.917,1.6092703629556147,1000 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9533333333333334,1.226639640579326,0.2502841114417097,0.971,1.7641639749309233,1000 -Linear,Logistic,experimental,False,1,0.9,0.8521666666666666,0.2947207725579002,0.08125759297583215,0.769,0.45936907943411753,1000 -Linear,Logistic,experimental,False,1,0.95,0.91225,0.3511814609181314,0.08125759297583215,0.857,0.5043035205016457,1000 -Linear,Logistic,experimental,False,4,0.9,0.3186666666666667,0.9766434317991218,0.8093782242316307,0.041,1.4141166689592581,1000 -Linear,Logistic,experimental,False,4,0.95,0.393,1.1637424271067696,0.8093782242316307,0.079,1.5775150069804904,1000 -Linear,Logistic,experimental,False,6,0.9,0.8959166666666666,0.9840677614600047,0.24403762906388674,0.889,1.4217412744627138,1000 -Linear,Logistic,experimental,False,6,0.95,0.9463333333333334,1.1725890615466086,0.24403762906388674,0.94,1.585826837431638,1000 -Linear,Logistic,experimental,True,1,0.9,0.8528333333333333,0.29471605836940884,0.08132561217659459,0.764,0.45923649733243366,1000 -Linear,Logistic,experimental,True,1,0.95,0.9120833333333334,0.351175843616076,0.08132561217659459,0.855,0.5041154397102058,1000 -Linear,Logistic,experimental,True,4,0.9,0.3188333333333333,0.9765337199818319,0.8092231167974461,0.041,1.4131241013020541,1000 -Linear,Logistic,experimental,True,4,0.95,0.39458333333333334,1.1636116974132316,0.8092231167974461,0.076,1.5757045631534141,1000 -Linear,Logistic,experimental,True,6,0.9,0.8968333333333334,0.984151005519869,0.24420883449452355,0.889,1.4208221899399172,1000 -Linear,Logistic,experimental,True,6,0.95,0.94675,1.1726882529619342,0.24420883449452355,0.936,1.5847814464063683,1000 -Linear,Logistic,observational,False,1,0.9,0.9001666666666667,0.3180779030235929,0.0773596882018309,0.88,0.49501077303673774,1000 -Linear,Logistic,observational,False,1,0.95,0.94775,0.3790131984933507,0.0773596882018309,0.947,0.5435398448787098,1000 -Linear,Logistic,observational,False,4,0.9,0.4245,1.237493689045589,0.7914353096522312,0.18,1.76761883251908,1000 -Linear,Logistic,observational,False,4,0.95,0.5209166666666666,1.4745646797278944,0.7914353096522312,0.275,1.9755226904473104,1000 -Linear,Logistic,observational,False,6,0.9,0.8929166666666666,1.0255283640310429,0.255700122939389,0.895,1.4816116459637685,1000 -Linear,Logistic,observational,False,6,0.95,0.9455,1.221992416644637,0.255700122939389,0.934,1.653527148130022,1000 -Linear,Logistic,observational,True,1,0.9,0.8965,0.31619901580629955,0.07736324119662329,0.885,0.49203725267016324,1000 -Linear,Logistic,observational,True,1,0.95,0.9463333333333334,0.3767743662856892,0.07736324119662329,0.947,0.5403636983384068,1000 -Linear,Logistic,observational,True,4,0.9,0.4231666666666667,1.2357474079437374,0.7915034422252056,0.193,1.7654458413619594,1000 -Linear,Logistic,observational,True,4,0.95,0.5213333333333334,1.472483857452629,0.7915034422252056,0.277,1.9735390159048785,1000 -Linear,Logistic,observational,True,6,0.9,0.8928333333333334,1.0212995120870143,0.25694449841811784,0.89,1.4744168456619084,1000 -Linear,Logistic,observational,True,6,0.95,0.9460833333333334,1.216953428755113,0.25694449841811784,0.933,1.6462184670600197,1000 diff --git a/results/did/did_multi_eventstudy.csv b/results/did/did_multi_eventstudy.csv deleted file mode 100644 index 1c294b6..0000000 --- a/results/did/did_multi_eventstudy.csv +++ /dev/null @@ -1,49 +0,0 @@ -Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.26816666666666666,0.6628004801492844,0.5217146610039974,0.063,0.8710835347098573,1000 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.356,0.7897754844217385,0.5217146610039974,0.106,0.9881445074099509,1000 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.3793333333333333,0.5430545733393948,0.3833854158756925,0.176,0.73981228970138,1000 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4698333333333333,0.6470894357680017,0.3833854158756925,0.246,0.833246241332189,1000 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8915,0.5395169482604701,0.13534305762469595,0.897,0.7348860485385167,1000 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9451666666666666,0.6428740954897616,0.13534305762469595,0.953,0.8259292882870245,1000 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.265,0.6629226425912405,0.5221091420199984,0.062,0.8722109625410929,1000 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.3551666666666667,0.7899210499496214,0.5221091420199984,0.114,0.9882984410974919,1000 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.3763333333333333,0.5429965130173356,0.3828566010032573,0.182,0.739132671094994,1000 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4711666666666667,0.6470202526271421,0.3828566010032573,0.262,0.8314461004262811,1000 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8931666666666667,0.5395318624534728,0.13758915825744059,0.893,0.7341757740643098,1000 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9463333333333334,0.6428918668468384,0.13758915825744059,0.946,0.8261153685402325,1000 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.907,2.635755403048029,0.6906219348022667,0.931,3.658094889963508,1000 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9618333333333333,3.140696276789984,0.6906219348022667,0.969,4.09528459799626,1000 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.8978333333333334,3.5932112684720123,1.044176882449884,0.953,4.920160668033086,1000 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9656666666666667,4.2815753083763015,1.044176882449884,0.982,5.521039436439466,1000 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9356666666666666,2.037043783641883,0.45673572364777687,0.967,2.8380713672583395,1000 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9768333333333333,2.4272873801354264,0.45673572364777687,0.99,3.173284984968741,1000 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9186666666666666,1.058073826869023,0.2586744612407186,0.924,1.4747488796573536,1000 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9613333333333334,1.2607727275351879,0.2586744612407186,0.965,1.649868845892826,1000 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9368333333333334,1.4039435411486991,0.3103897362727539,0.943,1.9287327010063178,1000 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9718333333333333,1.6729019116910515,0.3103897362727539,0.973,2.1651520912520748,1000 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9153333333333333,0.9537908163093017,0.2248364423186676,0.927,1.3333350854360233,1000 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9601666666666666,1.1365118562044807,0.2248364423186676,0.974,1.4916507037488054,1000 -Linear,Logistic,experimental,False,1,0.9,0.8085,0.21012919637470698,0.06454170881550668,0.737,0.3000298498259047,1000 -Linear,Logistic,experimental,False,1,0.95,0.8826666666666666,0.2503843808631616,0.06454170881550668,0.83,0.33342829695147896,1000 -Linear,Logistic,experimental,False,4,0.9,0.20066666666666666,0.9748536890898165,0.9456802642466463,0.041,1.2574525229653895,1000 -Linear,Logistic,experimental,False,4,0.95,0.26666666666666666,1.161609817132025,0.9456802642466463,0.074,1.4305039091925136,1000 -Linear,Logistic,experimental,False,6,0.9,0.8906666666666666,0.984266405297522,0.2447629940182398,0.885,1.2654037292644829,1000 -Linear,Logistic,experimental,False,6,0.95,0.9428333333333334,1.1728257602782801,0.2447629940182398,0.939,1.4411880110797295,1000 -Linear,Logistic,experimental,True,1,0.9,0.8095,0.21012765534784542,0.0646279391502723,0.733,0.29993071695819135,1000 -Linear,Logistic,experimental,True,1,0.95,0.8823333333333334,0.25038254461639875,0.0646279391502723,0.831,0.3331089494375711,1000 -Linear,Logistic,experimental,True,4,0.9,0.201,0.9747048561891234,0.9456904408930068,0.042,1.2571661797897207,1000 -Linear,Logistic,experimental,True,4,0.95,0.26666666666666666,1.1614324717924194,0.9456904408930068,0.075,1.4292891253140116,1000 -Linear,Logistic,experimental,True,6,0.9,0.8928333333333334,0.9843540987911897,0.24472186099367751,0.882,1.2667938573542303,1000 -Linear,Logistic,experimental,True,6,0.95,0.942,1.1729302535209924,0.24472186099367751,0.938,1.4425417792737238,1000 -Linear,Logistic,observational,False,1,0.9,0.8958333333333334,0.22595429878581444,0.05556582705974583,0.883,0.32193445273348237,1000 -Linear,Logistic,observational,False,1,0.95,0.9468333333333334,0.26924115344718413,0.05556582705974583,0.944,0.35775078341337774,1000 -Linear,Logistic,observational,False,4,0.9,0.325,1.2886928093204373,0.9238068984287002,0.175,1.6397882387716314,1000 -Linear,Logistic,observational,False,4,0.95,0.41933333333333334,1.535572194399471,0.9238068984287002,0.26,1.8701388333818019,1000 -Linear,Logistic,observational,False,6,0.9,0.8893333333333334,1.0302622288172734,0.2580572852924157,0.882,1.322817867501306,1000 -Linear,Logistic,observational,False,6,0.95,0.9411666666666666,1.2276331644514131,0.2580572852924157,0.934,1.5071984875399793,1000 -Linear,Logistic,observational,True,1,0.9,0.8921666666666667,0.22482298708967266,0.0554326928262382,0.882,0.3206665757666365,1000 -Linear,Logistic,observational,True,1,0.95,0.9446666666666667,0.2678931124158151,0.0554326928262382,0.941,0.3562766006057057,1000 -Linear,Logistic,observational,True,4,0.9,0.3243333333333333,1.2866810812005252,0.9246774960344024,0.177,1.6374108099983478,1000 -Linear,Logistic,observational,True,4,0.95,0.41833333333333333,1.5331750724932363,0.9246774960344024,0.258,1.869004453360348,1000 -Linear,Logistic,observational,True,6,0.9,0.8853333333333334,1.0254952185219397,0.25982703889124587,0.889,1.318157054156258,1000 -Linear,Logistic,observational,True,6,0.95,0.9436666666666667,1.2219529213345213,0.25982703889124587,0.94,1.502504660229367,1000 diff --git a/results/did/did_multi_group.csv b/results/did/did_multi_group.csv deleted file mode 100644 index e3035bc..0000000 --- a/results/did/did_multi_group.csv +++ /dev/null @@ -1,49 +0,0 @@ -Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.37966666666666665,0.7094842843054401,0.5063966456271116,0.07,0.8833695236671398,1000 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.463,0.8454026680860791,0.5063966456271116,0.119,1.0075828733731322,1000 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.48833333333333334,0.6093983296457081,0.3746967239327951,0.185,0.7748043075861907,1000 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.5676666666666667,0.7261428973215824,0.3746967239327951,0.257,0.8788079154824321,1000 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8953333333333334,0.6039293782365326,0.15156143782863715,0.905,0.7671387444581327,1000 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9506666666666667,0.7196262397785951,0.15156143782863715,0.952,0.8709721345495236,1000 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.3783333333333333,0.7096389669594401,0.5063175240399332,0.066,0.8838830720898343,1000 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4603333333333333,0.8455869838366757,0.5063175240399332,0.124,1.0089102464690358,1000 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.48533333333333334,0.609259041148495,0.37470955013293855,0.19,0.7754703791594748,1000 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.5703333333333332,0.7259769248401865,0.37470955013293855,0.27,0.8784156068436955,1000 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8876666666666666,0.6038826619307712,0.15256760709225262,0.904,0.7681471625685046,1000 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.948,0.7195705738668794,0.15256760709225262,0.954,0.8695993470187038,1000 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.915,2.6608678733015485,0.688154090500346,0.931,3.366696329353361,1000 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9656666666666667,3.1706196307305743,0.688154090500346,0.97,3.8192561069995286,1000 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.903,3.556747699127985,0.9996416099478588,0.942,4.472327607652934,1000 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.969,4.238126285623724,0.9996416099478588,0.984,5.088003820636524,1000 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9373333333333334,2.1290784823926696,0.46227527065430596,0.956,2.703189636335836,1000 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9743333333333334,2.5369534877597597,0.46227527065430596,0.991,3.0641146657072404,1000 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.91,1.0977119557447321,0.2651109928804438,0.939,1.3939185973124866,1000 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.968,1.3080044712830703,0.2651109928804438,0.981,1.580610122013623,1000 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9316666666666666,1.4368875613066505,0.30987690101867293,0.952,1.8152404100048158,1000 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.975,1.7121571329201994,0.30987690101867293,0.98,2.059654079760943,1000 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.908,1.0154733246915484,0.23971463199529405,0.929,1.2938281773467855,1000 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.96,1.210011098279508,0.23971463199529405,0.972,1.4652849487780493,1000 -Linear,Logistic,experimental,False,1,0.9,0.8176666666666667,0.2639777067863099,0.07735852107118578,0.764,0.3393389889591518,1000 -Linear,Logistic,experimental,False,1,0.95,0.891,0.3145488385988198,0.07735852107118578,0.839,0.3826999999595859,1000 -Linear,Logistic,experimental,False,4,0.9,0.31,1.0799528007716814,0.9173788544387992,0.045,1.361708244102074,1000 -Linear,Logistic,experimental,False,4,0.95,0.385,1.2868431329288754,0.9173788544387992,0.073,1.5474903461640912,1000 -Linear,Logistic,experimental,False,6,0.9,0.8963333333333334,1.0864824219772902,0.2703183844096557,0.893,1.3666259596934611,1000 -Linear,Logistic,experimental,False,6,0.95,0.949,1.2946236564879237,0.2703183844096557,0.943,1.5549441788053244,1000 -Linear,Logistic,experimental,True,1,0.9,0.8163333333333334,0.2639754882372008,0.07738204276505496,0.765,0.339331857870017,1000 -Linear,Logistic,experimental,True,1,0.95,0.8906666666666666,0.3145461950345047,0.07738204276505496,0.844,0.3833030006646118,1000 -Linear,Logistic,experimental,True,4,0.9,0.3103333333333333,1.0798559231989837,0.9168941023520879,0.046,1.3607552807065624,1000 -Linear,Logistic,experimental,True,4,0.95,0.3873333333333333,1.2867276961810175,0.9168941023520879,0.074,1.5475927725689025,1000 -Linear,Logistic,experimental,True,6,0.9,0.8986666666666666,1.0865640980568139,0.27054317418995893,0.896,1.3676856739628556,1000 -Linear,Logistic,experimental,True,6,0.95,0.95,1.2947209795394352,0.27054317418995893,0.941,1.5556519546992444,1000 -Linear,Logistic,observational,False,1,0.9,0.8956666666666666,0.28367316450513197,0.0700767685399924,0.903,0.3642537051104781,1000 -Linear,Logistic,observational,False,1,0.95,0.9523333333333334,0.3380174239825948,0.0700767685399924,0.948,0.4113211504413545,1000 -Linear,Logistic,observational,False,4,0.9,0.408,1.3776824648934105,0.9052340667640746,0.183,1.7236892476868426,1000 -Linear,Logistic,observational,False,4,0.95,0.5046666666666666,1.641609909282898,0.9052340667640746,0.277,1.964267689737536,1000 -Linear,Logistic,observational,False,6,0.9,0.8903333333333334,1.1309252917212975,0.282843171368907,0.89,1.4213783682706032,1000 -Linear,Logistic,observational,False,6,0.95,0.9493333333333334,1.3475806021033823,0.282843171368907,0.947,1.617450746310541,1000 -Linear,Logistic,observational,True,1,0.9,0.8876666666666666,0.28210363817731876,0.0702387016495634,0.9,0.36227342165859067,1000 -Linear,Logistic,observational,True,1,0.95,0.9513333333333334,0.3361472180111355,0.0702387016495634,0.945,0.40929779552981904,1000 -Linear,Logistic,observational,True,4,0.9,0.4043333333333333,1.3755156336478902,0.904918936330756,0.185,1.7226484388527301,1000 -Linear,Logistic,observational,True,4,0.95,0.497,1.6390279706032433,0.904918936330756,0.286,1.9605550323619565,1000 -Linear,Logistic,observational,True,6,0.9,0.892,1.125633507574361,0.2835718025458131,0.899,1.414839244504656,1000 -Linear,Logistic,observational,True,6,0.95,0.9473333333333334,1.341275052374208,0.2835718025458131,0.949,1.6097715181154,1000 diff --git a/results/did/did_multi_metadata.csv b/results/did/did_multi_metadata.csv deleted file mode 100644 index 86381a7..0000000 --- a/results/did/did_multi_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,DIDMultiCoverageSimulation,2025-06-03 09:09,162.553562772274,3.12.9,scripts/did/did_pa_multi_config.yml diff --git a/results/did/did_multi_time.csv b/results/did/did_multi_time.csv deleted file mode 100644 index 3dda54d..0000000 --- a/results/did/did_multi_time.csv +++ /dev/null @@ -1,49 +0,0 @@ -Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.09633333333333333,0.6730703224735359,0.5813847440636578,0.06,0.7994113182366382,1000 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.163,0.8020127563300868,0.5813847440636578,0.109,0.9222994395169547,1000 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.20866666666666667,0.545809228523173,0.4318166485465945,0.148,0.6621126753160614,1000 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.29,0.6503718098720356,0.4318166485465945,0.226,0.7607781957690535,1000 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8883333333333334,0.539451744339498,0.1374713900065768,0.888,0.6559678253242532,1000 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.943,0.6427964002257806,0.1374713900065768,0.942,0.7532136346784374,1000 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.095,0.673215344660322,0.5816575606523053,0.066,0.7987689454841812,1000 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.16966666666666666,0.8021855609240048,0.5816575606523053,0.116,0.9211448037237728,1000 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.212,0.5457365545641853,0.43118597517736307,0.158,0.662287763310214,1000 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.29633333333333334,0.6502852135087509,0.43118597517736307,0.223,0.7601307241685876,1000 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.882,0.5395052157780649,0.13917064701840512,0.889,0.6561053221874785,1000 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.938,0.6428601153747003,0.13917064701840512,0.947,0.753120590966105,1000 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.8976666666666666,2.8641489580508317,0.7473265850366615,0.92,3.5454942167899373,1000 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.962,3.41284398329212,0.7473265850366615,0.97,4.046408625403042,1000 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.8876666666666666,3.9648529818754157,1.1487073320114483,0.918,4.845766161863059,1000 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9593333333333334,4.724413723593485,1.1487073320114483,0.986,5.548216313166135,1000 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.938,2.0148113447449116,0.43840143925357783,0.955,2.5144593978114727,1000 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9783333333333334,2.4007957952232104,0.43840143925357783,0.99,2.861410408432865,1000 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9076666666666666,1.1003301479666492,0.27099358031240817,0.913,1.3627159220412297,1000 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9583333333333334,1.311124239738742,0.27099358031240817,0.97,1.5572874751641161,1000 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9333333333333333,1.4854173027590014,0.3229586888066522,0.95,1.8150130378533484,1000 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.976,1.7699838865395685,0.3229586888066522,0.978,2.0803451880507104,1000 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9163333333333333,0.9353518863595044,0.219408785264352,0.912,1.1653988211882675,1000 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9603333333333334,1.1145405160056325,0.219408785264352,0.959,1.3283422144863115,1000 -Linear,Logistic,experimental,False,1,0.9,0.7966666666666666,0.24428496421942136,0.07478155567895448,0.728,0.3129228235483047,1000 -Linear,Logistic,experimental,False,1,0.95,0.8733333333333334,0.2910834885181229,0.07478155567895448,0.833,0.35375833173558197,1000 -Linear,Logistic,experimental,False,4,0.9,0.038,0.9678137745111298,1.0814213490525186,0.03,1.1087428056263964,1000 -Linear,Logistic,experimental,False,4,0.95,0.06666666666666667,1.1532212415150949,1.0814213490525186,0.054,1.2896096218626036,1000 -Linear,Logistic,experimental,False,6,0.9,0.8883333333333334,0.9645120733919993,0.2398038156236055,0.884,1.1089972839130895,1000 -Linear,Logistic,experimental,False,6,0.95,0.9386666666666666,1.1492870219741105,0.2398038156236055,0.939,1.2860092104213576,1000 -Linear,Logistic,experimental,True,1,0.9,0.7976666666666666,0.244274323547519,0.0748075511921299,0.735,0.31288102428256387,1000 -Linear,Logistic,experimental,True,1,0.95,0.8716666666666666,0.2910708093755182,0.0748075511921299,0.833,0.35371101065661276,1000 -Linear,Logistic,experimental,True,4,0.9,0.03833333333333333,0.9676295529475489,1.0812355403537053,0.03,1.10872071383113,1000 -Linear,Logistic,experimental,True,4,0.95,0.067,1.1530017279827793,1.0812355403537053,0.054,1.289018545293176,1000 -Linear,Logistic,experimental,True,6,0.9,0.885,0.964486553866753,0.23961465238247193,0.887,1.109074934480792,1000 -Linear,Logistic,experimental,True,6,0.95,0.9383333333333334,1.1492566135842297,0.23961465238247193,0.933,1.2873487345102677,1000 -Linear,Logistic,observational,False,1,0.9,0.8873333333333334,0.2738660718613766,0.0672792367714224,0.875,0.35107388214992413,1000 -Linear,Logistic,observational,False,1,0.95,0.9376666666666666,0.32633155232820765,0.0672792367714224,0.938,0.39679498726255896,1000 -Linear,Logistic,observational,False,4,0.9,0.16433333333333333,1.3471569564681332,1.0662036471383771,0.133,1.5189433786007114,1000 -Linear,Logistic,observational,False,4,0.95,0.248,1.6052365225310308,1.0662036471383771,0.203,1.7701614407109725,1000 -Linear,Logistic,observational,False,6,0.9,0.8823333333333334,1.0106046290695898,0.25297949046206,0.886,1.1632254064709615,1000 -Linear,Logistic,observational,False,6,0.95,0.9396666666666667,1.20420968962261,0.25297949046206,0.941,1.3507861334703333,1000 -Linear,Logistic,observational,True,1,0.9,0.8836666666666666,0.27183527948043773,0.06710802675280161,0.87,0.3482504195565255,1000 -Linear,Logistic,observational,True,1,0.95,0.9373333333333334,0.3239117139538379,0.06710802675280161,0.934,0.3934241047155358,1000 -Linear,Logistic,observational,True,4,0.9,0.17166666666666666,1.3476603584428708,1.0647967745035234,0.133,1.5187476579231214,1000 -Linear,Logistic,observational,True,4,0.95,0.247,1.6058363629813088,1.0647967745035234,0.207,1.7714708628832339,1000 -Linear,Logistic,observational,True,6,0.9,0.889,1.0055746633220144,0.25465275272035065,0.884,1.1560383777095113,1000 -Linear,Logistic,observational,True,6,0.95,0.9386666666666666,1.1982161157585394,0.25465275272035065,0.939,1.3436875509579689,1000 diff --git a/results/did/did_pa_atte_coverage.csv b/results/did/did_pa_atte_coverage.csv deleted file mode 100644 index 9119e34..0000000 --- a/results/did/did_pa_atte_coverage.csv +++ /dev/null @@ -1,49 +0,0 @@ -Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,experimental,False,1,0.9,0.051,2.1624651397785515,2.1697505905302976,1000 -LGBM,LGBM,experimental,False,1,0.95,0.094,2.576736143777478,2.1697505905302976,1000 -LGBM,LGBM,experimental,False,2,0.9,0.378,2.1294819852681166,1.3213007503499328,1000 -LGBM,LGBM,experimental,False,2,0.95,0.475,2.537434291091179,1.3213007503499328,1000 -LGBM,LGBM,experimental,False,3,0.9,0.47,1.8796188347464011,1.0014962844281858,1000 -LGBM,LGBM,experimental,False,3,0.95,0.583,2.239703983626731,1.0014962844281858,1000 -LGBM,LGBM,experimental,False,4,0.9,0.078,1.8830006248174618,1.9173419242712182,1000 -LGBM,LGBM,experimental,False,4,0.95,0.122,2.243733635040054,1.9173419242712182,1000 -LGBM,LGBM,experimental,False,5,0.9,0.889,2.0642395778601936,0.5210006870170777,1000 -LGBM,LGBM,experimental,False,5,0.95,0.951,2.459693167693339,0.5210006870170777,1000 -LGBM,LGBM,experimental,False,6,0.9,0.908,1.8086231900602612,0.4337431389507495,1000 -LGBM,LGBM,experimental,False,6,0.95,0.948,2.1551074551794365,0.4337431389507495,1000 -LGBM,LGBM,experimental,True,1,0.9,0.049,2.159654648685151,2.17137681706751,1000 -LGBM,LGBM,experimental,True,1,0.95,0.099,2.573387237083486,2.17137681706751,1000 -LGBM,LGBM,experimental,True,2,0.9,0.373,2.129643042411216,1.319771507009933,1000 -LGBM,LGBM,experimental,True,2,0.95,0.47,2.537626202514028,1.319771507009933,1000 -LGBM,LGBM,experimental,True,3,0.9,0.477,1.8791031704354346,1.0000542552199656,1000 -LGBM,LGBM,experimental,True,3,0.95,0.598,2.23908953170162,1.0000542552199656,1000 -LGBM,LGBM,experimental,True,4,0.9,0.084,1.8855456999420728,1.9240618121414252,1000 -LGBM,LGBM,experimental,True,4,0.95,0.127,2.2467662791005663,1.9240618121414252,1000 -LGBM,LGBM,experimental,True,5,0.9,0.891,2.063666189492902,0.5200351889316859,1000 -LGBM,LGBM,experimental,True,5,0.95,0.947,2.459009933312703,0.5200351889316859,1000 -LGBM,LGBM,experimental,True,6,0.9,0.897,1.8093537325759845,0.43669034939483387,1000 -LGBM,LGBM,experimental,True,6,0.95,0.944,2.1559779502779253,0.43669034939483387,1000 -LGBM,LGBM,observational,False,1,0.9,0.893,12.590652453419517,3.3787304616783773,1000 -LGBM,LGBM,observational,False,1,0.95,0.953,15.002687744501154,3.3787304616783773,1000 -LGBM,LGBM,observational,False,2,0.9,0.914,14.716645515368727,3.622038823656133,1000 -LGBM,LGBM,observational,False,2,0.95,0.966,17.535964727040476,3.622038823656133,1000 -LGBM,LGBM,observational,False,3,0.9,0.933,14.387879061625718,3.413516510279929,1000 -LGBM,LGBM,observational,False,3,0.95,0.977,17.14421533481309,3.413516510279929,1000 -LGBM,LGBM,observational,False,4,0.9,0.843,18.129751335736472,5.765050835726839,1000 -LGBM,LGBM,observational,False,4,0.95,0.932,21.602931157204278,5.765050835726839,1000 -LGBM,LGBM,observational,False,5,0.9,0.917,7.704465948378402,1.901185439754437,1000 -LGBM,LGBM,observational,False,5,0.95,0.96,9.180437414922883,1.901185439754437,1000 -LGBM,LGBM,observational,False,6,0.9,0.922,7.569553123736534,1.7987428999886972,1000 -LGBM,LGBM,observational,False,6,0.95,0.971,9.019678868984235,1.7987428999886972,1000 -LGBM,LGBM,observational,True,1,0.9,0.906,4.1677645580041025,1.0303464962408189,1000 -LGBM,LGBM,observational,True,1,0.95,0.967,4.966197779476664,1.0303464962408189,1000 -LGBM,LGBM,observational,True,2,0.9,0.917,5.020279353776863,1.2235227285733399,1000 -LGBM,LGBM,observational,True,2,0.95,0.963,5.982031813960893,1.2235227285733399,1000 -LGBM,LGBM,observational,True,3,0.9,0.921,4.904759478369554,1.1326648179240986,1000 -LGBM,LGBM,observational,True,3,0.95,0.969,5.84438139231425,1.1326648179240986,1000 -LGBM,LGBM,observational,True,4,0.9,0.925,5.958922268906894,1.370042866021869,1000 -LGBM,LGBM,observational,True,4,0.95,0.971,7.1004938326197875,1.370042866021869,1000 -LGBM,LGBM,observational,True,5,0.9,0.92,3.449425982629519,0.8277123141255975,1000 -LGBM,LGBM,observational,True,5,0.95,0.965,4.110244572838216,0.8277123141255975,1000 -LGBM,LGBM,observational,True,6,0.9,0.925,3.40159548571924,0.8004114911308209,1000 -LGBM,LGBM,observational,True,6,0.95,0.97,4.053251020481494,0.8004114911308209,1000 diff --git a/results/did/did_pa_atte_coverage_metadata.csv b/results/did/did_pa_atte_coverage_metadata.csv deleted file mode 100644 index 961444b..0000000 --- a/results/did/did_pa_atte_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.11.dev0,did_pa_atte_coverage.py,2025-06-02 15:10:48,10769.977479457855,3.12.3 diff --git a/results/did/did_pa_multi_config.yml b/results/did/did_pa_multi_config.yml deleted file mode 100644 index ed4e23a..0000000 --- a/results/did/did_pa_multi_config.yml +++ /dev/null @@ -1,43 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - DGP: - - 1 - - 4 - - 6 - n_obs: - - 2000 -learner_definitions: - linear: &id001 - name: Linear - logistic: &id002 - name: Logistic - lgbmr: &id003 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.02 - lgbmc: &id004 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.02 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - score: - - observational - - experimental - in_sample_normalization: - - true - - false -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/did/did_pa_multi_coverage.csv b/results/did/did_pa_multi_coverage.csv deleted file mode 100644 index 8276aac..0000000 --- a/results/did/did_pa_multi_coverage.csv +++ /dev/null @@ -1,25 +0,0 @@ -Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -Linear,Linear,experimental,False,1,0.9,0.8875,0.589658704307476,0.14271881238392356,0.85,0.9184713776325004,20 -Linear,Linear,experimental,False,1,0.95,0.9458333333333334,0.7026216829731943,0.14271881238392356,0.9,1.0088572704968817,20 -Linear,Linear,experimental,False,4,0.9,0.7041666666666667,2.0297642964886093,0.9047675553035617,0.55,2.9260160123442596,20 -Linear,Linear,experimental,False,4,0.95,0.7708333333333333,2.418613336188561,0.9047675553035617,0.65,3.265541044699371,20 -Linear,Linear,experimental,False,6,0.9,0.9375,1.9858835644872976,0.4370880265956636,0.9,2.8747704680291024,20 -Linear,Linear,experimental,False,6,0.95,0.975,2.3663262190076697,0.4370880265956636,0.9,3.2067165503022985,20 -Linear,Linear,experimental,True,1,0.9,0.9041666666666666,0.5893929319394263,0.1419165948760371,0.9,0.9179189354368003,20 -Linear,Linear,experimental,True,1,0.95,0.9458333333333334,0.7023049956638021,0.1419165948760371,0.9,1.0086412447991253,20 -Linear,Linear,experimental,True,4,0.9,0.7041666666666667,2.031873164323966,0.9059623323417508,0.6,2.943810800692373,20 -Linear,Linear,experimental,True,4,0.95,0.7833333333333333,2.4211262072050013,0.9059623323417508,0.6,3.2739360531515684,20 -Linear,Linear,experimental,True,6,0.9,0.9458333333333334,1.9877755956053036,0.4386853538456557,0.9,2.8801526124840118,20 -Linear,Linear,experimental,True,6,0.95,0.975,2.3685807131390373,0.4386853538456557,0.9,3.2111476829248415,20 -Linear,Linear,observational,False,1,0.9,0.9125,0.6827078489949769,0.15467502808332462,0.9,1.0590069349882765,20 -Linear,Linear,observational,False,1,0.95,0.9583333333333334,0.8134965774875249,0.15467502808332462,1.0,1.1738447754301748,20 -Linear,Linear,observational,False,4,0.9,0.8,2.7854153587870214,0.8355183226583197,0.7,4.0015833335633415,20 -Linear,Linear,observational,False,4,0.95,0.8416666666666666,3.3190271132668636,0.8355183226583197,0.8,4.483472762454467,20 -Linear,Linear,observational,False,6,0.9,0.9333333333333333,2.5337718344427866,0.5967437923241619,0.9,3.6412755250672886,20 -Linear,Linear,observational,False,6,0.95,0.9708333333333334,3.0191753595448407,0.5967437923241619,0.95,4.081839476947988,20 -Linear,Linear,observational,True,1,0.9,0.9041666666666666,0.6620805479575324,0.15571526121686724,0.85,1.0317138682400495,20 -Linear,Linear,observational,True,1,0.95,0.9541666666666666,0.7889176323040639,0.15571526121686724,0.95,1.1316182486711308,20 -Linear,Linear,observational,True,4,0.9,0.7916666666666667,2.631108576842307,0.8079463860509168,0.7,3.77565721661986,20 -Linear,Linear,observational,True,4,0.95,0.8416666666666666,3.1351592418487586,0.8079463860509168,0.75,4.24016752287414,20 -Linear,Linear,observational,True,6,0.9,0.9333333333333333,2.3368968698600474,0.5421819294976428,0.9,3.361076606239287,20 -Linear,Linear,observational,True,6,0.95,0.9791666666666666,2.784584369977626,0.5421819294976428,0.95,3.7810902213750666,20 diff --git a/results/did/did_pa_multi_coverage_metadata.csv b/results/did/did_pa_multi_coverage_metadata.csv deleted file mode 100644 index 0cb88dc..0000000 --- a/results/did/did_pa_multi_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,did_pa_multi_coverage.py,2025-03-18 07:37:01,120.99546647071838,3.11.9 diff --git a/results/irm/apo_config.yml b/results/irm/apo_config.yml deleted file mode 100644 index 5f31101..0000000 --- a/results/irm/apo_config.yml +++ /dev/null @@ -1,49 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 500 - n_levels: - - 2 - linear: - - true -learner_definitions: - linear: &id001 - name: Linear - logit: &id002 - name: Logistic - lgbmr: &id003 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - min_child_samples: 10 - lgbmc: &id004 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 - min_child_samples: 10 -dml_parameters: - treatment_level: - - 0 - - 1 - - 2 - trimming_threshold: - - 0.01 - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id003 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id004 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/apo_coverage.csv b/results/irm/apo_coverage.csv deleted file mode 100644 index c352967..0000000 --- a/results/irm/apo_coverage.csv +++ /dev/null @@ -1,25 +0,0 @@ -Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Clas.,0,0.9,0.923,8.535462056273152,2.0255827803361752,1000 -LGBM Regr.,LGBM Clas.,0,0.95,0.968,10.170630351290903,2.0255827803361752,1000 -LGBM Regr.,LGBM Clas.,1,0.9,0.915,34.31276092012859,8.463655678765429,1000 -LGBM Regr.,LGBM Clas.,1,0.95,0.969,40.886176442476604,8.463655678765429,1000 -LGBM Regr.,LGBM Clas.,2,0.9,0.901,33.644464659560505,8.44195706768521,1000 -LGBM Regr.,LGBM Clas.,2,0.95,0.953,40.08985233177513,8.44195706768521,1000 -LGBM Regr.,Logistic,0,0.9,0.913,5.611262078064704,1.3129445023644262,1000 -LGBM Regr.,Logistic,0,0.95,0.96,6.686231164049146,1.3129445023644262,1000 -LGBM Regr.,Logistic,1,0.9,0.906,7.131142409013338,1.601689163860689,1000 -LGBM Regr.,Logistic,1,0.95,0.952,8.497280281526653,1.601689163860689,1000 -LGBM Regr.,Logistic,2,0.9,0.926,7.123330875617923,1.5793207930346633,1000 -LGBM Regr.,Logistic,2,0.95,0.961,8.487972265379696,1.5793207930346633,1000 -Linear,LGBM Clas.,0,0.9,0.91,5.450702479855432,1.2788345107461965,1000 -Linear,LGBM Clas.,0,0.95,0.953,6.494912602502952,1.2788345107461965,1000 -Linear,LGBM Clas.,1,0.9,0.934,9.871742629461385,2.0208159577876965,1000 -Linear,LGBM Clas.,1,0.95,0.977,11.762906863787386,2.0208159577876965,1000 -Linear,LGBM Clas.,2,0.9,0.935,7.196139854742809,1.5705772035604926,1000 -Linear,LGBM Clas.,2,0.95,0.971,8.574729515081229,1.5705772035604926,1000 -Linear,Logistic,0,0.9,0.915,5.333252473490304,1.262628265253654,1000 -Linear,Logistic,0,0.95,0.951,6.354962287965668,1.262628265253654,1000 -Linear,Logistic,1,0.9,0.907,5.409257288463179,1.2841258169966927,1000 -Linear,Logistic,1,0.95,0.947,6.445527610957087,1.2841258169966927,1000 -Linear,Logistic,2,0.9,0.909,5.362180231077127,1.2686475779103141,1000 -Linear,Logistic,2,0.95,0.949,6.389431837167295,1.2686475779103141,1000 diff --git a/results/irm/apo_metadata.csv b/results/irm/apo_metadata.csv deleted file mode 100644 index 601c446..0000000 --- a/results/irm/apo_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,APOCoverageSimulation,2025-06-04 15:38,79.44047049681346,3.12.3,scripts/irm/apo_config.yml diff --git a/results/irm/apos_causal_contrast.csv b/results/irm/apos_causal_contrast.csv deleted file mode 100644 index 5bc012d..0000000 --- a/results/irm/apos_causal_contrast.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,0.9,0.9035,33.48965631859352,8.5048896967406,0.918,39.64896208370754,1000 -LGBM Regr.,LGBM Clas.,0.95,0.9575,39.905386815920046,8.5048896967406,0.967,45.48932863359306,1000 -LGBM Regr.,Logistic,0.9,0.951,5.329365839934695,1.0762013117643536,0.951,6.300702382026109,1000 -LGBM Regr.,Logistic,0.95,0.9775,6.3503310784371845,1.0762013117643536,0.979,7.232560605885723,1000 -Linear,LGBM Clas.,0.9,0.965,6.721615473453926,1.3234802376469883,0.977,7.963378099405947,1000 -Linear,LGBM Clas.,0.95,0.989,8.009298839747467,1.3234802376469883,0.996,9.138185238253229,1000 -Linear,Logistic,0.9,0.8675,1.146829074707023,0.30695471324696894,0.836,1.3542396130508736,1000 -Linear,Logistic,0.95,0.918,1.366531128374668,0.30695471324696894,0.912,1.5554413731229015,1000 diff --git a/results/irm/apos_config.yml b/results/irm/apos_config.yml deleted file mode 100644 index 40be90e..0000000 --- a/results/irm/apos_config.yml +++ /dev/null @@ -1,49 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 500 - n_levels: - - 2 - linear: - - true -learner_definitions: - linear: &id001 - name: Linear - logit: &id002 - name: Logistic - lgbmr: &id003 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - min_child_samples: 10 - lgbmc: &id004 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 - min_child_samples: 10 -dml_parameters: - treatment_levels: - - - 0 - - 1 - - 2 - trimming_threshold: - - 0.01 - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id003 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id004 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/apos_coverage.csv b/results/irm/apos_coverage.csv deleted file mode 100644 index 4e630ec..0000000 --- a/results/irm/apos_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,0.9,0.9216666666666666,25.302332120467288,6.206005134767613,0.926,32.42413576207192,1000 -LGBM Regr.,LGBM Clas.,0.95,0.9623333333333334,30.149588308899382,6.206005134767613,0.975,36.66314428285753,1000 -LGBM Regr.,Logistic,0.9,0.9166666666666666,6.604807969876808,1.492507532099351,0.925,8.124328671640002,1000 -LGBM Regr.,Logistic,0.95,0.963,7.870114114502647,1.492507532099351,0.963,9.2992358185068,1000 -Linear,LGBM Clas.,0.9,0.927,7.536037974003498,1.6449388088327628,0.936,9.335529567192953,1000 -Linear,LGBM Clas.,0.95,0.968,8.979743104891375,1.6449388088327628,0.974,10.660074664816488,1000 -Linear,Logistic,0.9,0.9056666666666666,5.378673132414481,1.2747688188643604,0.907,5.79782063135373,1000 -Linear,Logistic,0.95,0.9536666666666667,6.409084341251623,1.2747688188643604,0.953,6.82033780870828,1000 diff --git a/results/irm/apos_metadata.csv b/results/irm/apos_metadata.csv deleted file mode 100644 index f463b04..0000000 --- a/results/irm/apos_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,APOSCoverageSimulation,2025-06-05 07:37,6.892837846279145,3.12.9,scripts/irm/apos_config.yml diff --git a/results/irm/cvar_Y0_coverage.csv b/results/irm/cvar_Y0_coverage.csv deleted file mode 100644 index 4c2e567..0000000 --- a/results/irm/cvar_Y0_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Clas.,0.9,0.85,0.5670142676988336,0.15380806524383983,200 -LGBM Regr.,LGBM Clas.,0.95,0.917142857142857,0.6756391725078731,0.15380806524383983,200 -LGBM Regr.,Logistic,0.9,0.7985714285714286,0.4385225256106473,0.13981066432860312,200 -LGBM Regr.,Logistic,0.95,0.8857142857142857,0.5225318183474872,0.13981066432860312,200 -Linear,LGBM Clas.,0.9,0.807142857142857,0.5780831437073561,0.16505729067291136,200 -Linear,LGBM Clas.,0.95,0.8778571428571429,0.6888285517757731,0.16505729067291136,200 -Linear,Logistic,0.9,0.7535714285714286,0.46127673668330255,0.14620789056223912,200 -Linear,Logistic,0.95,0.8271428571428571,0.5496451331545211,0.14620789056223912,200 diff --git a/results/irm/cvar_Y1_coverage.csv b/results/irm/cvar_Y1_coverage.csv deleted file mode 100644 index 5319243..0000000 --- a/results/irm/cvar_Y1_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Clas.,0.9,0.9214285714285714,0.1913460262511472,0.0431261794473061,200 -LGBM Regr.,LGBM Clas.,0.95,0.9592857142857143,0.22800285319744668,0.0431261794473061,200 -LGBM Regr.,Logistic,0.9,0.9192857142857143,0.18097901112163345,0.041041888975454605,200 -LGBM Regr.,Logistic,0.95,0.9614285714285714,0.2156497927499421,0.041041888975454605,200 -Linear,LGBM Clas.,0.9,0.9164285714285714,0.2132575331275393,0.046942310613367344,200 -Linear,LGBM Clas.,0.95,0.9621428571428571,0.2541120240203392,0.046942310613367344,200 -Linear,Logistic,0.9,0.9228571428571429,0.1968307741542601,0.04490834814057729,200 -Linear,Logistic,0.95,0.9557142857142857,0.23453833342391914,0.04490834814057729,200 diff --git a/results/irm/cvar_config.yml b/results/irm/cvar_config.yml deleted file mode 100644 index 5157d7e..0000000 --- a/results/irm/cvar_config.yml +++ /dev/null @@ -1,65 +0,0 @@ -simulation_parameters: - repetitions: 200 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 5000 - dim_x: - - 5 -learner_definitions: - linear: &id001 - name: Linear - logit: &id002 - name: Logistic - lgbmr: &id003 - name: LGBM Regr. - params: - n_estimators: 200 - learning_rate: 0.05 - num_leaves: 15 - max_depth: 5 - min_child_samples: 10 - subsample: 0.9 - colsample_bytree: 0.9 - reg_alpha: 0.0 - reg_lambda: 0.1 - random_state: 42 - lgbmc: &id004 - name: LGBM Clas. - params: - n_estimators: 200 - learning_rate: 0.05 - num_leaves: 15 - max_depth: 5 - min_child_samples: 10 - subsample: 0.9 - colsample_bytree: 0.9 - reg_alpha: 0.0 - reg_lambda: 0.1 - random_state: 42 -dml_parameters: - tau_vec: - - - 0.2 - - 0.3 - - 0.4 - - 0.5 - - 0.6 - - 0.7 - - 0.8 - trimming_threshold: - - 0.01 - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id003 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id004 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/cvar_coverage_metadata.csv b/results/irm/cvar_coverage_metadata.csv deleted file mode 100644 index f14d35e..0000000 --- a/results/irm/cvar_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,cvar_coverage.py,2025-05-22 15:47:04,15261.781089544296,3.12.10 diff --git a/results/irm/cvar_coverage_pq0.csv b/results/irm/cvar_coverage_pq0.csv deleted file mode 100644 index f94d6bb..0000000 --- a/results/irm/cvar_coverage_pq0.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.8892307692307692,0.5699689541680535,0.13916120902511853,100 -LGBM,LGBM,0.95,0.9461538461538461,0.6791598985897508,0.13916120902511853,100 -LGBM,Logistic Regression,0.9,0.8207692307692308,0.4060171336908242,0.11838313391016886,100 -LGBM,Logistic Regression,0.95,0.8907692307692308,0.4837992549009209,0.11838313391016886,100 -Linear,LGBM,0.9,0.7707692307692308,0.5801661718639771,0.1746635044255932,100 -Linear,LGBM,0.95,0.8630769230769231,0.691310632915921,0.1746635044255932,100 -Linear,Logistic Regression,0.9,0.69,0.4294697114538,0.1539651338207486,100 -Linear,Logistic Regression,0.95,0.7792307692307692,0.5117447249457237,0.1539651338207486,100 diff --git a/results/irm/cvar_coverage_pq1.csv b/results/irm/cvar_coverage_pq1.csv deleted file mode 100644 index 321647f..0000000 --- a/results/irm/cvar_coverage_pq1.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.9323076923076923,0.1908311409902022,0.04341246693737058,100 -LGBM,LGBM,0.95,0.9792307692307692,0.22738932956769187,0.04341246693737058,100 -LGBM,Logistic Regression,0.9,0.9130769230769231,0.1776451248637835,0.04403267398281018,100 -LGBM,Logistic Regression,0.95,0.963076923076923,0.21167722225073635,0.04403267398281018,100 -Linear,LGBM,0.9,0.9307692307692308,0.21584587388672186,0.047576051617349686,100 -Linear,LGBM,0.95,0.9776923076923077,0.25719622226423833,0.047576051617349686,100 -Linear,Logistic Regression,0.9,0.8884615384615384,0.1934218436107483,0.04906705213284509,100 -Linear,Logistic Regression,0.95,0.943076923076923,0.23047634214298993,0.04906705213284509,100 diff --git a/results/irm/cvar_coverage_qte.csv b/results/irm/cvar_coverage_qte.csv deleted file mode 100644 index 7abdfa2..0000000 --- a/results/irm/cvar_coverage_qte.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9007692307692308,0.5818966962059838,0.14389616733672375,0.88,0.7055777033989438,100 -LGBM,LGBM,0.95,0.9492307692307692,0.693372679853793,0.14389616733672375,0.95,0.8113769406399532,100 -LGBM,Logistic Regression,0.9,0.81,0.41715844354328135,0.12275759703405408,0.78,0.5062794648957274,100 -LGBM,Logistic Regression,0.95,0.8692307692307693,0.49707494441737266,0.12275759703405408,0.86,0.5835121707560351,100 -Linear,LGBM,0.9,0.8007692307692308,0.6065907561805581,0.1798354817301676,0.8,0.7182897195756133,100 -Linear,LGBM,0.95,0.8638461538461538,0.7227974671960807,0.1798354817301676,0.85,0.8301176550726784,100 -Linear,Logistic Regression,0.9,0.7084615384615384,0.45417244635105875,0.15263121995044057,0.69,0.5341807534621331,100 -Linear,Logistic Regression,0.95,0.8123076923076923,0.5411798490959508,0.15263121995044057,0.8,0.6185753871641957,100 diff --git a/results/irm/cvar_effect_coverage.csv b/results/irm/cvar_effect_coverage.csv deleted file mode 100644 index c54c59b..0000000 --- a/results/irm/cvar_effect_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,0.9,0.85,0.5796241888849147,0.15567372871154336,0.87,0.7009270285655491,200 -LGBM Regr.,LGBM Clas.,0.95,0.9292857142857143,0.6906648203634905,0.15567372871154336,0.92,0.8074443628687097,200 -LGBM Regr.,Logistic,0.9,0.8207142857142857,0.4505521904112434,0.1384841777144103,0.795,0.5426740272513005,200 -LGBM Regr.,Logistic,0.95,0.8928571428571429,0.5368660480740288,0.1384841777144103,0.88,0.6261735233895037,200 -Linear,LGBM Clas.,0.9,0.825,0.604031305594578,0.17336373457185203,0.78,0.7151775160876557,200 -Linear,LGBM Clas.,0.95,0.8971428571428571,0.719747693716827,0.17336373457185203,0.85,0.8269744576928215,200 -Linear,Logistic,0.9,0.775,0.4860331654639814,0.1489380406801104,0.74,0.5698020622218235,200 -Linear,Logistic,0.95,0.85,0.5791442375130592,0.1489380406801104,0.81,0.6599452421128806,200 diff --git a/results/irm/cvar_metadata.csv b/results/irm/cvar_metadata.csv deleted file mode 100644 index 63df236..0000000 --- a/results/irm/cvar_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,CVARCoverageSimulation,2025-06-05 12:27,9.002358218034109,3.12.9,scripts/irm/cvar_config.yml diff --git a/results/irm/iivm_late_coverage.csv b/results/irm/iivm_late_coverage.csv deleted file mode 100644 index c234ac0..0000000 --- a/results/irm/iivm_late_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -Lasso,Logistic Regression,0.9,0.902,0.9293147641314788,0.2241721725234453,1000 -Lasso,Logistic Regression,0.95,0.956,1.10734684117444,0.2241721725234453,1000 -Lasso,Random Forest,0.9,0.903,0.9550141700093412,0.23077333095440236,1000 -Lasso,Random Forest,0.95,0.957,1.1379695720480933,0.23077333095440236,1000 -Random Forest,Logistic Regression,0.9,0.9,0.9629145290806888,0.23355881870984344,1000 -Random Forest,Logistic Regression,0.95,0.951,1.1473834305161408,0.23355881870984344,1000 -Random Forest,Random Forest,0.9,0.904,0.9912410494922755,0.23401096266112748,1000 -Random Forest,Random Forest,0.95,0.959,1.1811365614357268,0.23401096266112748,1000 diff --git a/results/irm/iivm_late_coverage_metadata.csv b/results/irm/iivm_late_coverage_metadata.csv deleted file mode 100644 index e737bef..0000000 --- a/results/irm/iivm_late_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,iivm_late_coverage.py,2025-05-22 13:10:22,5866.210592031479,3.12.10 diff --git a/results/irm/irm_apo_coverage_apo.csv b/results/irm/irm_apo_coverage_apo.csv deleted file mode 100644 index e830e1b..0000000 --- a/results/irm/irm_apo_coverage_apo.csv +++ /dev/null @@ -1,25 +0,0 @@ -Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.0,0.9,0.912,8.657690136121921,2.076508232578001,1000 -LGBM,LGBM,0.0,0.95,0.966,10.316274091547035,2.076508232578001,1000 -LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.211992345283987,1000 -LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.211992345283987,1000 -LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.582632590363396,1000 -LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.582632590363396,1000 -LGBM,Logistic,0.0,0.9,0.904,5.625897101886533,1.3388185183680807,1000 -LGBM,Logistic,0.0,0.95,0.954,6.7036698705295725,1.3388185183680807,1000 -LGBM,Logistic,1.0,0.9,0.923,7.423300143143785,1.6937126309587676,1000 -LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6937126309587676,1000 -LGBM,Logistic,2.0,0.9,0.92,7.321275660150268,1.66124252169416,1000 -LGBM,Logistic,2.0,0.95,0.969,8.723838024042964,1.66124252169416,1000 -Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.309688195531907,1000 -Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.309688195531907,1000 -Linear,LGBM,1.0,0.9,0.949,10.700720020780512,2.128644427723186,1000 -Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.128644427723186,1000 -Linear,LGBM,2.0,0.9,0.933,7.513644049429104,1.6358873525441715,1000 -Linear,LGBM,2.0,0.95,0.968,8.953059097926168,1.6358873525441715,1000 -Linear,Logistic,0.0,0.9,0.902,5.335670092717667,1.2884949290748537,1000 -Linear,Logistic,0.0,0.95,0.953,6.357843058957276,1.2884949290748537,1000 -Linear,Logistic,1.0,0.9,0.908,5.417512107920403,1.280308177753247,1000 -Linear,Logistic,1.0,0.95,0.956,6.455363835025866,1.280308177753247,1000 -Linear,Logistic,2.0,0.9,0.906,5.366403391397197,1.28401173581695,1000 -Linear,Logistic,2.0,0.95,0.957,6.3944640430685675,1.28401173581695,1000 diff --git a/results/irm/irm_apo_coverage_apos.csv b/results/irm/irm_apo_coverage_apos.csv deleted file mode 100644 index 3531ab7..0000000 --- a/results/irm/irm_apo_coverage_apos.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9066666666666666,28.21021843292038,7.119697418816689,0.925,36.1943517986729,1000 -LGBM,LGBM,0.95,0.958,33.61454856442563,7.119697418816689,0.973,40.868020474452145,1000 -LGBM,Logistic,0.9,0.9126666666666666,6.789316986971467,1.5725415986639164,0.922,8.394070597867524,1000 -LGBM,Logistic,0.95,0.9626666666666667,8.089970168806186,1.5725415986639164,0.96,9.592284619044477,1000 -Linear,LGBM,0.9,0.927,7.903418354805325,1.7282923156600922,0.937,9.850445774370156,1000 -Linear,LGBM,0.95,0.9676666666666667,9.41750382912841,1.7282923156600922,0.974,11.234392064837424,1000 -Linear,Logistic,0.9,0.9043333333333333,5.372532806955153,1.2820240087685983,0.901,5.7964894175017925,1000 -Linear,Logistic,0.95,0.9566666666666667,6.4017676921854445,1.2820240087685983,0.952,6.8182385960659,1000 diff --git a/results/irm/irm_apo_coverage_apos_contrast.csv b/results/irm/irm_apo_coverage_apos_contrast.csv deleted file mode 100644 index 2c98fb5..0000000 --- a/results/irm/irm_apo_coverage_apos_contrast.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8885,37.87536523730769,9.788862143523833,0.898,44.828925781533165,1000 -LGBM,LGBM,0.95,0.9485,45.13128131893863,9.788862143523833,0.965,51.42387403222799,1000 -LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2673279809901974,0.927,6.774304315459575,1000 -LGBM,Logistic,0.95,0.9635,6.821911652197591,1.2673279809901974,0.963,7.7654094416927215,1000 -Linear,LGBM,0.9,0.958,7.430953406859927,1.5064252052940996,0.975,8.798628447594016,1000 -Linear,LGBM,0.95,0.989,8.85452711998085,1.5064252052940996,0.992,10.090116294778499,1000 -Linear,Logistic,0.9,0.8735,1.1425883251377777,0.29465524587422715,0.87,1.3505222299078565,1000 -Linear,Logistic,0.95,0.9355,1.3614779635902856,0.29465524587422715,0.92,1.5496710496635275,1000 diff --git a/results/irm/irm_apo_coverage_metadata.csv b/results/irm/irm_apo_coverage_metadata.csv deleted file mode 100644 index 1e26249..0000000 --- a/results/irm/irm_apo_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_apo_coverage.py,2025-05-22 13:10:29,5871.5619258880615,3.12.10 diff --git a/results/irm/irm_ate_config.yml b/results/irm/irm_ate_config.yml deleted file mode 100644 index d19a50a..0000000 --- a/results/irm/irm_ate_config.yml +++ /dev/null @@ -1,61 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - theta: - - 0.5 - n_obs: - - 500 - dim_x: - - 20 -learner_definitions: - lasso: &id001 - name: LassoCV - logit: &id002 - name: Logistic - rfr: &id003 - name: RF Regr. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - rfc: &id004 - name: RF Clas. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - lgbmr: &id005 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - lgbmc: &id006 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id001 - ml_m: *id004 - - ml_g: *id003 - ml_m: *id002 - - ml_g: *id005 - ml_m: *id006 - - ml_g: *id005 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id006 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv deleted file mode 100644 index 69935e2..0000000 --- a/results/irm/irm_ate_coverage.csv +++ /dev/null @@ -1,15 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Clas.,0.9,0.934,1.2170294870742426,0.28949513855358994,1000 -LGBM Regr.,LGBM Clas.,0.95,0.974,1.4501800790686068,0.28949513855358994,1000 -LGBM Regr.,Logistic,0.9,0.909,0.764475766453259,0.1851430173917419,1000 -LGBM Regr.,Logistic,0.95,0.955,0.9109290606478061,0.1851430173917419,1000 -LassoCV,LGBM Clas.,0.9,0.931,1.099023356166903,0.26125287479628606,1000 -LassoCV,LGBM Clas.,0.95,0.973,1.30956710126707,0.26125287479628606,1000 -LassoCV,Logistic,0.9,0.912,0.6518264483447356,0.15950540890700682,1000 -LassoCV,Logistic,0.95,0.962,0.7766991189934198,0.15950540890700682,1000 -LassoCV,RF Clas.,0.9,0.921,0.575659955473877,0.1328787221360119,1000 -LassoCV,RF Clas.,0.95,0.965,0.6859411449040855,0.1328787221360119,1000 -RF Regr.,Logistic,0.9,0.923,0.7334971446280483,0.1805477395572802,1000 -RF Regr.,Logistic,0.95,0.957,0.8740157559784838,0.1805477395572802,1000 -RF Regr.,RF Clas.,0.9,0.908,0.6176696649606507,0.1493303814432326,1000 -RF Regr.,RF Clas.,0.95,0.955,0.7359988012486619,0.1493303814432326,1000 diff --git a/results/irm/irm_ate_coverage_metadata.csv b/results/irm/irm_ate_coverage_metadata.csv deleted file mode 100644 index 5ed7ac7..0000000 --- a/results/irm/irm_ate_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_ate_coverage.py,2025-05-22 12:32:44,3604.067242860794,3.12.10 diff --git a/results/irm/irm_ate_metadata.csv b/results/irm/irm_ate_metadata.csv deleted file mode 100644 index 75d3d5f..0000000 --- a/results/irm/irm_ate_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,IRMATECoverageSimulation,2025-06-03 16:21,13.744510825475057,3.12.9,scripts/irm/irm_ate_config.yml diff --git a/results/irm/irm_ate_sensitivity_config.yml b/results/irm/irm_ate_sensitivity_config.yml deleted file mode 100644 index 74143aa..0000000 --- a/results/irm/irm_ate_sensitivity_config.yml +++ /dev/null @@ -1,53 +0,0 @@ -simulation_parameters: - repetitions: 500 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - theta: - - 5.0 - n_obs: - - 5000 - trimming_threshold: - - 0.05 - var_epsilon_y: - - 1.0 - linear: - - false - gamma_a: - - 0.198 - beta_a: - - 0.582 -learner_definitions: - linear: &id001 - name: Linear - logit: &id002 - name: Logistic - lgbmr: &id003 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - min_child_samples: 10 - lgbmc: &id004 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 - min_child_samples: 10 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id003 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id004 - trimming_threshold: - - 0.05 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/irm_ate_sensitivity_coverage.csv b/results/irm/irm_ate_sensitivity_coverage.csv deleted file mode 100644 index 4a63af0..0000000 --- a/results/irm/irm_ate_sensitivity_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition -LGBM Regr.,LGBM Clas.,0.9,0.104,0.2668157568060522,0.18026525524152154,0.966,1.0,0.12473788062439613,0.05508904509343494,0.04380159419714633,0.3234512135135403,500 -LGBM Regr.,LGBM Clas.,0.95,0.276,0.31793058377898364,0.18026525524152154,1.0,1.0,0.12473788062439613,0.03540613542014128,0.04380159419714633,0.3234512135135403,500 -LGBM Regr.,Logistic,0.9,0.248,0.25763009474845106,0.14980027719730954,0.998,1.0,0.10093696287018208,0.03536055959101725,0.027371865522670537,0.2987135019044584,500 -LGBM Regr.,Logistic,0.95,0.552,0.306985192339855,0.14980027719730954,1.0,1.0,0.10093696287018208,0.01878662839660344,0.027371865522670537,0.2987135019044584,500 -Linear,LGBM Clas.,0.9,0.112,0.26715210077204105,0.17865402724192433,0.962,1.0,0.12630248546657805,0.055046351043189806,0.04436057060203107,0.31869461365134216,500 -Linear,LGBM Clas.,0.95,0.282,0.318331362333959,0.17865402724192433,0.998,1.0,0.12630248546657805,0.03487837647302397,0.04436057060203107,0.31869461365134216,500 -Linear,Logistic,0.9,0.852,0.2589605314211221,0.09016543407251673,1.0,1.0,0.06325807709105473,0.00696464260372732,0.05686871205333632,0.23561615221802584,500 -Linear,Logistic,0.95,0.978,0.30857050541538944,0.09016543407251673,1.0,1.0,0.06325807709105473,0.0015852998947931969,0.05686871205333632,0.23561615221802584,500 diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv deleted file mode 100644 index d859488..0000000 --- a/results/irm/irm_ate_sensitivity_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,IRMATESensitivityCoverageSimulation,2025-06-04 10:16,29.540068797270457,3.12.3,scripts/irm/irm_ate_sensitivity_config.yml diff --git a/results/irm/irm_atte_config.yml b/results/irm/irm_atte_config.yml deleted file mode 100644 index 2d3c69a..0000000 --- a/results/irm/irm_atte_config.yml +++ /dev/null @@ -1,61 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - theta: - - 0.5 - n_obs: - - 500 - dim_x: - - 20 -learner_definitions: - lasso: &id001 - name: LassoCV - logit: &id002 - name: Logistic - rfr: &id003 - name: RF Regr. - params: - n_estimators: 200 - max_features: 20 - max_depth: 20 - min_samples_leaf: 2 - rfc: &id004 - name: RF Clas. - params: - n_estimators: 200 - max_features: 20 - max_depth: 20 - min_samples_leaf: 20 - lgbmr: &id005 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - lgbmc: &id006 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id001 - ml_m: *id004 - - ml_g: *id003 - ml_m: *id002 - - ml_g: *id005 - ml_m: *id006 - - ml_g: *id005 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id006 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv deleted file mode 100644 index 082860b..0000000 --- a/results/irm/irm_atte_coverage.csv +++ /dev/null @@ -1,15 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Clas.,0.9,0.935,1.4682839506185927,0.33857239275160667,1000 -LGBM Regr.,LGBM Clas.,0.95,0.974,1.7495682382536595,0.33857239275160667,1000 -LGBM Regr.,Logistic,0.9,0.903,0.827444945949616,0.20284041708199016,1000 -LGBM Regr.,Logistic,0.95,0.957,0.9859614659188063,0.20284041708199016,1000 -LassoCV,LGBM Clas.,0.9,0.916,1.364184861790926,0.3340226877296898,1000 -LassoCV,LGBM Clas.,0.95,0.968,1.625526523183968,0.3340226877296898,1000 -LassoCV,Logistic,0.9,0.913,0.7758018959411505,0.1948636425368796,1000 -LassoCV,Logistic,0.95,0.96,0.9244249763431417,0.1948636425368796,1000 -LassoCV,RF Clas.,0.9,0.892,0.5725347715806113,0.14886905941160222,1000 -LassoCV,RF Clas.,0.95,0.94,0.6822172586107998,0.14886905941160222,1000 -RF Regr.,Logistic,0.9,0.899,0.8139922164772362,0.2045901899504402,1000 -RF Regr.,Logistic,0.95,0.952,0.9699315500481204,0.2045901899504402,1000 -RF Regr.,RF Clas.,0.9,0.885,0.5863252811302729,0.15402934863308917,1000 -RF Regr.,RF Clas.,0.95,0.93,0.6986496642686135,0.15402934863308917,1000 diff --git a/results/irm/irm_atte_coverage_metadata.csv b/results/irm/irm_atte_coverage_metadata.csv deleted file mode 100644 index 99c4def..0000000 --- a/results/irm/irm_atte_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_atte_coverage.py,2025-05-22 13:53:23,3572.697674512863,3.12.10 diff --git a/results/irm/irm_atte_metadata.csv b/results/irm/irm_atte_metadata.csv deleted file mode 100644 index 92114f4..0000000 --- a/results/irm/irm_atte_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,IRMATTECoverageSimulation,2025-06-03 16:07,13.51489497423172,3.12.9,scripts/irm/irm_atte_config.yml diff --git a/results/irm/irm_atte_sensitivity_config.yml b/results/irm/irm_atte_sensitivity_config.yml deleted file mode 100644 index bf06bc6..0000000 --- a/results/irm/irm_atte_sensitivity_config.yml +++ /dev/null @@ -1,53 +0,0 @@ -simulation_parameters: - repetitions: 500 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - theta: - - 5.0 - n_obs: - - 5000 - trimming_threshold: - - 0.05 - var_epsilon_y: - - 1.0 - linear: - - false - gamma_a: - - 0.151 - beta_a: - - 0.582 -learner_definitions: - linear: &id001 - name: Linear - logit: &id002 - name: Logistic - lgbmr: &id003 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - min_child_samples: 10 - lgbmc: &id004 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 - min_child_samples: 10 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id003 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id004 - trimming_threshold: - - 0.05 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/irm_atte_sensitivity_coverage.csv b/results/irm/irm_atte_sensitivity_coverage.csv deleted file mode 100644 index 075e58b..0000000 --- a/results/irm/irm_atte_sensitivity_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition -LGBM Regr.,LGBM Clas.,0.9,0.724,0.3490468150328256,0.1335281439881145,0.95,1.0,0.1038722101661171,0.023169595411329586,0.06489839293328352,0.2564847068835367,500 -LGBM Regr.,LGBM Clas.,0.95,0.848,0.4159149332033151,0.1335281439881145,0.984,1.0,0.1038722101661171,0.011313984709802882,0.06489839293328352,0.2564847068835367,500 -LGBM Regr.,Logistic,0.9,0.712,0.34686964803507303,0.13021401649938852,0.968,1.0,0.09785329627117717,0.02088828773212853,0.06357450466836227,0.25752793576799426,500 -LGBM Regr.,Logistic,0.95,0.862,0.4133206787152529,0.13021401649938852,0.986,1.0,0.09785329627117717,0.010006537111844464,0.06357450466836227,0.25752793576799426,500 -Linear,LGBM Clas.,0.9,0.778,0.34985166701129805,0.12304199369830472,0.968,1.0,0.09780901861894682,0.018644199845226802,0.06344115655765417,0.24215873513324584,500 -Linear,LGBM Clas.,0.95,0.88,0.41687397348802135,0.12304199369830472,0.988,1.0,0.09780901861894682,0.008642793757112277,0.06344115655765417,0.24215873513324584,500 -Linear,Logistic,0.9,0.956,0.35055289955459806,0.0738271575494762,0.996,1.0,0.05790900052274957,0.004015909139992085,0.0975242247284002,0.17368819501662186,500 -Linear,Logistic,0.95,0.98,0.41770954360024026,0.0738271575494762,0.998,1.0,0.05790900052274957,0.0013840310375437,0.0975242247284002,0.17368819501662186,500 diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv deleted file mode 100644 index e0ae81f..0000000 --- a/results/irm/irm_atte_sensitivity_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,IRMATTESensitivityCoverageSimulation,2025-06-04 10:49,30.47395207484563,3.12.3,scripts/irm/irm_atte_sensitivity_config.yml diff --git a/results/irm/irm_cate_config.yml b/results/irm/irm_cate_config.yml deleted file mode 100644 index c1206fe..0000000 --- a/results/irm/irm_cate_config.yml +++ /dev/null @@ -1,63 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 500 - p: - - 10 - support_size: - - 5 - n_x: - - 1 -learner_definitions: - linear: &id001 - name: Linear - logit: &id002 - name: Logistic - rfr: &id003 - name: RF Regr. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - rfc: &id004 - name: RF Clas. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - lgbmr: &id005 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - lgbmc: &id006 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id001 - ml_m: *id004 - - ml_g: *id003 - ml_m: *id002 - - ml_g: *id005 - ml_m: *id006 - - ml_g: *id005 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id006 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/irm_cate_coverage.csv b/results/irm/irm_cate_coverage.csv deleted file mode 100644 index 000353f..0000000 --- a/results/irm/irm_cate_coverage.csv +++ /dev/null @@ -1,15 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,0.9,0.92823,1.0570613395560353,0.24271123774219397,1.0,2.6463187180937235,1000 -LGBM Regr.,LGBM Clas.,0.95,0.96976,1.259566274489309,0.24271123774219397,1.0,2.652748380799862,1000 -LGBM Regr.,Logistic,0.9,0.90431,0.4603024759110742,0.10988090751139157,0.996,1.157310107019297,1000 -LGBM Regr.,Logistic,0.95,0.94966,0.5484842298414058,0.10988090751139157,0.998,1.1595510257286086,1000 -Linear,LGBM Clas.,0.9,0.90906,1.0431407600660088,0.25074855426124026,0.999,2.6288585357985186,1000 -Linear,LGBM Clas.,0.95,0.95858,1.2429788809380993,0.25074855426124026,0.998,2.6131478268009114,1000 -Linear,Logistic,0.9,0.9102100000000001,0.4767188958750692,0.11111755516576532,0.999,1.1958150906203837,1000 -Linear,Logistic,0.95,0.95427,0.5680456007484012,0.11111755516576532,0.999,1.1961020063274401,1000 -Linear,RF Clas.,0.9,0.91604,0.5102732447654866,0.11800460653781775,0.999,1.2834578728038433,1000 -Linear,RF Clas.,0.95,0.95914,0.6080280735182108,0.11800460653781775,0.999,1.2816066988048282,1000 -RF Regr.,Logistic,0.9,0.902,0.4598055139749778,0.10994809691322913,0.999,1.1600848822586207,1000 -RF Regr.,Logistic,0.95,0.9499099999999999,0.5478920631704775,0.10994809691322913,0.998,1.1551525914690834,1000 -RF Regr.,RF Clas.,0.9,0.905,0.49735687162215936,0.11853266588343181,0.999,1.2483681652474525,1000 -RF Regr.,RF Clas.,0.95,0.95178,0.5926372656329394,0.11853266588343181,0.999,1.251522831166833,1000 diff --git a/results/irm/irm_cate_coverage_metadata.csv b/results/irm/irm_cate_coverage_metadata.csv deleted file mode 100644 index 771f45f..0000000 --- a/results/irm/irm_cate_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_cate_coverage.py,2025-05-22 14:27:33,5618.293743133545,3.12.10 diff --git a/results/irm/irm_cate_metadata.csv b/results/irm/irm_cate_metadata.csv deleted file mode 100644 index 1556fdc..0000000 --- a/results/irm/irm_cate_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,IRMCATECoverageSimulation,2025-06-03 14:24,10.22935619354248,3.12.9,scripts/irm/irm_cate_config.yml diff --git a/results/irm/irm_gate_config.yml b/results/irm/irm_gate_config.yml deleted file mode 100644 index c1206fe..0000000 --- a/results/irm/irm_gate_config.yml +++ /dev/null @@ -1,63 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 500 - p: - - 10 - support_size: - - 5 - n_x: - - 1 -learner_definitions: - linear: &id001 - name: Linear - logit: &id002 - name: Logistic - rfr: &id003 - name: RF Regr. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - rfc: &id004 - name: RF Clas. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - lgbmr: &id005 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - lgbmc: &id006 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id001 - ml_m: *id004 - - ml_g: *id003 - ml_m: *id002 - - ml_g: *id005 - ml_m: *id006 - - ml_g: *id005 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id006 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/irm_gate_coverage.csv b/results/irm/irm_gate_coverage.csv deleted file mode 100644 index ab738e0..0000000 --- a/results/irm/irm_gate_coverage.csv +++ /dev/null @@ -1,15 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,0.9,0.9236666666666666,0.851465317065706,0.19722702192668598,1.0,2.011914810456983,1000 -LGBM Regr.,LGBM Clas.,0.95,0.972,1.014583503473648,0.19722702192668598,1.0,2.0031648741801455,1000 -LGBM Regr.,Logistic,0.9,0.9016666666666666,0.40159876476481454,0.09695556310967947,0.999,0.9443179359677094,1000 -LGBM Regr.,Logistic,0.95,0.9473333333333334,0.47853444359887215,0.09695556310967947,0.999,0.9425971024906015,1000 -Linear,LGBM Clas.,0.9,0.923,0.8592796676557775,0.19934007812012308,1.0,2.0230623845143447,1000 -Linear,LGBM Clas.,0.95,0.966,1.023894876515087,0.19934007812012308,1.0,2.0203888246588932,1000 -Linear,Logistic,0.9,0.915,0.4195046438102248,0.09830647800141477,0.999,0.9863092550256803,1000 -Linear,Logistic,0.95,0.9533333333333334,0.49987061446872544,0.09830647800141477,0.999,0.9830466638434444,1000 -Linear,RF Clas.,0.9,0.9206666666666666,0.4437261411849741,0.10126652921113784,0.999,1.0415488911519282,1000 -Linear,RF Clas.,0.95,0.9596666666666667,0.5287323087424741,0.10126652921113784,1.0,1.040804022375608,1000 -RF Regr.,Logistic,0.9,0.897,0.4014791112025455,0.09669425998448095,1.0,0.9423536354277464,1000 -RF Regr.,Logistic,0.95,0.9506666666666667,0.4783918675855257,0.09669425998448095,1.0,0.9449044166570425,1000 -RF Regr.,RF Clas.,0.9,0.8993333333333333,0.4239547202094922,0.10104977503999726,1.0,0.9958597773023945,1000 -RF Regr.,RF Clas.,0.95,0.95,0.5051732075554921,0.10104977503999726,0.999,0.9961472336115996,1000 diff --git a/results/irm/irm_gate_coverage_metadata.csv b/results/irm/irm_gate_coverage_metadata.csv deleted file mode 100644 index 00fc71a..0000000 --- a/results/irm/irm_gate_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,irm_gate_coverage.py,2025-05-22 12:06:46,2053.3285892009735,3.12.10 diff --git a/results/irm/irm_gate_metadata.csv b/results/irm/irm_gate_metadata.csv deleted file mode 100644 index 13efdf0..0000000 --- a/results/irm/irm_gate_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,IRMGATECoverageSimulation,2025-06-03 13:48,9.251631820201874,3.12.9,scripts/irm/irm_gate_config.yml diff --git a/results/irm/lpq_Y0_coverage.csv b/results/irm/lpq_Y0_coverage.csv deleted file mode 100644 index 81556c0..0000000 --- a/results/irm/lpq_Y0_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM Clas.,LGBM Clas.,0.9,0.935,1.1830884138959616,0.24363577628567012,200 -LGBM Clas.,LGBM Clas.,0.95,0.966,1.4097367958876215,0.24363577628567012,200 -LGBM Clas.,Logistic,0.9,0.9470000000000001,1.1413178140016869,0.21854310075124617,200 -LGBM Clas.,Logistic,0.95,0.971,1.3599640561957957,0.21854310075124617,200 -Logistic,LGBM Clas.,0.9,0.932,1.1519919205445956,0.2330721806385721,200 -Logistic,LGBM Clas.,0.95,0.965,1.3726830386319526,0.2330721806385721,200 -Logistic,Logistic,0.9,0.9359999999999999,1.1121705811298108,0.21974489145697174,200 -Logistic,Logistic,0.95,0.9670000000000001,1.3252329860617573,0.21974489145697174,200 diff --git a/results/irm/lpq_Y1_coverage.csv b/results/irm/lpq_Y1_coverage.csv deleted file mode 100644 index 5f9e93b..0000000 --- a/results/irm/lpq_Y1_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM Clas.,LGBM Clas.,0.9,0.927,1.6166365116884824,0.31220031423527045,200 -LGBM Clas.,LGBM Clas.,0.95,0.963,1.9263412178957202,0.31220031423527045,200 -LGBM Clas.,Logistic,0.9,0.943,1.563138831397548,0.2947145026308053,200 -LGBM Clas.,Logistic,0.95,0.971,1.8625948000329942,0.2947145026308053,200 -Logistic,LGBM Clas.,0.9,0.9420000000000001,1.567514402933319,0.2888227334203665,200 -Logistic,LGBM Clas.,0.95,0.971,1.867808615099192,0.2888227334203665,200 -Logistic,Logistic,0.9,0.94,1.514063009772447,0.28729535217904684,200 -Logistic,Logistic,0.95,0.9690000000000001,1.8041173517537936,0.28729535217904684,200 diff --git a/results/irm/lpq_config.yml b/results/irm/lpq_config.yml deleted file mode 100644 index 85abd3f..0000000 --- a/results/irm/lpq_config.yml +++ /dev/null @@ -1,48 +0,0 @@ -simulation_parameters: - repetitions: 200 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 5000 - dim_x: - - 5 -learner_definitions: - logit: &id001 - name: Logistic - lgbmc: &id002 - name: LGBM Clas. - params: - n_estimators: 200 - learning_rate: 0.05 - num_leaves: 15 - max_depth: 5 - min_child_samples: 10 - subsample: 0.9 - colsample_bytree: 0.9 - reg_alpha: 0.0 - reg_lambda: 0.1 - random_state: 42 -dml_parameters: - tau_vec: - - - 0.3 - - 0.4 - - 0.5 - - 0.6 - - 0.7 - trimming_threshold: - - 0.01 - learners: - - ml_g: *id001 - ml_m: *id001 - - ml_g: *id002 - ml_m: *id002 - - ml_g: *id002 - ml_m: *id001 - - ml_g: *id001 - ml_m: *id002 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/lpq_coverage_lpq0.csv b/results/irm/lpq_coverage_lpq0.csv deleted file mode 100644 index d563335..0000000 --- a/results/irm/lpq_coverage_lpq0.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.94,1.1717023704378204,0.23180102735661154,100 -LGBM,LGBM,0.95,0.9722222222222223,1.396169488293374,0.23180102735661154,100 -LGBM,Logistic Regression,0.9,0.95,1.1248311184133066,0.2270596212748565,100 -LGBM,Logistic Regression,0.95,0.9755555555555556,1.3403189467174592,0.2270596212748565,100 -Logistic Regression,LGBM,0.9,0.9488888888888889,1.1516756381210271,0.22097455311738945,100 -Logistic Regression,LGBM,0.95,0.9811111111111112,1.372306164879188,0.22097455311738945,100 -Logistic Regression,Logistic Regression,0.9,0.9533333333333333,1.1095156391492575,0.20619468880954653,100 -Logistic Regression,Logistic Regression,0.95,0.9788888888888888,1.3220694275677582,0.20619468880954653,100 diff --git a/results/irm/lpq_coverage_lpq1.csv b/results/irm/lpq_coverage_lpq1.csv deleted file mode 100644 index 524d87a..0000000 --- a/results/irm/lpq_coverage_lpq1.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.9577777777777777,1.6624989751251327,0.3276833414641952,100 -LGBM,LGBM,0.95,0.99,1.9809897137285786,0.3276833414641952,100 -LGBM,Logistic Regression,0.9,0.9411111111111111,1.5914659538910119,0.32194623632362507,100 -LGBM,Logistic Regression,0.95,0.9711111111111111,1.8963486483773861,0.32194623632362507,100 -Logistic Regression,LGBM,0.9,0.96,1.60784858202307,0.30083496643464896,100 -Logistic Regression,LGBM,0.95,0.9822222222222223,1.915869753833108,0.30083496643464896,100 -Logistic Regression,Logistic Regression,0.9,0.9466666666666668,1.556189836416658,0.304958909712274,100 -Logistic Regression,Logistic Regression,0.95,0.9844444444444445,1.8543145617989472,0.304958909712274,100 diff --git a/results/irm/lpq_coverage_lqte.csv b/results/irm/lpq_coverage_lqte.csv deleted file mode 100644 index 2e788da..0000000 --- a/results/irm/lpq_coverage_lqte.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.9233333333333333,1.636872420792144,0.38490701730912247,0.94,2.2547716211801454,100 -LGBM,LGBM,0.95,0.9622222222222223,1.950453790824845,0.38490701730912247,0.97,2.5400452907969764,100 -LGBM,Logistic Regression,0.9,0.9088888888888889,1.5675334432220174,0.37520599979698516,0.91,2.1574364137437168,100 -LGBM,Logistic Regression,0.95,0.95,1.8678313030025362,0.37520599979698516,0.95,2.4273297337432616,100 -Logistic Regression,LGBM,0.9,0.9188888888888889,1.6190226009242576,0.3723322348549169,0.92,2.208051172256097,100 -Logistic Regression,LGBM,0.95,0.9644444444444444,1.9291844185850637,0.3723322348549169,0.98,2.487384422889672,100 -Logistic Regression,Logistic Regression,0.9,0.9066666666666667,1.5642497785562741,0.3720685867269132,0.9,2.1368366984439215,100 -Logistic Regression,Logistic Regression,0.95,0.9477777777777777,1.8639185752213465,0.3720685867269132,0.95,2.4083334749081065,100 diff --git a/results/irm/lpq_coverage_metadata.csv b/results/irm/lpq_coverage_metadata.csv deleted file mode 100644 index 53aaefa..0000000 --- a/results/irm/lpq_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,lpq_coverage.py,2025-05-22 16:36:53,18251.836868286133,3.12.10 diff --git a/results/irm/lpq_effect_coverage.csv b/results/irm/lpq_effect_coverage.csv deleted file mode 100644 index b4f076b..0000000 --- a/results/irm/lpq_effect_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Clas.,LGBM Clas.,0.9,0.872,1.6181705726529205,0.40951487126584607,0.845,2.1354108341964264,200 -LGBM Clas.,LGBM Clas.,0.95,0.9159999999999999,1.928169164280202,0.40951487126584607,0.9,2.415503519370637,200 -LGBM Clas.,Logistic,0.9,0.8740000000000001,1.5610121253821851,0.3740351656285603,0.85,2.0636125204752434,200 -LGBM Clas.,Logistic,0.95,0.922,1.860060673514064,0.3740351656285603,0.9,2.3356640837390916,200 -Logistic,LGBM Clas.,0.9,0.88,1.5741042679593236,0.3725851701898653,0.85,2.068634959359497,200 -Logistic,LGBM Clas.,0.95,0.924,1.8756609235978434,0.3725851701898653,0.9,2.3413181297561683,200 -Logistic,Logistic,0.9,0.867,1.517966599775595,0.37223450094097055,0.85,1.9954916237204523,200 -Logistic,Logistic,0.95,0.92,1.808768766135729,0.37223450094097055,0.92,2.260577010587138,200 diff --git a/results/irm/lpq_metadata.csv b/results/irm/lpq_metadata.csv deleted file mode 100644 index 1ef72ff..0000000 --- a/results/irm/lpq_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,LPQCoverageSimulation,2025-06-05 13:42,14.852794400850932,3.12.9,scripts/irm/lpq_config.yml diff --git a/results/irm/pq_Y0_coverage.csv b/results/irm/pq_Y0_coverage.csv deleted file mode 100644 index 904a016..0000000 --- a/results/irm/pq_Y0_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM Clas.,LGBM Clas.,0.9,0.8721428571428571,0.5855383193027298,0.15439049862379853,200 -LGBM Clas.,LGBM Clas.,0.95,0.932142857142857,0.6977119414135707,0.15439049862379853,200 -LGBM Clas.,Logistic,0.9,0.8607142857142857,0.4157006469047894,0.1135304583367824,200 -LGBM Clas.,Logistic,0.95,0.9264285714285714,0.4953378725822799,0.1135304583367824,200 -Logistic,LGBM Clas.,0.9,0.9007142857142857,0.57746009123863,0.1320561476852036,200 -Logistic,LGBM Clas.,0.95,0.9614285714285714,0.6880861389682303,0.1320561476852036,200 -Logistic,Logistic,0.9,0.8892857142857143,0.40888219424763206,0.10270056266342995,200 -Logistic,Logistic,0.95,0.9335714285714286,0.4872131851211298,0.10270056266342995,200 diff --git a/results/irm/pq_Y1_coverage.csv b/results/irm/pq_Y1_coverage.csv deleted file mode 100644 index 59ce777..0000000 --- a/results/irm/pq_Y1_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM Clas.,LGBM Clas.,0.9,0.9135714285714286,0.2547786121622827,0.06025439826660347,200 -LGBM Clas.,LGBM Clas.,0.95,0.9635714285714286,0.30358744126957204,0.06025439826660347,200 -LGBM Clas.,Logistic,0.9,0.925,0.23468112272452962,0.055254012231200356,200 -LGBM Clas.,Logistic,0.95,0.9621428571428571,0.27963980554548934,0.055254012231200356,200 -Logistic,LGBM Clas.,0.9,0.9292857142857143,0.25306018928392865,0.05756351160374336,200 -Logistic,LGBM Clas.,0.95,0.9728571428571429,0.30153981411503566,0.05756351160374336,200 -Logistic,Logistic,0.9,0.9235714285714286,0.23576437773394143,0.05403824556407575,200 -Logistic,Logistic,0.95,0.9721428571428571,0.2809305835027078,0.05403824556407575,200 diff --git a/results/irm/pq_config.yml b/results/irm/pq_config.yml deleted file mode 100644 index e106878..0000000 --- a/results/irm/pq_config.yml +++ /dev/null @@ -1,50 +0,0 @@ -simulation_parameters: - repetitions: 200 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 5000 - dim_x: - - 5 -learner_definitions: - logit: &id001 - name: Logistic - lgbmc: &id002 - name: LGBM Clas. - params: - n_estimators: 200 - learning_rate: 0.05 - num_leaves: 15 - max_depth: 5 - min_child_samples: 10 - subsample: 0.9 - colsample_bytree: 0.9 - reg_alpha: 0.0 - reg_lambda: 0.1 - random_state: 42 -dml_parameters: - tau_vec: - - - 0.2 - - 0.3 - - 0.4 - - 0.5 - - 0.6 - - 0.7 - - 0.8 - trimming_threshold: - - 0.01 - learners: - - ml_g: *id001 - ml_m: *id001 - - ml_g: *id002 - ml_m: *id002 - - ml_g: *id002 - ml_m: *id001 - - ml_g: *id001 - ml_m: *id002 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/irm/pq_coverage_metadata.csv b/results/irm/pq_coverage_metadata.csv deleted file mode 100644 index 4074d69..0000000 --- a/results/irm/pq_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,pq_coverage.py,2025-05-22 16:20:49,17287.969739675522,3.12.10 diff --git a/results/irm/pq_coverage_pq0.csv b/results/irm/pq_coverage_pq0.csv deleted file mode 100644 index e7b946a..0000000 --- a/results/irm/pq_coverage_pq0.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.8823076923076922,0.5804011224570385,0.14917558900311645,100 -LGBM,LGBM,0.95,0.9338461538461539,0.6915905938151775,0.14917558900311645,100 -LGBM,Logistic Regression,0.9,0.8115384615384617,0.38610406439934963,0.11426502373246265,100 -LGBM,Logistic Regression,0.95,0.8869230769230769,0.4600713693350327,0.11426502373246265,100 -Logistic Regression,LGBM,0.9,0.8869230769230769,0.5879335180942487,0.15107204460433907,100 -Logistic Regression,LGBM,0.95,0.9415384615384617,0.7005659968080872,0.15107204460433907,100 -Logistic Regression,Logistic Regression,0.9,0.8223076923076923,0.3894908707564202,0.11249469872415391,100 -Logistic Regression,Logistic Regression,0.95,0.9015384615384616,0.4641069980218067,0.11249469872415391,100 diff --git a/results/irm/pq_coverage_pq1.csv b/results/irm/pq_coverage_pq1.csv deleted file mode 100644 index ab97622..0000000 --- a/results/irm/pq_coverage_pq1.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,0.9,0.9153846153846155,0.2504138111093655,0.058637832346116046,100 -LGBM,LGBM,0.95,0.9623076923076923,0.298386460025268,0.058637832346116046,100 -LGBM,Logistic Regression,0.9,0.9138461538461539,0.2294851617480571,0.053811638122637194,100 -LGBM,Logistic Regression,0.95,0.9623076923076923,0.273448436166418,0.053811638122637194,100 -Logistic Regression,LGBM,0.9,0.9192307692307692,0.25410039964172587,0.059350250451121786,100 -Logistic Regression,LGBM,0.95,0.9607692307692308,0.3027793012063014,0.059350250451121786,100 -Logistic Regression,Logistic Regression,0.9,0.8976923076923078,0.23093621538720735,0.05722710381019568,100 -Logistic Regression,Logistic Regression,0.95,0.9538461538461539,0.2751774732222205,0.05722710381019568,100 diff --git a/results/irm/pq_coverage_qte.csv b/results/irm/pq_coverage_qte.csv deleted file mode 100644 index c1c85e9..0000000 --- a/results/irm/pq_coverage_qte.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM,LGBM,0.9,0.8823076923076922,0.6176207568284253,0.15519922876616388,0.84,0.9336715877443045,100 -LGBM,LGBM,0.95,0.9392307692307692,0.7359405236146267,0.15519922876616388,0.91,1.0306538899948843,100 -LGBM,Logistic Regression,0.9,0.8261538461538461,0.4257458252181111,0.1226898281791693,0.78,0.6459797086090959,100 -LGBM,Logistic Regression,0.95,0.9107692307692308,0.5073074408099907,0.1226898281791693,0.84,0.7142375039945987,100 -Logistic Regression,LGBM,0.9,0.8930769230769231,0.6286198118927271,0.1592095091097193,0.87,0.9308233463178209,100 -Logistic Regression,LGBM,0.95,0.9476923076923077,0.7490467060960181,0.1592095091097193,0.92,1.029300136308572,100 -Logistic Regression,Logistic Regression,0.9,0.8415384615384616,0.43446715078562453,0.12422823365772839,0.8,0.6474624649583857,100 -Logistic Regression,Logistic Regression,0.95,0.9123076923076923,0.5176995411949078,0.12422823365772839,0.86,0.719846486999326,100 diff --git a/results/irm/pq_effect_coverage.csv b/results/irm/pq_effect_coverage.csv deleted file mode 100644 index 2723ea4..0000000 --- a/results/irm/pq_effect_coverage.csv +++ /dev/null @@ -1,9 +0,0 @@ -Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Clas.,LGBM Clas.,0.9,0.8721428571428571,0.6250811142351042,0.16391196246394876,0.83,0.8938663559093165,200 -LGBM Clas.,LGBM Clas.,0.95,0.9328571428571428,0.74483008776143,0.16391196246394876,0.91,0.9951285879711892,200 -LGBM Clas.,Logistic,0.9,0.8664285714285714,0.4563475824127607,0.1242769519340373,0.825,0.6552349151667995,200 -LGBM Clas.,Logistic,0.95,0.927142857142857,0.5437716835744456,0.1242769519340373,0.905,0.7290641682104055,200 -Logistic,LGBM Clas.,0.9,0.92,0.6192520613043571,0.14050842467602845,0.905,0.873815817039262,200 -Logistic,LGBM Clas.,0.95,0.9585714285714286,0.7378843427899359,0.14050842467602845,0.94,0.97537292218002,200 -Logistic,Logistic,0.9,0.8907142857142857,0.4547624845968767,0.11573164990154354,0.88,0.6436547419121843,200 -Logistic,Logistic,0.95,0.9385714285714286,0.541882923030528,0.11573164990154354,0.94,0.7176332662822544,200 diff --git a/results/irm/pq_metadata.csv b/results/irm/pq_metadata.csv deleted file mode 100644 index ccb611e..0000000 --- a/results/irm/pq_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,PQCoverageSimulation,2025-06-05 12:51,16.240616850058238,3.12.9,scripts/irm/pq_config.yml diff --git a/results/irm/ssm_mar_ate_coverage.csv b/results/irm/ssm_mar_ate_coverage.csv deleted file mode 100644 index 1bb3c80..0000000 --- a/results/irm/ssm_mar_ate_coverage.csv +++ /dev/null @@ -1,17 +0,0 @@ -Learner g,Learner m,Learner pi,score,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,LGBM,missing-at-random,0.9,0.934,5.894613599209654,1.524603484598883,1000 -LGBM,LGBM,LGBM,missing-at-random,0.95,0.981,7.023865326329001,1.524603484598883,1000 -LGBM,LGBM,Logistic,missing-at-random,0.9,0.927,2.581588867556288,0.6151192173080174,1000 -LGBM,LGBM,Logistic,missing-at-random,0.95,0.973,3.0761528687981836,0.6151192173080174,1000 -LGBM,Logistic,LGBM,missing-at-random,0.9,0.945,2.5756414334122413,0.6542029876050892,1000 -LGBM,Logistic,LGBM,missing-at-random,0.95,0.985,3.0690660639106513,0.6542029876050892,1000 -LGBM,Logistic,Logistic,missing-at-random,0.9,0.914,0.5399737109966929,0.12672360617482703,1000 -LGBM,Logistic,Logistic,missing-at-random,0.95,0.958,0.6434183618596122,0.12672360617482703,1000 -Lasso,LGBM,LGBM,missing-at-random,0.9,0.939,5.030966633759897,1.2700343898139361,1000 -Lasso,LGBM,LGBM,missing-at-random,0.95,0.981,5.994766493519136,1.2700343898139361,1000 -Lasso,LGBM,Logistic,missing-at-random,0.9,0.887,2.3414252826432578,0.6221402191258447,1000 -Lasso,LGBM,Logistic,missing-at-random,0.95,0.955,2.78998030662317,0.6221402191258447,1000 -Lasso,Logistic,LGBM,missing-at-random,0.9,0.919,2.2995632695758177,0.6130089902400226,1000 -Lasso,Logistic,LGBM,missing-at-random,0.95,0.97,2.7400986414171338,0.6130089902400226,1000 -Lasso,Logistic,Logistic,missing-at-random,0.9,0.897,0.5116626905293399,0.12267262949094253,1000 -Lasso,Logistic,Logistic,missing-at-random,0.95,0.961,0.6096837002627445,0.12267262949094253,1000 diff --git a/results/irm/ssm_mar_ate_coverage_metadata.csv b/results/irm/ssm_mar_ate_coverage_metadata.csv deleted file mode 100644 index 1653aa7..0000000 --- a/results/irm/ssm_mar_ate_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,ssm_mar_ate_coverage.py,2025-05-22 12:57:04,5067.488474369049,3.12.10 diff --git a/results/irm/ssm_nonignorable_ate_coverage.csv b/results/irm/ssm_nonignorable_ate_coverage.csv deleted file mode 100644 index 7ba36c8..0000000 --- a/results/irm/ssm_nonignorable_ate_coverage.csv +++ /dev/null @@ -1,17 +0,0 @@ -Learner g,Learner m,Learner pi,score,level,Coverage,CI Length,Bias,repetition -LGBM,LGBM,LGBM,nonignorable,0.9,0.919,13.007296321798503,3.6668819438868456,1000 -LGBM,LGBM,LGBM,nonignorable,0.95,0.966,15.499149534791721,3.6668819438868456,1000 -LGBM,LGBM,Logistic,nonignorable,0.9,0.918,4.832148171492935,1.326909933210586,1000 -LGBM,LGBM,Logistic,nonignorable,0.95,0.974,5.757859683624388,1.326909933210586,1000 -LGBM,Logistic,LGBM,nonignorable,0.9,0.897,4.592630929679517,1.2319933070953262,1000 -LGBM,Logistic,LGBM,nonignorable,0.95,0.959,5.472457286755355,1.2319933070953262,1000 -LGBM,Logistic,Logistic,nonignorable,0.9,0.867,2.5301811727355124,0.727808897144389,1000 -LGBM,Logistic,Logistic,nonignorable,0.95,0.94,3.014896822226011,0.727808897144389,1000 -Lasso,LGBM,LGBM,nonignorable,0.9,0.931,10.203902842326483,2.892343371671614,1000 -Lasso,LGBM,LGBM,nonignorable,0.95,0.975,12.158700169432056,2.892343371671614,1000 -Lasso,LGBM,Logistic,nonignorable,0.9,0.917,7.0625497859220765,2.0505032981783877,1000 -Lasso,LGBM,Logistic,nonignorable,0.95,0.974,8.415547129919013,2.0505032981783877,1000 -Lasso,Logistic,LGBM,nonignorable,0.9,0.894,4.170596833405628,1.1867597841647146,1000 -Lasso,Logistic,LGBM,nonignorable,0.95,0.95,4.969572643774818,1.1867597841647146,1000 -Lasso,Logistic,Logistic,nonignorable,0.9,0.87,1.993316632836819,0.5662899386624597,1000 -Lasso,Logistic,Logistic,nonignorable,0.95,0.928,2.375183266237269,0.5662899386624597,1000 diff --git a/results/irm/ssm_nonignorable_ate_coverage_metadata.csv b/results/irm/ssm_nonignorable_ate_coverage_metadata.csv deleted file mode 100644 index cd143b6..0000000 --- a/results/irm/ssm_nonignorable_ate_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,ssm_nonignorable_ate_coverage.py,2025-05-22 13:29:53,7036.344848871231,3.12.10 diff --git a/results/plm/pliv_late_config.yml b/results/plm/pliv_late_config.yml deleted file mode 100644 index 9863dcf..0000000 --- a/results/plm/pliv_late_config.yml +++ /dev/null @@ -1,57 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - theta: - - 0.5 - n_obs: - - 500 - dim_x: - - 20 - dim_z: - - 1 -learner_definitions: - lasso: &id001 - name: LassoCV - rf: &id002 - name: RF Regr. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id001 - ml_r: *id001 - - ml_g: *id002 - ml_m: *id002 - ml_r: *id002 - - ml_g: *id001 - ml_m: *id002 - ml_r: *id002 - - ml_g: *id002 - ml_m: *id001 - ml_r: *id002 - - ml_g: *id002 - ml_m: *id002 - ml_r: *id001 - - ml_g: *id001 - ml_m: *id001 - ml_r: *id002 - - ml_g: *id002 - ml_m: *id001 - ml_r: *id001 - - ml_g: *id001 - ml_m: *id002 - ml_r: *id001 - score: - - partialling out - - IV-type -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/plm/pliv_late_coverage.csv b/results/plm/pliv_late_coverage.csv deleted file mode 100644 index dabc338..0000000 --- a/results/plm/pliv_late_coverage.csv +++ /dev/null @@ -1,33 +0,0 @@ -Learner g,Learner m,Learner r,Score,level,Coverage,CI Length,Bias,repetition -LassoCV,LassoCV,LassoCV,IV-type,0.9,0.7766143106457243,0.22989223679605483,0.07287153111653735,573 -LassoCV,LassoCV,LassoCV,IV-type,0.95,0.8534031413612565,0.2739334960039667,0.07287153111653735,573 -LassoCV,LassoCV,LassoCV,partialling out,0.9,0.881326352530541,0.29700780857260933,0.07271080850444225,573 -LassoCV,LassoCV,LassoCV,partialling out,0.95,0.9476439790575916,0.35390663241468806,0.07271080850444225,573 -LassoCV,LassoCV,RF Regr.,IV-type,0.9,0.7975567190226877,0.23015212335471688,0.07216217057387413,573 -LassoCV,LassoCV,RF Regr.,IV-type,0.95,0.8691099476439791,0.27424316993889775,0.07216217057387413,573 -LassoCV,LassoCV,RF Regr.,partialling out,0.9,0.8987783595113438,0.30254607837243314,0.07481695240309204,573 -LassoCV,LassoCV,RF Regr.,partialling out,0.95,0.9511343804537522,0.3605058879146674,0.07481695240309204,573 -LassoCV,RF Regr.,LassoCV,IV-type,0.9,0.8289703315881326,0.2609713252114788,0.075760272295832,573 -LassoCV,RF Regr.,LassoCV,IV-type,0.95,0.8830715532286213,0.3109665139992902,0.075760272295832,573 -LassoCV,RF Regr.,LassoCV,partialling out,0.9,0.900523560209424,0.31348377219475954,0.07615050625397395,573 -LassoCV,RF Regr.,LassoCV,partialling out,0.95,0.9476439790575916,0.3735389539665188,0.07615050625397395,573 -LassoCV,RF Regr.,RF Regr.,IV-type,0.9,0.8342059336823735,0.2618973110694574,0.07635382529994407,573 -LassoCV,RF Regr.,RF Regr.,IV-type,0.95,0.9022687609075044,0.3120698942041265,0.07635382529994407,573 -LassoCV,RF Regr.,RF Regr.,partialling out,0.9,0.900523560209424,0.2969325926809845,0.08064896780776748,573 -LassoCV,RF Regr.,RF Regr.,partialling out,0.95,0.9493891797556719,0.35381700715184744,0.08064896780776748,573 -RF Regr.,LassoCV,LassoCV,IV-type,0.9,0.7888307155322862,0.2399734669407101,0.07565892605977774,573 -RF Regr.,LassoCV,LassoCV,IV-type,0.95,0.8603839441535777,0.28594602263833013,0.07565892605977774,573 -RF Regr.,LassoCV,LassoCV,partialling out,0.9,0.8952879581151832,0.32748151131844716,0.08012239409438285,573 -RF Regr.,LassoCV,LassoCV,partialling out,0.95,0.9476439790575916,0.39021828889206017,0.08012239409438285,573 -RF Regr.,LassoCV,RF Regr.,IV-type,0.9,0.7975567190226877,0.23928903589865372,0.07494587967257611,573 -RF Regr.,LassoCV,RF Regr.,IV-type,0.95,0.8568935427574171,0.2851304727496643,0.07494587967257611,573 -RF Regr.,LassoCV,RF Regr.,partialling out,0.9,0.8952879581151832,0.3140855171898558,0.07607988220815401,573 -RF Regr.,LassoCV,RF Regr.,partialling out,0.95,0.9424083769633508,0.3742559773532452,0.07607988220815401,573 -RF Regr.,RF Regr.,LassoCV,IV-type,0.9,0.8132635253054101,0.2731895612733841,0.07830955419542239,573 -RF Regr.,RF Regr.,LassoCV,IV-type,0.95,0.8708551483420593,0.3255254402426704,0.07830955419542239,573 -RF Regr.,RF Regr.,LassoCV,partialling out,0.9,0.7801047120418848,0.3462668556955946,0.10647619537980385,573 -RF Regr.,RF Regr.,LassoCV,partialling out,0.95,0.8481675392670157,0.41260240734072084,0.10647619537980385,573 -RF Regr.,RF Regr.,RF Regr.,IV-type,0.9,0.8150087260034904,0.2716740654611092,0.07804596547692132,573 -RF Regr.,RF Regr.,RF Regr.,IV-type,0.95,0.8778359511343804,0.3237196156014316,0.07804596547692132,573 -RF Regr.,RF Regr.,RF Regr.,partialling out,0.9,0.8691099476439791,0.2991970298118071,0.07619100964756245,573 -RF Regr.,RF Regr.,RF Regr.,partialling out,0.95,0.9109947643979057,0.356515250417355,0.07619100964756245,573 diff --git a/results/plm/pliv_late_coverage_metadata.csv b/results/plm/pliv_late_coverage_metadata.csv deleted file mode 100644 index 7b97bb6..0000000 --- a/results/plm/pliv_late_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,pliv_late_coverage.py,2025-05-22 16:55:09,19353.43654370308,3.12.10 diff --git a/results/plm/pliv_late_metadata.csv b/results/plm/pliv_late_metadata.csv deleted file mode 100644 index ea4b643..0000000 --- a/results/plm/pliv_late_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,PLIVLATECoverageSimulation,2025-05-28 17:55,333.0460760434469,3.12.3,scripts/plm/pliv_late_config.yml diff --git a/results/plm/plr_ate_config.yml b/results/plm/plr_ate_config.yml deleted file mode 100644 index d504ba6..0000000 --- a/results/plm/plr_ate_config.yml +++ /dev/null @@ -1,50 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - theta: - - 0.5 - n_obs: - - 500 - dim_x: - - 20 -learner_definitions: - lasso: &id001 - name: LassoCV - rf: &id002 - name: RF Regr. - params: - n_estimators: 200 - max_features: 10 - max_depth: 5 - min_samples_leaf: 20 - lgbm: &id003 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id001 - - ml_g: *id002 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id002 - ml_m: *id001 - - ml_g: *id003 - ml_m: *id003 - - ml_g: *id003 - ml_m: *id001 - - ml_g: *id001 - ml_m: *id003 - score: - - partialling out - - IV-type -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/plm/plr_ate_coverage.csv b/results/plm/plr_ate_coverage.csv deleted file mode 100644 index 6049e48..0000000 --- a/results/plm/plr_ate_coverage.csv +++ /dev/null @@ -1,29 +0,0 @@ -Learner g,Learner m,Score,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Regr.,IV-type,0.9,0.883,0.15961877895273296,0.04068372543442765,1000 -LGBM Regr.,LGBM Regr.,IV-type,0.95,0.933,0.1901975062567962,0.04068372543442765,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.9,0.817,0.14642066864789335,0.041735779410315844,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.95,0.897,0.17447098783739418,0.041735779410315844,1000 -LGBM Regr.,LassoCV,IV-type,0.9,0.888,0.14828359010688336,0.03751445724526263,1000 -LGBM Regr.,LassoCV,IV-type,0.95,0.944,0.17669079567063856,0.03751445724526263,1000 -LGBM Regr.,LassoCV,partialling out,0.9,0.904,0.1593118503428552,0.03940645884290757,1000 -LGBM Regr.,LassoCV,partialling out,0.95,0.946,0.1898317782605001,0.03940645884290757,1000 -LassoCV,LGBM Regr.,IV-type,0.9,0.888,0.1501213938923901,0.037389002407502744,1000 -LassoCV,LGBM Regr.,IV-type,0.95,0.934,0.17888067394991158,0.037389002407502744,1000 -LassoCV,LGBM Regr.,partialling out,0.9,0.49,0.13868359580763356,0.07134433277798699,1000 -LassoCV,LGBM Regr.,partialling out,0.95,0.628,0.16525169691436084,0.07134433277798699,1000 -LassoCV,LassoCV,IV-type,0.9,0.879,0.1395848933214119,0.03508684359283747,1000 -LassoCV,LassoCV,IV-type,0.95,0.936,0.16632565914262007,0.03508684359283747,1000 -LassoCV,LassoCV,partialling out,0.9,0.895,0.14655835450119167,0.034987652787981494,1000 -LassoCV,LassoCV,partialling out,0.95,0.944,0.17463505065077992,0.034987652787981494,1000 -LassoCV,RF Regr.,IV-type,0.9,0.859,0.1302363383352069,0.03561064803310554,1000 -LassoCV,RF Regr.,IV-type,0.95,0.915,0.15518616880729266,0.03561064803310554,1000 -LassoCV,RF Regr.,partialling out,0.9,0.781,0.1426668302933166,0.04651298597227526,1000 -LassoCV,RF Regr.,partialling out,0.95,0.869,0.16999801355068422,0.04651298597227526,1000 -RF Regr.,LassoCV,IV-type,0.9,0.885,0.14079639422615867,0.034650404336531236,1000 -RF Regr.,LassoCV,IV-type,0.95,0.936,0.16776925150952393,0.034650404336531236,1000 -RF Regr.,LassoCV,partialling out,0.9,0.888,0.15051167153657427,0.036505990374521784,1000 -RF Regr.,LassoCV,partialling out,0.95,0.948,0.17934571844629627,0.036505990374521784,1000 -RF Regr.,RF Regr.,IV-type,0.9,0.856,0.13145465044384871,0.03623204217504653,1000 -RF Regr.,RF Regr.,IV-type,0.95,0.909,0.1566378772242249,0.03623204217504653,1000 -RF Regr.,RF Regr.,partialling out,0.9,0.886,0.14234151080307436,0.03605936619645565,1000 -RF Regr.,RF Regr.,partialling out,0.95,0.931,0.1696103714688016,0.03605936619645565,1000 diff --git a/results/plm/plr_ate_coverage_metadata.csv b/results/plm/plr_ate_coverage_metadata.csv deleted file mode 100644 index f234205..0000000 --- a/results/plm/plr_ate_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,plr_ate_coverage.py,2025-05-22 13:19:23,6440.37045955658,3.12.3 diff --git a/results/plm/plr_ate_metadata.csv b/results/plm/plr_ate_metadata.csv deleted file mode 100644 index 9eb81ce..0000000 --- a/results/plm/plr_ate_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,PLRATECoverageSimulation,2025-05-28 15:36,194.11949107646942,3.12.3,scripts/plm/plr_ate_config.yml diff --git a/results/plm/plr_ate_sensitivity_config.yml b/results/plm/plr_ate_sensitivity_config.yml deleted file mode 100644 index f575860..0000000 --- a/results/plm/plr_ate_sensitivity_config.yml +++ /dev/null @@ -1,49 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - theta: - - 0.5 - n_obs: - - 1000 -learner_definitions: - lasso: &id001 - name: LassoCV - rf: &id002 - name: RF Regr. - params: - n_estimators: 200 - max_features: 10 - max_depth: 5 - min_samples_leaf: 2 - lgbm: &id003 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.05 - min_child_samples: 5 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id001 - - ml_g: *id002 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id002 - ml_m: *id001 - - ml_g: *id003 - ml_m: *id003 - - ml_g: *id003 - ml_m: *id001 - - ml_g: *id001 - ml_m: *id003 - score: - - partialling out - - IV-type -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/plm/plr_ate_sensitivity_coverage.csv b/results/plm/plr_ate_sensitivity_coverage.csv deleted file mode 100644 index 434f452..0000000 --- a/results/plm/plr_ate_sensitivity_coverage.csv +++ /dev/null @@ -1,29 +0,0 @@ -Learner g,Learner m,Score,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition -LGBM Regr.,LGBM Regr.,IV-type,0.9,0.385,1.4663299122305629,0.7925421322234831,1.0,0.985,0.10725651393439574,0.03326460518365192,1.4913903235634014,0.2926564371041883,1000 -LGBM Regr.,LGBM Regr.,IV-type,0.95,0.576,1.7472398579028525,0.7925421322234831,1.0,0.996,0.10725651393439574,0.0186351412063672,1.4913903235634014,0.2926564371041883,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.9,0.182,1.1497967634349104,0.7729578872365019,1.0,0.97,0.10474768112597262,0.04431238181012191,1.4751597629464743,0.2712920302703979,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.95,0.318,1.3700673476033447,0.7729578872365019,1.0,0.995,0.10474768112597262,0.030097055769805838,1.4751597629464743,0.2712920302703979,1000 -LGBM Regr.,LassoCV,IV-type,0.9,0.016,1.5675897053004566,1.4881399909307544,1.0,0.349,0.18835258787429676,0.11061603940247629,2.22500572268596,0.7547108921538096,1000 -LGBM Regr.,LassoCV,IV-type,0.95,0.049,1.8678983434039613,1.4881399909307544,1.0,0.589,0.18835258787429676,0.08965505199962495,2.22500572268596,0.7547108921538096,1000 -LGBM Regr.,LassoCV,partialling out,0.9,0.033,1.5608447161075107,1.3368140143494018,1.0,0.538,0.17297048595824702,0.09456285640366094,2.0670435317928293,0.609504417265797,1000 -LGBM Regr.,LassoCV,partialling out,0.95,0.093,1.8598611930595945,1.3368140143494018,1.0,0.767,0.17297048595824702,0.07389205339318292,2.0670435317928293,0.609504417265797,1000 -LassoCV,LGBM Regr.,IV-type,0.9,0.73,2.5656781122758883,1.0677570591023937,1.0,1.0,0.0711666272086956,0.011464853941823843,2.578295926341774,0.5631859759280742,1000 -LassoCV,LGBM Regr.,IV-type,0.95,0.891,3.057194034525367,1.0677570591023937,1.0,1.0,0.0711666272086956,0.003910618305845746,2.578295926341774,0.5631859759280742,1000 -LassoCV,LGBM Regr.,partialling out,0.9,0.641,2.027924924331959,0.9237208344896396,1.0,1.0,0.06130931642754562,0.011850855336480514,2.4482326562106262,0.6449506923859979,1000 -LassoCV,LGBM Regr.,partialling out,0.95,0.857,2.416421589079803,0.9237208344896396,1.0,1.0,0.06130931642754562,0.004197479989729969,2.4482326562106262,0.6449506923859979,1000 -LassoCV,LassoCV,IV-type,0.9,0.0,2.6390903058880437,4.873595405404249,1.0,0.0,0.2835323346090944,0.2235995852752004,6.403190100985858,3.344000709822638,1000 -LassoCV,LassoCV,IV-type,0.95,0.0,3.144670058621553,4.873595405404249,1.0,0.0,0.2835323346090944,0.20668125629682246,6.403190100985858,3.344000709822638,1000 -LassoCV,LassoCV,partialling out,0.9,0.0,2.6512157416595166,4.8742982878654155,1.0,0.0,0.28358055454345027,0.22344281290628232,6.403739456275626,3.3448571194552046,1000 -LassoCV,LassoCV,partialling out,0.95,0.0,3.159118406498553,4.8742982878654155,1.0,0.0,0.28358055454345027,0.20645960241254674,6.403739456275626,3.3448571194552046,1000 -LassoCV,RF Regr.,IV-type,0.9,0.033,2.2828984031735264,1.7549860217797124,1.0,0.992,0.10540674548204913,0.05179748157769331,3.4080328923671543,0.34365428027936945,1000 -LassoCV,RF Regr.,IV-type,0.95,0.117,2.720241228319411,1.7549860217797124,1.0,1.0,0.10540674548204913,0.03703505470417302,3.4080328923671543,0.34365428027936945,1000 -LassoCV,RF Regr.,partialling out,0.9,0.037,2.3209438874895922,1.7056607970976996,1.0,0.998,0.10089770825684756,0.0472619615205151,3.386769951832876,0.32870269184314155,1000 -LassoCV,RF Regr.,partialling out,0.95,0.147,2.7655752190235416,1.7056607970976996,1.0,1.0,0.10089770825684756,0.03263938744702106,3.386769951832876,0.32870269184314155,1000 -RF Regr.,LassoCV,IV-type,0.9,0.002,2.0336458826639054,2.5168441311266254,1.0,0.15,0.18897217034656957,0.13108215640403256,3.7726967118723413,1.2630723489756102,1000 -RF Regr.,LassoCV,IV-type,0.95,0.004,2.4232385313924456,2.5168441311266254,1.0,0.296,0.18897217034656957,0.11456339186569718,3.7726967118723413,1.2630723489756102,1000 -RF Regr.,LassoCV,partialling out,0.9,0.004,2.0042262177980654,2.2020124873834774,1.0,0.326,0.1673566906972531,0.10968485912427584,3.461182283094903,0.945607640026297,1000 -RF Regr.,LassoCV,partialling out,0.95,0.013,2.388182838515291,2.2020124873834774,1.0,0.586,0.1673566906972531,0.093218820211599,3.461182283094903,0.945607640026297,1000 -RF Regr.,RF Regr.,IV-type,0.9,0.017,1.832711921828132,1.666112390968602,1.0,0.903,0.12210557487205914,0.07016425910777375,2.9998575741792273,0.43566994445193274,1000 -RF Regr.,RF Regr.,IV-type,0.95,0.06,2.1838109494750206,1.666112390968602,1.0,0.967,0.12210557487205914,0.05560391034447919,2.9998575741792273,0.43566994445193274,1000 -RF Regr.,RF Regr.,partialling out,0.9,0.016,1.8212178132592944,1.6223242047256947,1.0,0.919,0.1187876211842705,0.0672786914116626,2.9607373471379326,0.4017526397593657,1000 -RF Regr.,RF Regr.,partialling out,0.95,0.06,2.170114874359165,1.6223242047256947,1.0,0.983,0.1187876211842705,0.05284651175118434,2.9607373471379326,0.4017526397593657,1000 diff --git a/results/plm/plr_ate_sensitivity_metadata.csv b/results/plm/plr_ate_sensitivity_metadata.csv deleted file mode 100644 index d9b4047..0000000 --- a/results/plm/plr_ate_sensitivity_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,PLRATESensitivityCoverageSimulation,2025-05-28 16:12,229.2524726430575,3.12.3,scripts/plm/plr_ate_sensitivity_config.yml diff --git a/results/plm/plr_cate_config.yml b/results/plm/plr_cate_config.yml deleted file mode 100644 index 20ce744..0000000 --- a/results/plm/plr_cate_config.yml +++ /dev/null @@ -1,52 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 500 - p: - - 10 - support_size: - - 5 - n_x: - - 1 -learner_definitions: - lasso: &id001 - name: LassoCV - rf: &id002 - name: RF Regr. - params: - n_estimators: 200 - max_features: 10 - max_depth: 5 - min_samples_leaf: 2 - lgbm: &id003 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id001 - - ml_g: *id002 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id002 - ml_m: *id001 - - ml_g: *id003 - ml_m: *id003 - - ml_g: *id003 - ml_m: *id001 - - ml_g: *id001 - ml_m: *id003 - score: - - partialling out - - IV-type -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/plm/plr_cate_coverage.csv b/results/plm/plr_cate_coverage.csv deleted file mode 100644 index bf7168b..0000000 --- a/results/plm/plr_cate_coverage.csv +++ /dev/null @@ -1,29 +0,0 @@ -Learner g,Learner m,Score,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Regr.,IV-type,0.9,0.81356,0.3481406151640484,0.10430800140217082,0.983,0.8752657879172515,1000 -LGBM Regr.,LGBM Regr.,IV-type,0.95,0.88477,0.4148351294587775,0.10430800140217082,0.982,0.8732288511102869,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.9,0.74012,0.4560685341081391,0.15783062735200043,0.977,1.14744422301648,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.95,0.8299500000000001,0.5434391770109168,0.15783062735200043,0.98,1.1454708849896886,1000 -LGBM Regr.,LassoCV,IV-type,0.9,0.88352,0.36584955467139657,0.09269456203727587,0.997,0.9190652139310769,1000 -LGBM Regr.,LassoCV,IV-type,0.95,0.93592,0.4359366323950171,0.09269456203727587,0.997,0.9182701523375287,1000 -LGBM Regr.,LassoCV,partialling out,0.9,0.8598600000000001,0.6464984966564067,0.17497861883142152,0.997,1.6227149811912818,1000 -LGBM Regr.,LassoCV,partialling out,0.95,0.9182,0.7703504729805534,0.17497861883142152,0.993,1.6256277670143164,1000 -LassoCV,LGBM Regr.,IV-type,0.9,0.76988,0.35642448629422974,0.11632423194012062,0.967,0.8974918718098766,1000 -LassoCV,LGBM Regr.,IV-type,0.95,0.85114,0.42470597073102984,0.11632423194012062,0.969,0.8968672652695018,1000 -LassoCV,LGBM Regr.,partialling out,0.9,0.11622,0.5619854399953333,0.5256323682345825,0.234,1.410142913244516,1000 -LassoCV,LGBM Regr.,partialling out,0.95,0.17595,0.6696469547069583,0.5256323682345825,0.229,1.40931387611625,1000 -LassoCV,LassoCV,IV-type,0.9,0.89151,0.36264059342442617,0.08911225886649077,0.999,0.9175790450518392,1000 -LassoCV,LassoCV,IV-type,0.95,0.9436899999999999,0.432112919227601,0.08911225886649077,0.997,0.9122947604257755,1000 -LassoCV,LassoCV,partialling out,0.9,0.88915,0.37658641629597944,0.09294192591629252,0.998,0.9493314635761626,1000 -LassoCV,LassoCV,partialling out,0.95,0.9430700000000001,0.4487303921231547,0.09294192591629252,0.999,0.9492044628231284,1000 -LassoCV,RF Regr.,IV-type,0.9,0.8881699999999999,0.36007271468635116,0.08945011832697905,0.997,0.905729315492467,1000 -LassoCV,RF Regr.,IV-type,0.95,0.94083,0.42905310298570154,0.08945011832697905,0.996,0.905560466242843,1000 -LassoCV,RF Regr.,partialling out,0.9,0.75973,0.4327531492388262,0.14496239788299442,0.979,1.0861336481238588,1000 -LassoCV,RF Regr.,partialling out,0.95,0.8421900000000001,0.5156571823818642,0.14496239788299442,0.978,1.0839002162247489,1000 -RF Regr.,LassoCV,IV-type,0.9,0.88758,0.3483083251389444,0.08779327663792416,0.998,0.8779489030402857,1000 -RF Regr.,LassoCV,IV-type,0.95,0.94112,0.41503496821966074,0.08779327663792416,0.996,0.8735133292532407,1000 -RF Regr.,LassoCV,partialling out,0.9,0.86795,0.4443503476360394,0.11714250183161505,0.994,1.1172789302634005,1000 -RF Regr.,LassoCV,partialling out,0.95,0.92418,0.5294760966047857,0.11714250183161505,0.995,1.114769235628846,1000 -RF Regr.,RF Regr.,IV-type,0.9,0.8849,0.34393624819903923,0.08639122390266744,0.996,0.8665106551289745,1000 -RF Regr.,RF Regr.,IV-type,0.95,0.9396100000000001,0.40982531722127136,0.08639122390266744,0.996,0.8683144995935241,1000 -RF Regr.,RF Regr.,partialling out,0.9,0.88555,0.3841724761396707,0.09707379630217602,0.998,0.9672632389200293,1000 -RF Regr.,RF Regr.,partialling out,0.95,0.93972,0.45776973996212145,0.09707379630217602,0.999,0.9635850313374369,1000 diff --git a/results/plm/plr_cate_coverage_metadata.csv b/results/plm/plr_cate_coverage_metadata.csv deleted file mode 100644 index fe8d34a..0000000 --- a/results/plm/plr_cate_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,plr_cate_coverage.py,2025-05-22 12:47:29,4525.569060087204,3.12.3 diff --git a/results/plm/plr_cate_metadata.csv b/results/plm/plr_cate_metadata.csv deleted file mode 100644 index f90c6c2..0000000 --- a/results/plm/plr_cate_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,PLRCATECoverageSimulation,2025-05-28 15:29,186.339128780365,3.12.3,scripts/plm/plr_cate_config.yml diff --git a/results/plm/plr_gate_config.yml b/results/plm/plr_gate_config.yml deleted file mode 100644 index 20ce744..0000000 --- a/results/plm/plr_gate_config.yml +++ /dev/null @@ -1,52 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 500 - p: - - 10 - support_size: - - 5 - n_x: - - 1 -learner_definitions: - lasso: &id001 - name: LassoCV - rf: &id002 - name: RF Regr. - params: - n_estimators: 200 - max_features: 10 - max_depth: 5 - min_samples_leaf: 2 - lgbm: &id003 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id001 - - ml_g: *id002 - ml_m: *id002 - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id002 - ml_m: *id001 - - ml_g: *id003 - ml_m: *id003 - - ml_g: *id003 - ml_m: *id001 - - ml_g: *id001 - ml_m: *id003 - score: - - partialling out - - IV-type -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/plm/plr_gate_coverage.csv b/results/plm/plr_gate_coverage.csv deleted file mode 100644 index 90e8c35..0000000 --- a/results/plm/plr_gate_coverage.csv +++ /dev/null @@ -1,29 +0,0 @@ -Learner g,Learner m,Score,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Regr.,IV-type,0.9,0.792,0.3409717982379027,0.10939094977617905,0.993,0.8022444119321607,1000 -LGBM Regr.,LGBM Regr.,IV-type,0.95,0.868,0.4062929572212102,0.10939094977617905,0.993,0.8004798193792318,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.9,0.7256666666666667,0.41261199381065716,0.14296451918548386,0.97,0.9675570982673285,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.95,0.815,0.49165751542098196,0.14296451918548386,0.967,0.9691480724040933,1000 -LGBM Regr.,LassoCV,IV-type,0.9,0.877,0.35826783725657413,0.09169163906741995,0.999,0.8408389172164787,1000 -LGBM Regr.,LassoCV,IV-type,0.95,0.93,0.42690245887919315,0.09169163906741995,0.997,0.8447866030999811,1000 -LGBM Regr.,LassoCV,partialling out,0.9,0.8436666666666667,0.5545701417184685,0.15552961423549327,0.998,1.3051628808444764,1000 -LGBM Regr.,LassoCV,partialling out,0.95,0.901,0.6608110818249359,0.15552961423549327,0.995,1.3025794190401572,1000 -LassoCV,LGBM Regr.,IV-type,0.9,0.7456666666666666,0.3540709326890836,0.12416752914751389,0.987,0.8320224393479907,1000 -LassoCV,LGBM Regr.,IV-type,0.95,0.837,0.42190153863677704,0.12416752914751389,0.984,0.8317500001409704,1000 -LassoCV,LGBM Regr.,partialling out,0.9,0.1333333333333333,0.48177803693340226,0.48575358485471176,0.177,1.1312945673131298,1000 -LassoCV,LGBM Regr.,partialling out,0.95,0.18,0.5740739391394702,0.48575358485471176,0.163,1.1319600186032173,1000 -LassoCV,LassoCV,IV-type,0.9,0.8926666666666666,0.35742311987476943,0.0870152370686674,1.0,0.8398941995619424,1000 -LassoCV,LassoCV,IV-type,0.95,0.9413333333333334,0.4258959160365205,0.0870152370686674,1.0,0.8405778234098288,1000 -LassoCV,LassoCV,partialling out,0.9,0.8856666666666666,0.3675924080229034,0.09140883729216352,0.999,0.8634618636354112,1000 -LassoCV,LassoCV,partialling out,0.95,0.937,0.4380133702538254,0.09140883729216352,0.998,0.8645135749451283,1000 -LassoCV,RF Regr.,IV-type,0.9,0.8906666666666666,0.35602270119404505,0.08734257475647278,1.0,0.8374639259045487,1000 -LassoCV,RF Regr.,IV-type,0.95,0.9416666666666667,0.4242272142550836,0.08734257475647278,0.999,0.836868604185174,1000 -LassoCV,RF Regr.,partialling out,0.9,0.738,0.40491199800883637,0.13642422484844158,0.983,0.9500817559029185,1000 -LassoCV,RF Regr.,partialling out,0.95,0.8273333333333334,0.4824824045142146,0.13642422484844158,0.983,0.9523261430293178,1000 -RF Regr.,LassoCV,IV-type,0.9,0.8773333333333334,0.34720295214031466,0.08839198383861967,0.998,0.8185421027515423,1000 -RF Regr.,LassoCV,IV-type,0.95,0.9343333333333333,0.41371783505273413,0.08839198383861967,0.998,0.8146669237329567,1000 -RF Regr.,LassoCV,partialling out,0.9,0.8533333333333334,0.41301433989509856,0.10966336398968128,0.996,0.9683790675299301,1000 -RF Regr.,LassoCV,partialling out,0.95,0.9136666666666666,0.49213694035089944,0.10966336398968128,0.998,0.9669097488144158,1000 -RF Regr.,RF Regr.,IV-type,0.9,0.8783333333333334,0.3444973770845564,0.08786209409395863,1.0,0.8103855381771613,1000 -RF Regr.,RF Regr.,IV-type,0.95,0.9303333333333333,0.4104939435283654,0.08786209409395863,0.998,0.8085416729445144,1000 -RF Regr.,RF Regr.,partialling out,0.9,0.8793333333333334,0.3693751123936641,0.0927826393282892,0.999,0.8668280374940234,1000 -RF Regr.,RF Regr.,partialling out,0.95,0.9333333333333333,0.44013759353091364,0.0927826393282892,0.998,0.8658046899691123,1000 diff --git a/results/plm/plr_gate_coverage_metadata.csv b/results/plm/plr_gate_coverage_metadata.csv deleted file mode 100644 index 334d6ba..0000000 --- a/results/plm/plr_gate_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.10.dev0,plr_gate_coverage.py,2025-05-22 11:58:55,1613.3821394443512,3.12.3 diff --git a/results/plm/plr_gate_metadata.csv b/results/plm/plr_gate_metadata.csv deleted file mode 100644 index ef66b20..0000000 --- a/results/plm/plr_gate_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,PLRGATECoverageSimulation,2025-05-28 15:27,184.28431547085444,3.12.3,scripts/plm/plr_gate_config.yml diff --git a/results/rdd/rdd_fuzzy_config.yml b/results/rdd/rdd_fuzzy_config.yml deleted file mode 100644 index 1c010bd..0000000 --- a/results/rdd/rdd_fuzzy_config.yml +++ /dev/null @@ -1,63 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 2000 - fuzzy: - - true - cutoff: - - 0.0 -learner_definitions: - lgbmr: &id001 - name: LGBM Regr. - params: - n_estimators: 200 - learning_rate: 0.02 - max_depth: 5 - lgbmc: &id002 - name: LGBM Clas. - params: - n_estimators: 200 - learning_rate: 0.02 - max_depth: 5 - global_linear: &id003 - name: Global Linear - global_logistic: &id004 - name: Global Logistic - local_linear: &id005 - name: Linear - local_logistic: &id006 - name: Logistic - stacked_reg: &id007 - name: Stacked Regr. - params: - n_estimators: 200 - learning_rate: 0.02 - max_depth: 5 - stacked_cls: &id008 - name: Stacked Clas. - params: - n_estimators: 200 - learning_rate: 0.02 - max_depth: 5 -dml_parameters: - fs_specification: - - cutoff - - cutoff and score - - interacted cutoff and score - learners: - - ml_g: *id001 - ml_m: *id002 - - ml_g: *id003 - ml_m: *id004 - - ml_g: *id005 - ml_m: *id006 - - ml_g: *id007 - ml_m: *id008 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/rdd/rdd_fuzzy_coverage.csv b/results/rdd/rdd_fuzzy_coverage.csv deleted file mode 100644 index a9a6dea..0000000 --- a/results/rdd/rdd_fuzzy_coverage.csv +++ /dev/null @@ -1,27 +0,0 @@ -Method,fs_specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -RDFlex,cutoff,Global Linear,Global Logistic,0.9,0.8943333333333334,9.45138989718295,2.373538922365825,1000 -RDFlex,cutoff,Global Linear,Global Logistic,0.95,0.9466666666666667,11.26202568957878,2.373538922365825,1000 -RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.9,0.9106666666666666,2.098970644289801,0.5229709395139468,1000 -RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.95,0.9606666666666667,2.5010777858935986,0.5229709395139468,1000 -RDFlex,cutoff,Linear,Logistic,0.9,0.898,9.475050602811462,2.38122109508206,1000 -RDFlex,cutoff,Linear,Logistic,0.95,0.9516666666666667,11.290219159271665,2.38122109508206,1000 -RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.9,0.9143333333333333,2.006664818935384,0.4926541115149405,1000 -RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.95,0.9643333333333334,2.3910886109946707,0.4926541115149405,1000 -RDFlex,cutoff and score,Global Linear,Global Logistic,0.9,0.896,9.45192675987028,2.3708700911814633,1000 -RDFlex,cutoff and score,Global Linear,Global Logistic,0.95,0.9483333333333334,11.262665400927295,2.3708700911814633,1000 -RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.9206666666666666,2.137979587602868,0.5310240118035273,1000 -RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9706666666666667,2.547559808801788,0.5310240118035273,1000 -RDFlex,cutoff and score,Linear,Logistic,0.9,0.8993333333333333,9.431787596286874,2.370891486466498,1000 -RDFlex,cutoff and score,Linear,Logistic,0.95,0.9506666666666667,11.238668107395837,2.370891486466498,1000 -RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.9206666666666666,2.021432595475123,0.4848163059226252,1000 -RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.968,2.408685502095107,0.4848163059226252,1000 -RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.9,0.8986666666666666,9.417430458001911,2.3532260082168035,1000 -RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.95,0.9506666666666667,11.221560521955702,2.3532260082168035,1000 -RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.9203333333333333,2.1443333037007934,0.5343555930536144,1000 -RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9696666666666667,2.555130728496937,0.5343555930536144,1000 -RDFlex,interacted cutoff and score,Linear,Logistic,0.9,0.8983333333333333,9.463034507707663,2.36467701566644,1000 -RDFlex,interacted cutoff and score,Linear,Logistic,0.95,0.9486666666666667,11.275901098836155,2.36467701566644,1000 -RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.9233333333333333,2.0607292105190114,0.5038394189514153,1000 -RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.9763333333333334,2.455510307012918,0.5038394189514153,1000 -rdrobust,cutoff,Linear,Logistic,0.9,0.925,10.188022696252679,2.4726265121800406,1000 -rdrobust,cutoff,Linear,Logistic,0.95,0.97,12.13977780827851,2.4726265121800406,1000 diff --git a/results/rdd/rdd_fuzzy_coverage_metadata.csv b/results/rdd/rdd_fuzzy_coverage_metadata.csv deleted file mode 100644 index 6c9e78a..0000000 --- a/results/rdd/rdd_fuzzy_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.dev0,rdd_fuzzy_coverage.py,2025-05-22 17:03:51,19875.749115228653,3.12.10 diff --git a/results/rdd/rdd_fuzzy_metadata.csv b/results/rdd/rdd_fuzzy_metadata.csv deleted file mode 100644 index 0c28df1..0000000 --- a/results/rdd/rdd_fuzzy_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,RDDCoverageSimulation,2025-06-03 10:17,24.518820464611053,3.12.9,scripts/rdd/rdd_fuzzy_config.yml diff --git a/results/rdd/rdd_sharp_config.yml b/results/rdd/rdd_sharp_config.yml deleted file mode 100644 index 57d0a43..0000000 --- a/results/rdd/rdd_sharp_config.yml +++ /dev/null @@ -1,41 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - n_obs: - - 1000 - fuzzy: - - false - cutoff: - - 0.0 -learner_definitions: - lgbmr: &id001 - name: LGBM Regr. - params: - n_estimators: 100 - learning_rate: 0.05 - global_linear: &id002 - name: Global Linear - local_linear: &id003 - name: Linear - stacked_reg: &id004 - name: Stacked Regr. - params: - n_estimators: 100 - learning_rate: 0.05 -dml_parameters: - fs_specification: - - cutoff - - cutoff and score - - interacted cutoff and score - learners: - - ml_g: *id001 - - ml_g: *id002 - - ml_g: *id003 - - ml_g: *id004 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/rdd/rdd_sharp_coverage.csv b/results/rdd/rdd_sharp_coverage.csv deleted file mode 100644 index f9b942e..0000000 --- a/results/rdd/rdd_sharp_coverage.csv +++ /dev/null @@ -1,27 +0,0 @@ -Method,fs_specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition -RDFlex,cutoff,Global Linear,N/A,0.9,0.8783333333333334,1.9718196650713942,0.5093136469698075,1000 -RDFlex,cutoff,Global Linear,N/A,0.95,0.9376666666666666,2.3495680492315225,0.5093136469698075,1000 -RDFlex,cutoff,LGBM Regr.,N/A,0.9,0.8753333333333334,0.5757111703216161,0.15519089437437322,1000 -RDFlex,cutoff,LGBM Regr.,N/A,0.95,0.9316666666666666,0.6860021711591865,0.15519089437437322,1000 -RDFlex,cutoff,Linear,N/A,0.9,0.8823333333333334,1.9835872651996145,0.5100383308045482,1000 -RDFlex,cutoff,Linear,N/A,0.95,0.942,2.363590009640563,0.5100383308045482,1000 -RDFlex,cutoff,Stacked Regr.,N/A,0.9,0.882,0.5590578781507813,0.14332464663035524,1000 -RDFlex,cutoff,Stacked Regr.,N/A,0.95,0.9366666666666666,0.6661585496088891,0.14332464663035524,1000 -RDFlex,cutoff and score,Global Linear,N/A,0.9,0.882,1.971849810438757,0.5080389655599933,1000 -RDFlex,cutoff and score,Global Linear,N/A,0.95,0.9383333333333334,2.34960396965226,0.5080389655599933,1000 -RDFlex,cutoff and score,LGBM Regr.,N/A,0.9,0.8846666666666666,0.6017477258893029,0.15725459328740565,1000 -RDFlex,cutoff and score,LGBM Regr.,N/A,0.95,0.9423333333333334,0.7170266406669815,0.15725459328740565,1000 -RDFlex,cutoff and score,Linear,N/A,0.9,0.8826666666666666,1.9835459135793096,0.5104954020103588,1000 -RDFlex,cutoff and score,Linear,N/A,0.95,0.9403333333333334,2.363540736145845,0.5104954020103588,1000 -RDFlex,cutoff and score,Stacked Regr.,N/A,0.9,0.8973333333333333,0.5804002753516253,0.14754701246061155,1000 -RDFlex,cutoff and score,Stacked Regr.,N/A,0.95,0.9413333333333334,0.6915895844268198,0.14754701246061155,1000 -RDFlex,interacted cutoff and score,Global Linear,N/A,0.9,0.882,1.9748055181211774,0.508213715385231,1000 -RDFlex,interacted cutoff and score,Global Linear,N/A,0.95,0.938,2.3531259125847197,0.508213715385231,1000 -RDFlex,interacted cutoff and score,LGBM Regr.,N/A,0.9,0.889,0.6013814494909888,0.15466636196782954,1000 -RDFlex,interacted cutoff and score,LGBM Regr.,N/A,0.95,0.9433333333333334,0.7165901954190156,0.15466636196782954,1000 -RDFlex,interacted cutoff and score,Linear,N/A,0.9,0.88,1.992308168049787,0.5113923914828905,1000 -RDFlex,interacted cutoff and score,Linear,N/A,0.95,0.938,2.373981606326701,0.5113923914828905,1000 -RDFlex,interacted cutoff and score,Stacked Regr.,N/A,0.9,0.8756666666666666,0.5801425055800902,0.14962735197253088,1000 -RDFlex,interacted cutoff and score,Stacked Regr.,N/A,0.95,0.929,0.6912824327993231,0.14962735197253088,1000 -rdrobust,cutoff,Linear,Logistic,0.9,0.896,2.178474435204289,0.5326223192522394,1000 -rdrobust,cutoff,Linear,Logistic,0.95,0.954,2.595812395875642,0.5326223192522394,1000 diff --git a/results/rdd/rdd_sharp_coverage_metadata.csv b/results/rdd/rdd_sharp_coverage_metadata.csv deleted file mode 100644 index 586a02d..0000000 --- a/results/rdd/rdd_sharp_coverage_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (seconds),Python Version -0.9.dev0,rdd_sharp_coverage.py,2025-05-22 12:43:00,4216.746794462204,3.12.10 diff --git a/results/rdd/rdd_sharp_metadata.csv b/results/rdd/rdd_sharp_metadata.csv deleted file mode 100644 index 375d001..0000000 --- a/results/rdd/rdd_sharp_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,RDDCoverageSimulation,2025-06-02 13:17,65.69356084664663,3.12.3,scripts/rdd/rdd_sharp_config.yml diff --git a/results/ssm/ssm_mar_ate_config.yml b/results/ssm/ssm_mar_ate_config.yml deleted file mode 100644 index 6c5f926..0000000 --- a/results/ssm/ssm_mar_ate_config.yml +++ /dev/null @@ -1,74 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - theta: - - 1.0 - n_obs: - - 500 - dim_x: - - 20 -learner_definitions: - lasso: &id001 - name: LassoCV - logit: &id002 - name: Logistic - rfr: &id003 - name: RF Regr. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - rfc: &id004 - name: RF Clas. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - lgbmr: &id005 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - lgbmc: &id006 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id002 - ml_pi: *id002 - - ml_g: *id003 - ml_m: *id004 - ml_pi: *id004 - - ml_g: *id001 - ml_m: *id004 - ml_pi: *id004 - - ml_g: *id003 - ml_m: *id002 - ml_pi: *id004 - - ml_g: *id003 - ml_m: *id004 - ml_pi: *id002 - - ml_g: *id005 - ml_m: *id006 - ml_pi: *id006 - - ml_g: *id001 - ml_m: *id006 - ml_pi: *id006 - - ml_g: *id005 - ml_m: *id002 - ml_pi: *id006 - - ml_g: *id005 - ml_m: *id006 - ml_pi: *id002 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/ssm/ssm_mar_ate_coverage.csv b/results/ssm/ssm_mar_ate_coverage.csv deleted file mode 100644 index 91f8e89..0000000 --- a/results/ssm/ssm_mar_ate_coverage.csv +++ /dev/null @@ -1,19 +0,0 @@ -Learner g,Learner m,Learner pi,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Clas.,LGBM Clas.,0.9,0.937,1.1020474646866227,0.25225717557270916,1000 -LGBM Regr.,LGBM Clas.,LGBM Clas.,0.95,0.976,1.3131705488242713,0.25225717557270916,1000 -LGBM Regr.,LGBM Clas.,Logistic,0.9,0.925,0.9396009375831231,0.21523318689504592,1000 -LGBM Regr.,LGBM Clas.,Logistic,0.95,0.967,1.1196035728213285,0.21523318689504592,1000 -LGBM Regr.,Logistic,LGBM Clas.,0.9,0.927,0.7652547530061172,0.16892900789678272,1000 -LGBM Regr.,Logistic,LGBM Clas.,0.95,0.976,0.9118572803769212,0.16892900789678272,1000 -LassoCV,LGBM Clas.,LGBM Clas.,0.9,0.94,1.0416524703569756,0.23136599807480204,1000 -LassoCV,LGBM Clas.,LGBM Clas.,0.95,0.979,1.2412054743683776,0.23136599807480204,1000 -LassoCV,Logistic,Logistic,0.9,0.918,0.5983979591318811,0.13819869402679225,1000 -LassoCV,Logistic,Logistic,0.95,0.961,0.7130351473854032,0.13819869402679225,1000 -LassoCV,RF Clas.,RF Clas.,0.9,0.915,0.5159726010062256,0.11685996635828219,1000 -LassoCV,RF Clas.,RF Clas.,0.95,0.966,0.6148192753515406,0.11685996635828219,1000 -RF Regr.,Logistic,RF Clas.,0.9,0.914,0.5740315353572116,0.13374033079721487,1000 -RF Regr.,Logistic,RF Clas.,0.95,0.967,0.6840007626548271,0.13374033079721487,1000 -RF Regr.,RF Clas.,Logistic,0.9,0.916,0.5717174746911912,0.13251444813486235,1000 -RF Regr.,RF Clas.,Logistic,0.95,0.964,0.6812433893000642,0.13251444813486235,1000 -RF Regr.,RF Clas.,RF Clas.,0.9,0.917,0.5204895028483878,0.12032384089569902,1000 -RF Regr.,RF Clas.,RF Clas.,0.95,0.957,0.6202014958648324,0.12032384089569902,1000 diff --git a/results/ssm/ssm_mar_ate_metadata.csv b/results/ssm/ssm_mar_ate_metadata.csv deleted file mode 100644 index b21daf8..0000000 --- a/results/ssm/ssm_mar_ate_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,SSMMarATECoverageSimulation,2025-06-05 10:19,32.703589328130086,3.12.9,scripts/ssm/ssm_mar_ate_config.yml diff --git a/results/ssm/ssm_nonig_ate_config.yml b/results/ssm/ssm_nonig_ate_config.yml deleted file mode 100644 index 6c5f926..0000000 --- a/results/ssm/ssm_nonig_ate_config.yml +++ /dev/null @@ -1,74 +0,0 @@ -simulation_parameters: - repetitions: 1000 - max_runtime: 19800 - random_seed: 42 - n_jobs: -2 -dgp_parameters: - theta: - - 1.0 - n_obs: - - 500 - dim_x: - - 20 -learner_definitions: - lasso: &id001 - name: LassoCV - logit: &id002 - name: Logistic - rfr: &id003 - name: RF Regr. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - rfc: &id004 - name: RF Clas. - params: - n_estimators: 200 - max_features: 20 - max_depth: 5 - min_samples_leaf: 2 - lgbmr: &id005 - name: LGBM Regr. - params: - n_estimators: 500 - learning_rate: 0.01 - lgbmc: &id006 - name: LGBM Clas. - params: - n_estimators: 500 - learning_rate: 0.01 -dml_parameters: - learners: - - ml_g: *id001 - ml_m: *id002 - ml_pi: *id002 - - ml_g: *id003 - ml_m: *id004 - ml_pi: *id004 - - ml_g: *id001 - ml_m: *id004 - ml_pi: *id004 - - ml_g: *id003 - ml_m: *id002 - ml_pi: *id004 - - ml_g: *id003 - ml_m: *id004 - ml_pi: *id002 - - ml_g: *id005 - ml_m: *id006 - ml_pi: *id006 - - ml_g: *id001 - ml_m: *id006 - ml_pi: *id006 - - ml_g: *id005 - ml_m: *id002 - ml_pi: *id006 - - ml_g: *id005 - ml_m: *id006 - ml_pi: *id002 -confidence_parameters: - level: - - 0.95 - - 0.9 diff --git a/results/ssm/ssm_nonig_ate_coverage.csv b/results/ssm/ssm_nonig_ate_coverage.csv deleted file mode 100644 index 3fe8260..0000000 --- a/results/ssm/ssm_nonig_ate_coverage.csv +++ /dev/null @@ -1,19 +0,0 @@ -Learner g,Learner m,Learner pi,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Clas.,LGBM Clas.,0.9,0.906,1.5821140101993176,0.39941487645605,1000 -LGBM Regr.,LGBM Clas.,LGBM Clas.,0.95,0.958,1.8852051201504163,0.39941487645605,1000 -LGBM Regr.,LGBM Clas.,Logistic,0.9,0.935,2.3216975230755064,0.6217489058523816,1000 -LGBM Regr.,LGBM Clas.,Logistic,0.95,0.974,2.7664732312123816,0.6217489058523816,1000 -LGBM Regr.,Logistic,LGBM Clas.,0.9,0.834,1.1194612001997861,0.31096250226975036,1000 -LGBM Regr.,Logistic,LGBM Clas.,0.95,0.904,1.3339202945055102,0.31096250226975036,1000 -LassoCV,LGBM Clas.,LGBM Clas.,0.9,0.907,1.4773761863865273,0.3819267464776803,1000 -LassoCV,LGBM Clas.,LGBM Clas.,0.95,0.957,1.760402305402313,0.3819267464776803,1000 -LassoCV,Logistic,Logistic,0.9,0.857,1.8486984724368598,0.524196741214989,1000 -LassoCV,Logistic,Logistic,0.95,0.92,2.202860099452095,0.524196741214989,1000 -LassoCV,RF Clas.,RF Clas.,0.9,0.78,0.6505940752158618,0.20476023599261262,1000 -LassoCV,RF Clas.,RF Clas.,0.95,0.872,0.7752306558374706,0.20476023599261262,1000 -RF Regr.,Logistic,RF Clas.,0.9,0.708,0.7687129167631139,0.2667490928514026,1000 -RF Regr.,Logistic,RF Clas.,0.95,0.814,0.9159779366500982,0.2667490928514026,1000 -RF Regr.,RF Clas.,Logistic,0.9,0.892,1.437280071189879,0.396115809059242,1000 -RF Regr.,RF Clas.,Logistic,0.95,0.949,1.7126248372934632,0.396115809059242,1000 -RF Regr.,RF Clas.,RF Clas.,0.9,0.779,0.6653233603546214,0.20524894117905498,1000 -RF Regr.,RF Clas.,RF Clas.,0.95,0.857,0.7927816816047267,0.20524894117905498,1000 diff --git a/results/ssm/ssm_nonig_ate_metadata.csv b/results/ssm/ssm_nonig_ate_metadata.csv deleted file mode 100644 index f8e47b8..0000000 --- a/results/ssm/ssm_nonig_ate_metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,SSMNonIgnorableATECoverageSimulation,2025-06-05 10:40,19.940552759170533,3.12.9,scripts/ssm/ssm_nonig_ate_config.yml From 7e4e364b8930b8a51e548bfac6d84a6ee328b617 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Thu, 5 Jun 2025 14:23:18 +0200 Subject: [PATCH 03/35] add iivm to monte-cover --- monte-cover/src/montecover/irm/__init__.py | 2 + monte-cover/src/montecover/irm/iivm_late.py | 122 ++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 monte-cover/src/montecover/irm/iivm_late.py diff --git a/monte-cover/src/montecover/irm/__init__.py b/monte-cover/src/montecover/irm/__init__.py index 57050ae..6c09726 100644 --- a/monte-cover/src/montecover/irm/__init__.py +++ b/monte-cover/src/montecover/irm/__init__.py @@ -3,6 +3,7 @@ from montecover.irm.apo import APOCoverageSimulation from montecover.irm.apos import APOSCoverageSimulation from montecover.irm.cvar import CVARCoverageSimulation +from montecover.irm.iivm_late import IIVMLATECoverageSimulation from montecover.irm.irm_ate import IRMATECoverageSimulation from montecover.irm.irm_ate_sensitivity import IRMATESensitivityCoverageSimulation from montecover.irm.irm_atte import IRMATTECoverageSimulation @@ -17,6 +18,7 @@ "APOSCoverageSimulation", "CVARCoverageSimulation", "IRMATECoverageSimulation", + "IIVMLATECoverageSimulation", "IRMATESensitivityCoverageSimulation", "IRMATTECoverageSimulation", "IRMATTESensitivityCoverageSimulation", diff --git a/monte-cover/src/montecover/irm/iivm_late.py b/monte-cover/src/montecover/irm/iivm_late.py new file mode 100644 index 0000000..2f1ac1f --- /dev/null +++ b/monte-cover/src/montecover/irm/iivm_late.py @@ -0,0 +1,122 @@ +from typing import Any, Dict, Optional + +import doubleml as dml +from doubleml.datasets import make_iivm_data + +from montecover.base import BaseSimulation +from montecover.utils import create_learner_from_config + + +class IIVMLATECoverageSimulation(BaseSimulation): + """Simulation class for coverage properties of DoubleMLIIVM for LATE estimation.""" + + def __init__( + self, + config_file: str, + suppress_warnings: bool = True, + log_level: str = "INFO", + log_file: Optional[str] = None, + ): + super().__init__( + config_file=config_file, + suppress_warnings=suppress_warnings, + log_level=log_level, + log_file=log_file, + ) + + # Calculate oracle values + self._calculate_oracle_values() + + def _process_config_parameters(self): + """Process simulation-specific parameters from config""" + # Process ML models in parameter grid + assert "learners" in self.dml_parameters, "No learners specified in the config file" + + required_learners = ["ml_g", "ml_m", "ml_r"] + for learner in self.dml_parameters["learners"]: + for ml in required_learners: + assert ml in learner, f"No {ml} specified in the config file" + + def _calculate_oracle_values(self): + """Calculate oracle values for the simulation.""" + self.logger.info("Calculating oracle values") + + self.oracle_values = dict() + self.oracle_values["theta"] = self.dgp_parameters["theta"] + + def run_single_rep(self, dml_data: dml.DoubleMLData, dml_params: Dict[str, Any]) -> Dict[str, Any]: + """Run a single repetition with the given parameters.""" + # Extract parameters + learner_config = dml_params["learners"] + learner_g_name, ml_g = create_learner_from_config(learner_config["ml_g"]) + learner_m_name, ml_m = create_learner_from_config(learner_config["ml_m"]) + learner_r_name, ml_r = create_learner_from_config(learner_config["ml_r"]) + + # Model + dml_model = dml.DoubleMLIIVM( + obj_dml_data=dml_data, + ml_g=ml_g, + ml_m=ml_m, + ml_r=ml_r, + ) + dml_model.fit() + + result = { + "coverage": [], + } + for level in self.confidence_parameters["level"]: + level_result = dict() + level_result["coverage"] = self._compute_coverage( + thetas=dml_model.coef, + oracle_thetas=self.oracle_values["theta"], + confint=dml_model.confint(level=level), + joint_confint=None, + ) + + # add parameters to the result + for res_metric in level_result.values(): + res_metric.update( + { + "Learner g": learner_g_name, + "Learner m": learner_m_name, + "Learner r": learner_r_name, + "level": level, + } + ) + for key, res in level_result.items(): + result[key].append(res) + + return result + + def summarize_results(self): + """Summarize the simulation results.""" + self.logger.info("Summarizing simulation results") + + # Group by parameter combinations + groupby_cols = ["Learner g", "Learner m", "Learner r", "level"] + aggregation_dict = { + "Coverage": "mean", + "CI Length": "mean", + "Bias": "mean", + "repetition": "count", + } + + # Aggregate results (possibly multiple result dfs) + result_summary = dict() + for result_name, result_df in self.results.items(): + result_summary[result_name] = result_df.groupby(groupby_cols).agg(aggregation_dict).reset_index() + self.logger.debug(f"Summarized {result_name} results") + + return result_summary + + def _generate_dml_data(self, dgp_params: Dict[str, Any]) -> dml.DoubleMLData: + """Generate data for the simulation.""" + data = make_iivm_data( + theta=dgp_params["theta"], + n_obs=dgp_params["n_obs"], + dim_x=dgp_params["dim_x"], + alpha_x=dgp_params["alpha_x"], + return_type="DataFrame", + ) + dml_data = dml.DoubleMLData(data, "y", "d", z_cols="z") + return dml_data From 6b3b92a8c570dc43d35a24d1520a690a1450ca60 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Thu, 5 Jun 2025 14:23:31 +0200 Subject: [PATCH 04/35] update iivm scripts --- scripts/irm/iivm_late.py | 13 ++++++ scripts/irm/iivm_late_config.yml | 80 ++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 scripts/irm/iivm_late.py create mode 100644 scripts/irm/iivm_late_config.yml diff --git a/scripts/irm/iivm_late.py b/scripts/irm/iivm_late.py new file mode 100644 index 0000000..c6b7942 --- /dev/null +++ b/scripts/irm/iivm_late.py @@ -0,0 +1,13 @@ +from montecover.irm import IIVMLATECoverageSimulation + +# Create and run simulation with config file +sim = IIVMLATECoverageSimulation( + config_file="scripts/irm/iivm_late_config.yml", + log_level="INFO", + log_file="logs/irm/iivm_late_sim.log", +) +sim.run_simulation() +sim.save_results(output_path="results/irm/", file_prefix="iivm_late") + +# Save config file for reproducibility +sim.save_config("results/irm/iivm_late_config.yml") diff --git a/scripts/irm/iivm_late_config.yml b/scripts/irm/iivm_late_config.yml new file mode 100644 index 0000000..07a7185 --- /dev/null +++ b/scripts/irm/iivm_late_config.yml @@ -0,0 +1,80 @@ +# Simulation parameters for IIVM LATE Coverage + +simulation_parameters: + repetitions: 200 + max_runtime: 19800 # 5.5 hours in seconds + random_seed: 42 + n_jobs: -2 + +dgp_parameters: + theta: [0.5] # Treatment effect + n_obs: [500] # Sample size + dim_x: [20] # Number of covariates + alpha_x: [1.0] # Covariate effect + +# Define reusable learner configurations +learner_definitions: + lasso: &lasso + name: "LassoCV" + + logit: &logit + name: "Logistic" + + lgbmr: &lgbmr + name: "LGBM Regr." + params: + n_estimators: 100 # Fewer trees; with small data, fewer is often better + learning_rate: 0.05 # Reasonable speed without sacrificing much accuracy + num_leaves: 7 # Smaller trees reduce overfitting risk + max_depth: 3 # Shallow trees generalize better on tiny datasets + min_child_samples: 20 # Avoids splitting on noise + subsample: 1.0 # Use all rows — subsampling adds variance with small data + colsample_bytree: 0.8 # Still good to randomly drop some features per tree + reg_alpha: 0.1 # L1 regularization helps when there are many features + reg_lambda: 1.0 # Stronger L2 regularization improves generalization + random_state: 42 # Reproducibility + + lgbmc: &lgbmc + name: "LGBM Clas." + params: + n_estimators: 100 # Fewer trees; with small data, fewer is often better + learning_rate: 0.05 # Reasonable speed without sacrificing much accuracy + num_leaves: 7 # Smaller trees reduce overfitting risk + max_depth: 3 # Shallow trees generalize better on tiny datasets + min_child_samples: 20 # Avoids splitting on noise + subsample: 1.0 # Use all rows — subsampling adds variance with small data + colsample_bytree: 0.8 # Still good to randomly drop some features per tree + reg_alpha: 0.1 # L1 regularization helps when there are many features + reg_lambda: 1.0 # Stronger L2 regularization improves generalization + random_state: 42 # Reproducibility + +dml_parameters: + learners: + - ml_g: *lasso + ml_m: *logit + ml_r: *logit + - ml_g: *lasso + ml_m: *logit + ml_r: *lgbmc + - ml_g: *lasso + ml_m: *lgbmc + ml_r: *logit + - ml_g: *lasso + ml_m: *lgbmc + ml_r: *lgbmc + - ml_g: *lgbmr + ml_m: *logit + ml_r: *logit + - ml_g: *lgbmr + ml_m: *logit + ml_r: *lgbmc + - ml_g: *lgbmr + ml_m: *lgbmc + ml_r: *logit + - ml_g: *lgbmr + ml_m: *lgbmc + ml_r: *lgbmc + + +confidence_parameters: + level: [0.95, 0.90] # Confidence levels From 4dcb55adeab121e22b6cac7a20737df23d38b035 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Thu, 5 Jun 2025 14:23:48 +0200 Subject: [PATCH 05/35] update qmd and workflow files --- .github/workflows/iivm_sim.yml | 23 ++++++++++++++++------- doc/irm/iivm.qmd | 4 ++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/iivm_sim.yml b/.github/workflows/iivm_sim.yml index 681070b..a8dcc98 100644 --- a/.github/workflows/iivm_sim.yml +++ b/.github/workflows/iivm_sim.yml @@ -47,20 +47,27 @@ jobs: with: ref: ${{ env.TARGET_BRANCH }} + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.7.8" + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version-file: "monte-cover/pyproject.toml" - - name: Install dependencies + - name: Install Monte-Cover run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + cd monte-cover + uv venv + uv sync - name: Install DoubleML from correct branch run: | - pip uninstall -y doubleml - pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" + source monte-cover/.venv/bin/activate + uv pip uninstall doubleml + uv pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | @@ -68,7 +75,9 @@ jobs: git config --global user.email 'github-actions@github.com' - name: Run scripts - run: python ${{ matrix.script }} + run: | + source monte-cover/.venv/bin/activate + uv run ${{ matrix.script }} - name: Commit any existing changes run: | diff --git a/doc/irm/iivm.qmd b/doc/irm/iivm.qmd index 00f4184..7dd53c2 100644 --- a/doc/irm/iivm.qmd +++ b/doc/irm/iivm.qmd @@ -30,7 +30,7 @@ The simulations are based on the the [make_iivm_data](https://docs.doubleml.org ```{python} #| echo: false -metadata_file = '../../results/irm/iivm_late_coverage_metadata.csv' +metadata_file = '../../results/irm/iivm_late_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ``` @@ -46,7 +46,7 @@ df = pd.read_csv("../../results/irm/iivm_late_coverage.csv", index_col=None) assert df["repetition"].nunique() == 1 n_rep = df["repetition"].unique()[0] -display_columns = ["Learner g", "Learner m", "Bias", "CI Length", "Coverage"] +display_columns = ["Learner g", "Learner m", "Learner r", "Bias", "CI Length", "Coverage"] ``` From a3f53ec3378c385ea808e72203d9398dd1620054 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Thu, 5 Jun 2025 14:34:13 +0200 Subject: [PATCH 06/35] fix iivm workflow --- .github/workflows/iivm_sim.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/iivm_sim.yml b/.github/workflows/iivm_sim.yml index a8dcc98..b7cb787 100644 --- a/.github/workflows/iivm_sim.yml +++ b/.github/workflows/iivm_sim.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: script: [ - 'scripts/irm/iivm_late_coverage.py', + 'scripts/irm/iivm_late.py', ] steps: From c0a9a27163cfd698fdf946020ff34f7c37795a10 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 12:42:43 +0000 Subject: [PATCH 07/35] Update results from script: scripts/irm/iivm_late.py --- results/irm/iivm_late_config.yml | 75 ++++++++++++++++++++++++++++++ results/irm/iivm_late_coverage.csv | 17 +++++++ results/irm/iivm_late_metadata.csv | 2 + 3 files changed, 94 insertions(+) create mode 100644 results/irm/iivm_late_config.yml create mode 100644 results/irm/iivm_late_coverage.csv create mode 100644 results/irm/iivm_late_metadata.csv diff --git a/results/irm/iivm_late_config.yml b/results/irm/iivm_late_config.yml new file mode 100644 index 0000000..262ed8d --- /dev/null +++ b/results/irm/iivm_late_config.yml @@ -0,0 +1,75 @@ +simulation_parameters: + repetitions: 200 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 0.5 + n_obs: + - 500 + dim_x: + - 20 + alpha_x: + - 1.0 +learner_definitions: + lasso: &id001 + name: LassoCV + logit: &id002 + name: Logistic + lgbmr: &id004 + name: LGBM Regr. + params: + n_estimators: 100 + learning_rate: 0.05 + num_leaves: 7 + max_depth: 3 + min_child_samples: 20 + subsample: 1.0 + colsample_bytree: 0.8 + reg_alpha: 0.1 + reg_lambda: 1.0 + random_state: 42 + lgbmc: &id003 + name: LGBM Clas. + params: + n_estimators: 100 + learning_rate: 0.05 + num_leaves: 7 + max_depth: 3 + min_child_samples: 20 + subsample: 1.0 + colsample_bytree: 0.8 + reg_alpha: 0.1 + reg_lambda: 1.0 + random_state: 42 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + ml_r: *id002 + - ml_g: *id001 + ml_m: *id002 + ml_r: *id003 + - ml_g: *id001 + ml_m: *id003 + ml_r: *id002 + - ml_g: *id001 + ml_m: *id003 + ml_r: *id003 + - ml_g: *id004 + ml_m: *id002 + ml_r: *id002 + - ml_g: *id004 + ml_m: *id002 + ml_r: *id003 + - ml_g: *id004 + ml_m: *id003 + ml_r: *id002 + - ml_g: *id004 + ml_m: *id003 + ml_r: *id003 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/iivm_late_coverage.csv b/results/irm/iivm_late_coverage.csv new file mode 100644 index 0000000..174966e --- /dev/null +++ b/results/irm/iivm_late_coverage.csv @@ -0,0 +1,17 @@ +Learner g,Learner m,Learner r,level,Coverage,CI Length,Bias,repetition +LGBM Regr.,LGBM Clas.,LGBM Clas.,0.9,0.935,1.1168835154580528,0.23046338682103318,200 +LGBM Regr.,LGBM Clas.,LGBM Clas.,0.95,0.97,1.3308487936895623,0.23046338682103318,200 +LGBM Regr.,LGBM Clas.,Logistic,0.9,0.955,1.1132643708033312,0.2387703838551479,200 +LGBM Regr.,LGBM Clas.,Logistic,0.95,0.97,1.326536316845503,0.2387703838551479,200 +LGBM Regr.,Logistic,LGBM Clas.,0.9,0.925,1.0676961140157748,0.2404768410266537,200 +LGBM Regr.,Logistic,LGBM Clas.,0.95,0.955,1.2722383898576701,0.2404768410266537,200 +LGBM Regr.,Logistic,Logistic,0.9,0.945,1.0542837892752488,0.2314227063303991,200 +LGBM Regr.,Logistic,Logistic,0.95,0.965,1.2562566191945217,0.2314227063303991,200 +LassoCV,LGBM Clas.,LGBM Clas.,0.9,0.95,1.050618151197297,0.219243438592031,200 +LassoCV,LGBM Clas.,LGBM Clas.,0.95,0.975,1.2518887420196634,0.219243438592031,200 +LassoCV,LGBM Clas.,Logistic,0.9,0.935,1.0466111753459595,0.22595938006238492,200 +LassoCV,LGBM Clas.,Logistic,0.95,0.975,1.247114135801298,0.22595938006238492,200 +LassoCV,Logistic,LGBM Clas.,0.9,0.935,1.0013259753676422,0.21864342992355934,200 +LassoCV,Logistic,LGBM Clas.,0.95,0.96,1.1931534918048492,0.21864342992355934,200 +LassoCV,Logistic,Logistic,0.9,0.94,1.0039031105588596,0.22138652272701975,200 +LassoCV,Logistic,Logistic,0.95,0.955,1.196224337790968,0.22138652272701975,200 diff --git a/results/irm/iivm_late_metadata.csv b/results/irm/iivm_late_metadata.csv new file mode 100644 index 0000000..136e8e0 --- /dev/null +++ b/results/irm/iivm_late_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,IIVMLATECoverageSimulation,2025-06-05 12:42,4.740816859404246,3.12.3,scripts/irm/iivm_late_config.yml From 028a99ab66d8deeaca88b86c238fcac3e51409ab Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Thu, 5 Jun 2025 14:49:36 +0200 Subject: [PATCH 08/35] increase iivm repetitions --- results/irm/iivm_late_config.yml | 2 +- results/irm/iivm_late_coverage.csv | 32 +++++++++++++++--------------- results/irm/iivm_late_metadata.csv | 2 +- scripts/irm/iivm_late_config.yml | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/results/irm/iivm_late_config.yml b/results/irm/iivm_late_config.yml index 262ed8d..d549111 100644 --- a/results/irm/iivm_late_config.yml +++ b/results/irm/iivm_late_config.yml @@ -1,5 +1,5 @@ simulation_parameters: - repetitions: 200 + repetitions: 1000 max_runtime: 19800 random_seed: 42 n_jobs: -2 diff --git a/results/irm/iivm_late_coverage.csv b/results/irm/iivm_late_coverage.csv index 174966e..dcd3993 100644 --- a/results/irm/iivm_late_coverage.csv +++ b/results/irm/iivm_late_coverage.csv @@ -1,17 +1,17 @@ Learner g,Learner m,Learner r,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Clas.,LGBM Clas.,0.9,0.935,1.1168835154580528,0.23046338682103318,200 -LGBM Regr.,LGBM Clas.,LGBM Clas.,0.95,0.97,1.3308487936895623,0.23046338682103318,200 -LGBM Regr.,LGBM Clas.,Logistic,0.9,0.955,1.1132643708033312,0.2387703838551479,200 -LGBM Regr.,LGBM Clas.,Logistic,0.95,0.97,1.326536316845503,0.2387703838551479,200 -LGBM Regr.,Logistic,LGBM Clas.,0.9,0.925,1.0676961140157748,0.2404768410266537,200 -LGBM Regr.,Logistic,LGBM Clas.,0.95,0.955,1.2722383898576701,0.2404768410266537,200 -LGBM Regr.,Logistic,Logistic,0.9,0.945,1.0542837892752488,0.2314227063303991,200 -LGBM Regr.,Logistic,Logistic,0.95,0.965,1.2562566191945217,0.2314227063303991,200 -LassoCV,LGBM Clas.,LGBM Clas.,0.9,0.95,1.050618151197297,0.219243438592031,200 -LassoCV,LGBM Clas.,LGBM Clas.,0.95,0.975,1.2518887420196634,0.219243438592031,200 -LassoCV,LGBM Clas.,Logistic,0.9,0.935,1.0466111753459595,0.22595938006238492,200 -LassoCV,LGBM Clas.,Logistic,0.95,0.975,1.247114135801298,0.22595938006238492,200 -LassoCV,Logistic,LGBM Clas.,0.9,0.935,1.0013259753676422,0.21864342992355934,200 -LassoCV,Logistic,LGBM Clas.,0.95,0.96,1.1931534918048492,0.21864342992355934,200 -LassoCV,Logistic,Logistic,0.9,0.94,1.0039031105588596,0.22138652272701975,200 -LassoCV,Logistic,Logistic,0.95,0.955,1.196224337790968,0.22138652272701975,200 +LGBM Regr.,LGBM Clas.,LGBM Clas.,0.9,0.914,1.1170031612339868,0.26443161772788826,1000 +LGBM Regr.,LGBM Clas.,LGBM Clas.,0.95,0.968,1.3309913604249184,0.26443161772788826,1000 +LGBM Regr.,LGBM Clas.,Logistic,0.9,0.921,1.111120207889174,0.2669195093646889,1000 +LGBM Regr.,LGBM Clas.,Logistic,0.95,0.969,1.32398138914867,0.2669195093646889,1000 +LGBM Regr.,Logistic,LGBM Clas.,0.9,0.923,1.0567499407213161,0.25374822037756817,1000 +LGBM Regr.,Logistic,LGBM Clas.,0.95,0.965,1.2591952198915768,0.25374822037756817,1000 +LGBM Regr.,Logistic,Logistic,0.9,0.915,1.0557640391096161,0.2504657709294685,1000 +LGBM Regr.,Logistic,Logistic,0.95,0.966,1.2580204456626907,0.2504657709294685,1000 +LassoCV,LGBM Clas.,LGBM Clas.,0.9,0.92,1.0534935963070167,0.2478314316215499,1000 +LassoCV,LGBM Clas.,LGBM Clas.,0.95,0.964,1.2553150461978761,0.2478314316215499,1000 +LassoCV,LGBM Clas.,Logistic,0.9,0.915,1.048839165759454,0.2487976919643911,1000 +LassoCV,LGBM Clas.,Logistic,0.95,0.964,1.2497689501244686,0.2487976919643911,1000 +LassoCV,Logistic,LGBM Clas.,0.9,0.918,1.0011806143989663,0.24364342853702406,1000 +LassoCV,Logistic,LGBM Clas.,0.95,0.959,1.1929802835274108,0.24364342853702406,1000 +LassoCV,Logistic,Logistic,0.9,0.916,0.9983343843722963,0.24211448303931346,1000 +LassoCV,Logistic,Logistic,0.95,0.967,1.1895887912678056,0.24211448303931346,1000 diff --git a/results/irm/iivm_late_metadata.csv b/results/irm/iivm_late_metadata.csv index 136e8e0..0ab74ee 100644 --- a/results/irm/iivm_late_metadata.csv +++ b/results/irm/iivm_late_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,IIVMLATECoverageSimulation,2025-06-05 12:42,4.740816859404246,3.12.3,scripts/irm/iivm_late_config.yml +0.10.0,IIVMLATECoverageSimulation,2025-06-05 14:57,2.219698127110799,3.12.9,scripts/irm/iivm_late_config.yml diff --git a/scripts/irm/iivm_late_config.yml b/scripts/irm/iivm_late_config.yml index 07a7185..b81c856 100644 --- a/scripts/irm/iivm_late_config.yml +++ b/scripts/irm/iivm_late_config.yml @@ -1,7 +1,7 @@ # Simulation parameters for IIVM LATE Coverage simulation_parameters: - repetitions: 200 + repetitions: 1000 max_runtime: 19800 # 5.5 hours in seconds random_seed: 42 n_jobs: -2 @@ -74,7 +74,7 @@ dml_parameters: - ml_g: *lgbmr ml_m: *lgbmc ml_r: *lgbmc - + confidence_parameters: level: [0.95, 0.90] # Confidence levels From 685ce7b51ffbad6e072c8dec11b4dcf5564517f2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 13:14:03 +0000 Subject: [PATCH 09/35] Update results from script: scripts/irm/irm_ate_sensitivity.py --- results/irm/irm_ate_sensitivity_config.yml | 53 ++++++++++++++++++++ results/irm/irm_ate_sensitivity_coverage.csv | 9 ++++ results/irm/irm_ate_sensitivity_metadata.csv | 2 + 3 files changed, 64 insertions(+) create mode 100644 results/irm/irm_ate_sensitivity_config.yml create mode 100644 results/irm/irm_ate_sensitivity_coverage.csv create mode 100644 results/irm/irm_ate_sensitivity_metadata.csv diff --git a/results/irm/irm_ate_sensitivity_config.yml b/results/irm/irm_ate_sensitivity_config.yml new file mode 100644 index 0000000..74143aa --- /dev/null +++ b/results/irm/irm_ate_sensitivity_config.yml @@ -0,0 +1,53 @@ +simulation_parameters: + repetitions: 500 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 5.0 + n_obs: + - 5000 + trimming_threshold: + - 0.05 + var_epsilon_y: + - 1.0 + linear: + - false + gamma_a: + - 0.198 + beta_a: + - 0.582 +learner_definitions: + linear: &id001 + name: Linear + logit: &id002 + name: Logistic + lgbmr: &id003 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + min_child_samples: 10 + lgbmc: &id004 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 + min_child_samples: 10 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id003 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id004 + trimming_threshold: + - 0.05 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/irm_ate_sensitivity_coverage.csv b/results/irm/irm_ate_sensitivity_coverage.csv new file mode 100644 index 0000000..f538604 --- /dev/null +++ b/results/irm/irm_ate_sensitivity_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition +LGBM Regr.,LGBM Clas.,0.9,0.084,0.2668540258210539,0.18198204659615963,0.95,1.0,0.12589776145327045,0.05628191414414868,0.044383096119999466,0.32513075000880287,500 +LGBM Regr.,LGBM Clas.,0.95,0.246,0.317976184122928,0.18198204659615963,0.998,1.0,0.12589776145327045,0.03630227835056437,0.044383096119999466,0.32513075000880287,500 +LGBM Regr.,Logistic,0.9,0.26,0.2574630882167088,0.14916839267522197,1.0,1.0,0.10064526477515615,0.034829417508842296,0.026887536242060982,0.297933622145966,500 +LGBM Regr.,Logistic,0.95,0.572,0.3067861917831888,0.14916839267522197,1.0,1.0,0.10064526477515615,0.018255506137347777,0.026887536242060982,0.297933622145966,500 +Linear,LGBM Clas.,0.9,0.082,0.2672263041294733,0.1800922704825838,0.964,1.0,0.12741937328027908,0.056153203215479244,0.04433563312353054,0.31995377591144475,500 +Linear,LGBM Clas.,0.95,0.258,0.31841978108789315,0.1800922704825838,0.996,1.0,0.12741937328027908,0.03559871249886926,0.04433563312353054,0.31995377591144475,500 +Linear,Logistic,0.9,0.868,0.2588792120747325,0.08970647188763244,1.0,1.0,0.06307809186280441,0.006372043222062277,0.0574078098172687,0.23496351259089188,500 +Linear,Logistic,0.95,0.976,0.3084736074376251,0.08970647188763244,1.0,1.0,0.06307809186280441,0.001546577328924639,0.0574078098172687,0.23496351259089188,500 diff --git a/results/irm/irm_ate_sensitivity_metadata.csv b/results/irm/irm_ate_sensitivity_metadata.csv new file mode 100644 index 0000000..e47f137 --- /dev/null +++ b/results/irm/irm_ate_sensitivity_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,IRMATESensitivityCoverageSimulation,2025-06-05 13:14,37.417966898282366,3.12.3,scripts/irm/irm_ate_sensitivity_config.yml From 424860afd094445488209e14fa3091181a71e6b8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 13:14:20 +0000 Subject: [PATCH 10/35] Update results from script: scripts/irm/irm_atte_sensitivity.py --- results/irm/irm_atte_sensitivity_config.yml | 53 +++++++++++++++++++ results/irm/irm_atte_sensitivity_coverage.csv | 9 ++++ results/irm/irm_atte_sensitivity_metadata.csv | 2 + 3 files changed, 64 insertions(+) create mode 100644 results/irm/irm_atte_sensitivity_config.yml create mode 100644 results/irm/irm_atte_sensitivity_coverage.csv create mode 100644 results/irm/irm_atte_sensitivity_metadata.csv diff --git a/results/irm/irm_atte_sensitivity_config.yml b/results/irm/irm_atte_sensitivity_config.yml new file mode 100644 index 0000000..bf06bc6 --- /dev/null +++ b/results/irm/irm_atte_sensitivity_config.yml @@ -0,0 +1,53 @@ +simulation_parameters: + repetitions: 500 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 5.0 + n_obs: + - 5000 + trimming_threshold: + - 0.05 + var_epsilon_y: + - 1.0 + linear: + - false + gamma_a: + - 0.151 + beta_a: + - 0.582 +learner_definitions: + linear: &id001 + name: Linear + logit: &id002 + name: Logistic + lgbmr: &id003 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + min_child_samples: 10 + lgbmc: &id004 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 + min_child_samples: 10 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id003 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id004 + trimming_threshold: + - 0.05 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/irm_atte_sensitivity_coverage.csv b/results/irm/irm_atte_sensitivity_coverage.csv new file mode 100644 index 0000000..d592177 --- /dev/null +++ b/results/irm/irm_atte_sensitivity_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition +LGBM Regr.,LGBM Clas.,0.9,0.708,0.3489511146392163,0.13756848436233937,0.94,1.0,0.10664020023640788,0.02469821575907808,0.06297064757762019,0.26272747008699665,500 +LGBM Regr.,LGBM Clas.,0.95,0.826,0.41580089915085766,0.13756848436233937,0.978,1.0,0.10664020023640788,0.012557304495326774,0.06297064757762019,0.26272747008699665,500 +LGBM Regr.,Logistic,0.9,0.728,0.3466996212937586,0.13203748795264333,0.96,1.0,0.0989531978785329,0.02152136562411849,0.061845450573817774,0.261622864513939,500 +LGBM Regr.,Logistic,0.95,0.834,0.41311807935691197,0.13203748795264333,0.988,1.0,0.0989531978785329,0.01068777225560622,0.061845450573817774,0.261622864513939,500 +Linear,LGBM Clas.,0.9,0.77,0.3499025013245624,0.12450248762578023,0.968,1.0,0.09911082424887455,0.019375250746672557,0.061685830635415634,0.24526004965885637,500 +Linear,LGBM Clas.,0.95,0.866,0.41693454630832866,0.12450248762578023,0.988,1.0,0.09911082424887455,0.009539652027944212,0.061685830635415634,0.24526004965885637,500 +Linear,Logistic,0.9,0.936,0.3503606994029222,0.07357271764564946,0.998,1.0,0.05747974737962404,0.004779092271966848,0.0939884966306456,0.17946572893838733,500 +Linear,Logistic,0.95,0.976,0.41748052299382554,0.07357271764564946,1.0,1.0,0.05747974737962404,0.0017099292842988314,0.0939884966306456,0.17946572893838733,500 diff --git a/results/irm/irm_atte_sensitivity_metadata.csv b/results/irm/irm_atte_sensitivity_metadata.csv new file mode 100644 index 0000000..06469e7 --- /dev/null +++ b/results/irm/irm_atte_sensitivity_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,IRMATTESensitivityCoverageSimulation,2025-06-05 13:14,37.61970745722453,3.12.3,scripts/irm/irm_atte_sensitivity_config.yml From 3d8ff93a013b873529a8d741c2806aabee049da0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 13:49:51 +0000 Subject: [PATCH 11/35] Update results from script: scripts/irm/apos.py --- results/irm/apos_causal_contrast.csv | 9 +++++ results/irm/apos_config.yml | 49 ++++++++++++++++++++++++++++ results/irm/apos_coverage.csv | 9 +++++ results/irm/apos_metadata.csv | 2 ++ 4 files changed, 69 insertions(+) create mode 100644 results/irm/apos_causal_contrast.csv create mode 100644 results/irm/apos_config.yml create mode 100644 results/irm/apos_coverage.csv create mode 100644 results/irm/apos_metadata.csv diff --git a/results/irm/apos_causal_contrast.csv b/results/irm/apos_causal_contrast.csv new file mode 100644 index 0000000..aa9f305 --- /dev/null +++ b/results/irm/apos_causal_contrast.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Clas.,0.9,0.904,33.69662163571549,8.483004437185645,0.926,39.85455370375964,1000 +LGBM Regr.,LGBM Clas.,0.95,0.962,40.152001202125206,8.483004437185645,0.971,45.73641635076257,1000 +LGBM Regr.,Logistic,0.9,0.94,5.358652779061842,1.1112961153926972,0.938,6.338924854198347,1000 +LGBM Regr.,Logistic,0.95,0.9705,6.385228618842048,1.1112961153926972,0.971,7.278594548770065,1000 +Linear,LGBM Clas.,0.9,0.961,6.6486166061347785,1.2777994119295877,0.974,7.879408694900085,1000 +Linear,LGBM Clas.,0.95,0.987,7.922315324306693,1.2777994119295877,0.992,9.038546968123926,1000 +Linear,Logistic,0.9,0.863,1.1418873926566593,0.3053856763981124,0.855,1.3481716926442315,1000 +Linear,Logistic,0.95,0.9275,1.3606427510242092,0.3053856763981124,0.921,1.5482567665033886,1000 diff --git a/results/irm/apos_config.yml b/results/irm/apos_config.yml new file mode 100644 index 0000000..40be90e --- /dev/null +++ b/results/irm/apos_config.yml @@ -0,0 +1,49 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 500 + n_levels: + - 2 + linear: + - true +learner_definitions: + linear: &id001 + name: Linear + logit: &id002 + name: Logistic + lgbmr: &id003 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + min_child_samples: 10 + lgbmc: &id004 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 + min_child_samples: 10 +dml_parameters: + treatment_levels: + - - 0 + - 1 + - 2 + trimming_threshold: + - 0.01 + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id003 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id004 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/apos_coverage.csv b/results/irm/apos_coverage.csv new file mode 100644 index 0000000..d672bd2 --- /dev/null +++ b/results/irm/apos_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Clas.,0.9,0.9203333333333333,25.45613129678526,6.166381695751374,0.946,32.623943884889655,1000 +LGBM Regr.,LGBM Clas.,0.95,0.9643333333333334,30.3328513309069,6.166381695751374,0.986,36.86913655114933,1000 +LGBM Regr.,Logistic,0.9,0.917,6.627738366241373,1.5016545559757806,0.92,8.149255859131907,1000 +LGBM Regr.,Logistic,0.95,0.9596666666666667,7.897437367033678,1.5016545559757806,0.959,9.320000315514479,1000 +Linear,LGBM Clas.,0.9,0.9376666666666666,7.5134083953879935,1.6006918303560986,0.951,9.29091412660083,1000 +Linear,LGBM Clas.,0.95,0.974,8.952778298816861,1.6006918303560986,0.974,10.609555760885549,1000 +Linear,Logistic,0.9,0.915,5.39079994210578,1.2559293498489636,0.914,5.8155441975859725,1000 +Linear,Logistic,0.95,0.96,6.423534326255072,1.2559293498489636,0.959,6.833558835632949,1000 diff --git a/results/irm/apos_metadata.csv b/results/irm/apos_metadata.csv new file mode 100644 index 0000000..10c6d4c --- /dev/null +++ b/results/irm/apos_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,APOSCoverageSimulation,2025-06-05 13:49,73.850344034036,3.12.3,scripts/irm/apos_config.yml From 47a4aac6c83311b453ca66c4c196a6579f65ebc6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 13:49:58 +0000 Subject: [PATCH 12/35] Update results from script: scripts/irm/apo.py --- results/irm/apo_config.yml | 49 ++++++++++++++++++++++++++++++++++++ results/irm/apo_coverage.csv | 25 ++++++++++++++++++ results/irm/apo_metadata.csv | 2 ++ 3 files changed, 76 insertions(+) create mode 100644 results/irm/apo_config.yml create mode 100644 results/irm/apo_coverage.csv create mode 100644 results/irm/apo_metadata.csv diff --git a/results/irm/apo_config.yml b/results/irm/apo_config.yml new file mode 100644 index 0000000..5f31101 --- /dev/null +++ b/results/irm/apo_config.yml @@ -0,0 +1,49 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 500 + n_levels: + - 2 + linear: + - true +learner_definitions: + linear: &id001 + name: Linear + logit: &id002 + name: Logistic + lgbmr: &id003 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + min_child_samples: 10 + lgbmc: &id004 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 + min_child_samples: 10 +dml_parameters: + treatment_level: + - 0 + - 1 + - 2 + trimming_threshold: + - 0.01 + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id003 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id004 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/apo_coverage.csv b/results/irm/apo_coverage.csv new file mode 100644 index 0000000..cb1bc37 --- /dev/null +++ b/results/irm/apo_coverage.csv @@ -0,0 +1,25 @@ +Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition +LGBM Regr.,LGBM Clas.,0,0.9,0.921,8.43473750706522,2.0020174052425626,1000 +LGBM Regr.,LGBM Clas.,0,0.95,0.968,10.050609648188914,2.0020174052425626,1000 +LGBM Regr.,LGBM Clas.,1,0.9,0.945,34.49785903442266,8.012907711983175,1000 +LGBM Regr.,LGBM Clas.,1,0.95,0.983,41.10673444938877,8.012907711983175,1000 +LGBM Regr.,LGBM Clas.,2,0.9,0.909,33.42971449937157,8.333206392907996,1000 +LGBM Regr.,LGBM Clas.,2,0.95,0.973,39.83396173291094,8.333206392907996,1000 +LGBM Regr.,Logistic,0,0.9,0.905,5.626372580880361,1.390211734015615,1000 +LGBM Regr.,Logistic,0,0.95,0.958,6.704236438695905,1.390211734015615,1000 +LGBM Regr.,Logistic,1,0.9,0.922,7.220302724612175,1.6901658245391882,1000 +LGBM Regr.,Logistic,1,0.95,0.952,8.603521350373505,1.6901658245391882,1000 +LGBM Regr.,Logistic,2,0.9,0.91,7.160030685666201,1.6407106001549503,1000 +LGBM Regr.,Logistic,2,0.95,0.957,8.531702786293828,1.6407106001549503,1000 +Linear,LGBM Clas.,0,0.9,0.902,5.4602785529816185,1.3727521781733092,1000 +Linear,LGBM Clas.,0,0.95,0.95,6.506323197423445,1.3727521781733092,1000 +Linear,LGBM Clas.,1,0.9,0.946,9.92791552556521,2.067430586356608,1000 +Linear,LGBM Clas.,1,0.95,0.979,11.82984099790536,2.067430586356608,1000 +Linear,LGBM Clas.,2,0.9,0.93,7.18671690478672,1.5695466395151154,1000 +Linear,LGBM Clas.,2,0.95,0.971,8.563501377671649,1.5695466395151154,1000 +Linear,Logistic,0,0.9,0.897,5.347321418519404,1.344857338767798,1000 +Linear,Logistic,0,0.95,0.949,6.371726469960765,1.344857338767798,1000 +Linear,Logistic,1,0.9,0.901,5.430231052306794,1.3595988005998974,1000 +Linear,Logistic,1,0.95,0.947,6.470519392037281,1.3595988005998974,1000 +Linear,Logistic,2,0.9,0.898,5.376486330054867,1.3454318063315418,1000 +Linear,Logistic,2,0.95,0.947,6.406478605521755,1.3454318063315418,1000 diff --git a/results/irm/apo_metadata.csv b/results/irm/apo_metadata.csv new file mode 100644 index 0000000..443288c --- /dev/null +++ b/results/irm/apo_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,APOCoverageSimulation,2025-06-05 13:49,73.86938846111298,3.12.3,scripts/irm/apo_config.yml From df190c5cc01a38ee9f78a5667dd0e01640d08762 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 13:55:56 +0000 Subject: [PATCH 13/35] Update results from script: scripts/irm/irm_gate.py --- results/irm/irm_gate_config.yml | 63 +++++++++++++++++++++++++++++++ results/irm/irm_gate_coverage.csv | 15 ++++++++ results/irm/irm_gate_metadata.csv | 2 + 3 files changed, 80 insertions(+) create mode 100644 results/irm/irm_gate_config.yml create mode 100644 results/irm/irm_gate_coverage.csv create mode 100644 results/irm/irm_gate_metadata.csv diff --git a/results/irm/irm_gate_config.yml b/results/irm/irm_gate_config.yml new file mode 100644 index 0000000..c1206fe --- /dev/null +++ b/results/irm/irm_gate_config.yml @@ -0,0 +1,63 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 500 + p: + - 10 + support_size: + - 5 + n_x: + - 1 +learner_definitions: + linear: &id001 + name: Linear + logit: &id002 + name: Logistic + rfr: &id003 + name: RF Regr. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + rfc: &id004 + name: RF Clas. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + lgbmr: &id005 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + lgbmc: &id006 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id001 + ml_m: *id004 + - ml_g: *id003 + ml_m: *id002 + - ml_g: *id005 + ml_m: *id006 + - ml_g: *id005 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id006 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/irm_gate_coverage.csv b/results/irm/irm_gate_coverage.csv new file mode 100644 index 0000000..30f5e71 --- /dev/null +++ b/results/irm/irm_gate_coverage.csv @@ -0,0 +1,15 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Clas.,0.9,0.9346666666666666,0.8364542894818474,0.18422904419208913,1.0,1.9724400757209293,1000 +LGBM Regr.,LGBM Clas.,0.95,0.9756666666666667,0.9966967608764793,0.18422904419208913,1.0,1.9651735219350885,1000 +LGBM Regr.,Logistic,0.9,0.9006666666666666,0.40046180389774216,0.09740590411838057,0.998,0.9434012510097336,1000 +LGBM Regr.,Logistic,0.95,0.9523333333333334,0.4771796711651552,0.09740590411838057,0.999,0.9366053028101997,1000 +Linear,LGBM Clas.,0.9,0.9226666666666666,0.8421150432332748,0.1918698759485984,1.0,1.9758022948217957,1000 +Linear,LGBM Clas.,0.95,0.9686666666666667,1.003441965006716,0.1918698759485984,1.0,1.9855094815511516,1000 +Linear,Logistic,0.9,0.9123333333333333,0.41818791810731526,0.09904291484604033,1.0,0.9842536479155779,1000 +Linear,Logistic,0.95,0.951,0.4983016390213454,0.09904291484604033,1.0,0.985193431203212,1000 +Linear,RF Clas.,0.9,0.9166666666666666,0.44173892078977606,0.10153218721035738,1.0,1.0388647556648747,1000 +Linear,RF Clas.,0.95,0.9593333333333334,0.5263643895914247,0.10153218721035738,1.0,1.0383096121913078,1000 +RF Regr.,Logistic,0.9,0.9026666666666666,0.4004544456677431,0.0967060927359184,1.0,0.9427533643825874,1000 +RF Regr.,Logistic,0.95,0.9486666666666667,0.4771709032933203,0.0967060927359184,0.999,0.9365571482746528,1000 +RF Regr.,RF Clas.,0.9,0.9026666666666666,0.4211186636375361,0.10090471591950194,1.0,0.9859887811490382,1000 +RF Regr.,RF Clas.,0.95,0.9506666666666667,0.5017938377148734,0.10090471591950194,1.0,0.9865161484854005,1000 diff --git a/results/irm/irm_gate_metadata.csv b/results/irm/irm_gate_metadata.csv new file mode 100644 index 0000000..b66fe1e --- /dev/null +++ b/results/irm/irm_gate_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,IRMGATECoverageSimulation,2025-06-05 13:55,79.23283307154973,3.12.3,scripts/irm/irm_gate_config.yml From 6c9197e642fc20441898343c9b3bd71d472a3f7d Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 13:56:18 +0000 Subject: [PATCH 14/35] Update results from script: scripts/irm/irm_cate.py --- results/irm/irm_cate_config.yml | 63 +++++++++++++++++++++++++++++++ results/irm/irm_cate_coverage.csv | 15 ++++++++ results/irm/irm_cate_metadata.csv | 2 + 3 files changed, 80 insertions(+) create mode 100644 results/irm/irm_cate_config.yml create mode 100644 results/irm/irm_cate_coverage.csv create mode 100644 results/irm/irm_cate_metadata.csv diff --git a/results/irm/irm_cate_config.yml b/results/irm/irm_cate_config.yml new file mode 100644 index 0000000..c1206fe --- /dev/null +++ b/results/irm/irm_cate_config.yml @@ -0,0 +1,63 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 500 + p: + - 10 + support_size: + - 5 + n_x: + - 1 +learner_definitions: + linear: &id001 + name: Linear + logit: &id002 + name: Logistic + rfr: &id003 + name: RF Regr. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + rfc: &id004 + name: RF Clas. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + lgbmr: &id005 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + lgbmc: &id006 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id001 + ml_m: *id004 + - ml_g: *id003 + ml_m: *id002 + - ml_g: *id005 + ml_m: *id006 + - ml_g: *id005 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id006 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/irm_cate_coverage.csv b/results/irm/irm_cate_coverage.csv new file mode 100644 index 0000000..14cd160 --- /dev/null +++ b/results/irm/irm_cate_coverage.csv @@ -0,0 +1,15 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Clas.,0.9,0.92506,1.0375110625997357,0.2365318243837263,1.0,2.596617666316375,1000 +LGBM Regr.,LGBM Clas.,0.95,0.96602,1.2362706826540963,0.2365318243837263,1.0,2.6133853117978934,1000 +LGBM Regr.,Logistic,0.9,0.89561,0.45975570518256736,0.11084262270269904,1.0,1.1581396191978983,1000 +LGBM Regr.,Logistic,0.95,0.94487,0.547832712333638,0.11084262270269904,1.0,1.1623001863971778,1000 +Linear,LGBM Clas.,0.9,0.90998,1.0435330991760245,0.2472414883409576,0.998,2.618492131430461,1000 +Linear,LGBM Clas.,0.95,0.95692,1.243446381822529,0.2472414883409576,0.999,2.626489143651275,1000 +Linear,Logistic,0.9,0.89899,0.47571933545085376,0.11363003649173503,1.0,1.1954349623588154,1000 +Linear,Logistic,0.95,0.9459299999999999,0.5668545510405528,0.11363003649173503,0.998,1.1979576781693033,1000 +Linear,RF Clas.,0.9,0.90489,0.5110714313286231,0.12032009678319744,1.0,1.2817085692658767,1000 +Linear,RF Clas.,0.95,0.9514600000000001,0.6089791714706715,0.12032009678319744,1.0,1.2861291474394618,1000 +RF Regr.,Logistic,0.9,0.89376,0.4592745091137625,0.11114309499883832,0.999,1.1543665958014406,1000 +RF Regr.,Logistic,0.95,0.94267,0.5472593318522952,0.11114309499883832,1.0,1.1532136852815742,1000 +RF Regr.,RF Clas.,0.9,0.89648,0.4916798706519477,0.11789806419426764,1.0,1.2340869862770245,1000 +RF Regr.,RF Clas.,0.95,0.9448,0.5858727017474369,0.11789806419426764,1.0,1.2368879574968341,1000 diff --git a/results/irm/irm_cate_metadata.csv b/results/irm/irm_cate_metadata.csv new file mode 100644 index 0000000..4bd0baf --- /dev/null +++ b/results/irm/irm_cate_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,IRMCATECoverageSimulation,2025-06-05 13:56,79.61121084690095,3.12.3,scripts/irm/irm_cate_config.yml From 07a6398edec8e08bfe35a5f43ebca6d13145b540 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 14:11:04 +0000 Subject: [PATCH 15/35] Update results from script: scripts/irm/cvar.py --- results/irm/cvar_Y0_coverage.csv | 9 ++++ results/irm/cvar_Y1_coverage.csv | 9 ++++ results/irm/cvar_config.yml | 65 ++++++++++++++++++++++++++++ results/irm/cvar_effect_coverage.csv | 9 ++++ results/irm/cvar_metadata.csv | 2 + 5 files changed, 94 insertions(+) create mode 100644 results/irm/cvar_Y0_coverage.csv create mode 100644 results/irm/cvar_Y1_coverage.csv create mode 100644 results/irm/cvar_config.yml create mode 100644 results/irm/cvar_effect_coverage.csv create mode 100644 results/irm/cvar_metadata.csv diff --git a/results/irm/cvar_Y0_coverage.csv b/results/irm/cvar_Y0_coverage.csv new file mode 100644 index 0000000..4c25ee7 --- /dev/null +++ b/results/irm/cvar_Y0_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +LGBM Regr.,LGBM Clas.,0.9,0.8564285714285714,0.5599803695184898,0.15691823669038846,200 +LGBM Regr.,LGBM Clas.,0.95,0.9242857142857143,0.6672577658717421,0.15691823669038846,200 +LGBM Regr.,Logistic,0.9,0.8,0.4488498613139841,0.13502164231417138,200 +LGBM Regr.,Logistic,0.95,0.8842857142857143,0.5348375978424744,0.13502164231417138,200 +Linear,LGBM Clas.,0.9,0.7778571428571429,0.5748146502742429,0.16876670012237052,200 +Linear,LGBM Clas.,0.95,0.8607142857142857,0.6849339016332675,0.16876670012237052,200 +Linear,Logistic,0.9,0.7521428571428571,0.4599365576395126,0.14286782087753735,200 +Linear,Logistic,0.95,0.832857142857143,0.5480482113277858,0.14286782087753735,200 diff --git a/results/irm/cvar_Y1_coverage.csv b/results/irm/cvar_Y1_coverage.csv new file mode 100644 index 0000000..8fddf73 --- /dev/null +++ b/results/irm/cvar_Y1_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +LGBM Regr.,LGBM Clas.,0.9,0.9014285714285714,0.19064863373047444,0.046573467818208994,200 +LGBM Regr.,LGBM Clas.,0.95,0.9535714285714286,0.22717185875440954,0.046573467818208994,200 +LGBM Regr.,Logistic,0.9,0.8921428571428571,0.18035991253115108,0.044703609418269445,200 +LGBM Regr.,Logistic,0.95,0.942857142857143,0.2149120912789158,0.044703609418269445,200 +Linear,LGBM Clas.,0.9,0.9064285714285714,0.21197545188306893,0.04818749120158227,200 +Linear,LGBM Clas.,0.95,0.957857142857143,0.25258432999137354,0.04818749120158227,200 +Linear,Logistic,0.9,0.9007142857142857,0.1965222149886573,0.04731821601748326,200 +Linear,Logistic,0.95,0.9457142857142857,0.23417066250063942,0.04731821601748326,200 diff --git a/results/irm/cvar_config.yml b/results/irm/cvar_config.yml new file mode 100644 index 0000000..5157d7e --- /dev/null +++ b/results/irm/cvar_config.yml @@ -0,0 +1,65 @@ +simulation_parameters: + repetitions: 200 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 5000 + dim_x: + - 5 +learner_definitions: + linear: &id001 + name: Linear + logit: &id002 + name: Logistic + lgbmr: &id003 + name: LGBM Regr. + params: + n_estimators: 200 + learning_rate: 0.05 + num_leaves: 15 + max_depth: 5 + min_child_samples: 10 + subsample: 0.9 + colsample_bytree: 0.9 + reg_alpha: 0.0 + reg_lambda: 0.1 + random_state: 42 + lgbmc: &id004 + name: LGBM Clas. + params: + n_estimators: 200 + learning_rate: 0.05 + num_leaves: 15 + max_depth: 5 + min_child_samples: 10 + subsample: 0.9 + colsample_bytree: 0.9 + reg_alpha: 0.0 + reg_lambda: 0.1 + random_state: 42 +dml_parameters: + tau_vec: + - - 0.2 + - 0.3 + - 0.4 + - 0.5 + - 0.6 + - 0.7 + - 0.8 + trimming_threshold: + - 0.01 + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id003 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id004 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/cvar_effect_coverage.csv b/results/irm/cvar_effect_coverage.csv new file mode 100644 index 0000000..b17f3f4 --- /dev/null +++ b/results/irm/cvar_effect_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Clas.,0.9,0.8357142857142857,0.572356196179233,0.16148728231793735,0.8,0.6933257488655766,200 +LGBM Regr.,LGBM Clas.,0.95,0.912142857142857,0.6820044728957118,0.16148728231793735,0.89,0.7996943722331994,200 +LGBM Regr.,Logistic,0.9,0.812142857142857,0.4603475364424448,0.13621838163720854,0.785,0.5540435315601864,200 +LGBM Regr.,Logistic,0.95,0.885,0.5485379227762442,0.13621838163720854,0.86,0.6395859491088156,200 +Linear,LGBM Clas.,0.9,0.7835714285714286,0.6002467096290228,0.17327648690690606,0.75,0.7089505880560413,200 +Linear,LGBM Clas.,0.95,0.8592857142857143,0.7152380694761148,0.17327648690690606,0.815,0.8220830080641385,200 +Linear,Logistic,0.9,0.7742857142857144,0.48428688399508923,0.14834465693639912,0.755,0.5678723651822354,200 +Linear,Logistic,0.95,0.85,0.5770634148004378,0.14834465693639912,0.82,0.6581721908143731,200 diff --git a/results/irm/cvar_metadata.csv b/results/irm/cvar_metadata.csv new file mode 100644 index 0000000..6db12ae --- /dev/null +++ b/results/irm/cvar_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,CVARCoverageSimulation,2025-06-05 14:11,94.70462875763575,3.12.3,scripts/irm/cvar_config.yml From b614f94da3d9951d47ce58eb68271ad87acac23d Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 14:29:18 +0000 Subject: [PATCH 16/35] Update results from script: scripts/irm/lpq.py --- results/irm/lpq_Y0_coverage.csv | 9 ++++++ results/irm/lpq_Y1_coverage.csv | 9 ++++++ results/irm/lpq_config.yml | 48 +++++++++++++++++++++++++++++ results/irm/lpq_effect_coverage.csv | 9 ++++++ results/irm/lpq_metadata.csv | 2 ++ 5 files changed, 77 insertions(+) create mode 100644 results/irm/lpq_Y0_coverage.csv create mode 100644 results/irm/lpq_Y1_coverage.csv create mode 100644 results/irm/lpq_config.yml create mode 100644 results/irm/lpq_effect_coverage.csv create mode 100644 results/irm/lpq_metadata.csv diff --git a/results/irm/lpq_Y0_coverage.csv b/results/irm/lpq_Y0_coverage.csv new file mode 100644 index 0000000..fa7c0a3 --- /dev/null +++ b/results/irm/lpq_Y0_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +LGBM Clas.,LGBM Clas.,0.9,0.938,1.182292560755257,0.23325691846991042,200 +LGBM Clas.,LGBM Clas.,0.95,0.9690000000000001,1.4087884783794826,0.23325691846991042,200 +LGBM Clas.,Logistic,0.9,0.9390000000000001,1.137086069984906,0.22432594978342146,200 +LGBM Clas.,Logistic,0.95,0.9690000000000001,1.3549216221890352,0.22432594978342146,200 +Logistic,LGBM Clas.,0.9,0.938,1.1527775627918269,0.22374215890669022,200 +Logistic,LGBM Clas.,0.95,0.9690000000000001,1.3736191891100717,0.22374215890669022,200 +Logistic,Logistic,0.9,0.943,1.111906655099774,0.2212690310065874,200 +Logistic,Logistic,0.95,0.9690000000000001,1.3249184987998035,0.2212690310065874,200 diff --git a/results/irm/lpq_Y1_coverage.csv b/results/irm/lpq_Y1_coverage.csv new file mode 100644 index 0000000..ba4fa63 --- /dev/null +++ b/results/irm/lpq_Y1_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +LGBM Clas.,LGBM Clas.,0.9,0.946,1.6296637808266206,0.31922961803439465,200 +LGBM Clas.,LGBM Clas.,0.95,0.965,1.9418641665090766,0.31922961803439465,200 +LGBM Clas.,Logistic,0.9,0.94,1.5840129690335032,0.3094090506782375,200 +LGBM Clas.,Logistic,0.95,0.97,1.8874678691647622,0.3094090506782375,200 +Logistic,LGBM Clas.,0.9,0.93,1.5829510778239204,0.31056212030323144,200 +Logistic,LGBM Clas.,0.95,0.965,1.8862025477451665,0.31056212030323144,200 +Logistic,Logistic,0.9,0.941,1.5420148214413294,0.2867899782486625,200 +Logistic,Logistic,0.95,0.97,1.8374239896673397,0.2867899782486625,200 diff --git a/results/irm/lpq_config.yml b/results/irm/lpq_config.yml new file mode 100644 index 0000000..85abd3f --- /dev/null +++ b/results/irm/lpq_config.yml @@ -0,0 +1,48 @@ +simulation_parameters: + repetitions: 200 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 5000 + dim_x: + - 5 +learner_definitions: + logit: &id001 + name: Logistic + lgbmc: &id002 + name: LGBM Clas. + params: + n_estimators: 200 + learning_rate: 0.05 + num_leaves: 15 + max_depth: 5 + min_child_samples: 10 + subsample: 0.9 + colsample_bytree: 0.9 + reg_alpha: 0.0 + reg_lambda: 0.1 + random_state: 42 +dml_parameters: + tau_vec: + - - 0.3 + - 0.4 + - 0.5 + - 0.6 + - 0.7 + trimming_threshold: + - 0.01 + learners: + - ml_g: *id001 + ml_m: *id001 + - ml_g: *id002 + ml_m: *id002 + - ml_g: *id002 + ml_m: *id001 + - ml_g: *id001 + ml_m: *id002 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/lpq_effect_coverage.csv b/results/irm/lpq_effect_coverage.csv new file mode 100644 index 0000000..2e1488a --- /dev/null +++ b/results/irm/lpq_effect_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Clas.,LGBM Clas.,0.9,0.882,1.6179566817511408,0.38892858697985444,0.85,2.134204151727998,200 +LGBM Clas.,LGBM Clas.,0.95,0.9329999999999999,1.9279142975508834,0.38892858697985444,0.93,2.4114959415166863,200 +LGBM Clas.,Logistic,0.9,0.907,1.57231832862624,0.36746763608272737,0.865,2.0750759388885753,200 +LGBM Clas.,Logistic,0.95,0.9520000000000001,1.873532845625395,0.36746763608272737,0.935,2.3491301230114727,200 +Logistic,LGBM Clas.,0.9,0.892,1.5819115069451675,0.37355342356664595,0.835,2.0754682406547134,200 +Logistic,LGBM Clas.,0.95,0.943,1.8849638226401801,0.37355342356664595,0.93,2.3505474278366396,200 +Logistic,Logistic,0.9,0.895,1.5376032362171548,0.3646953928818029,0.86,2.0170674200183667,200 +Logistic,Logistic,0.95,0.941,1.8321672616445936,0.3646953928818029,0.91,2.2852783686545495,200 diff --git a/results/irm/lpq_metadata.csv b/results/irm/lpq_metadata.csv new file mode 100644 index 0000000..47bab20 --- /dev/null +++ b/results/irm/lpq_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,LPQCoverageSimulation,2025-06-05 14:29,112.94002043803533,3.12.3,scripts/irm/lpq_config.yml From 4495d6017cfce946d7913fb7d03a883aad1d4ae4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 14:33:35 +0000 Subject: [PATCH 17/35] Update results from script: scripts/irm/pq.py --- results/irm/pq_Y0_coverage.csv | 9 ++++++ results/irm/pq_Y1_coverage.csv | 9 ++++++ results/irm/pq_config.yml | 50 ++++++++++++++++++++++++++++++ results/irm/pq_effect_coverage.csv | 9 ++++++ results/irm/pq_metadata.csv | 2 ++ 5 files changed, 79 insertions(+) create mode 100644 results/irm/pq_Y0_coverage.csv create mode 100644 results/irm/pq_Y1_coverage.csv create mode 100644 results/irm/pq_config.yml create mode 100644 results/irm/pq_effect_coverage.csv create mode 100644 results/irm/pq_metadata.csv diff --git a/results/irm/pq_Y0_coverage.csv b/results/irm/pq_Y0_coverage.csv new file mode 100644 index 0000000..ff0b3ac --- /dev/null +++ b/results/irm/pq_Y0_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +LGBM Clas.,LGBM Clas.,0.9,0.8835714285714286,0.5723701824339644,0.14549176231056815,200 +LGBM Clas.,LGBM Clas.,0.95,0.9378571428571427,0.6820211385461397,0.14549176231056815,200 +LGBM Clas.,Logistic,0.9,0.84,0.4044754891818755,0.1130627263596336,200 +LGBM Clas.,Logistic,0.95,0.9078571428571429,0.4819622721658046,0.1130627263596336,200 +Logistic,LGBM Clas.,0.9,0.8878571428571429,0.5701038626834825,0.14084059793538922,200 +Logistic,LGBM Clas.,0.95,0.9328571428571429,0.6793206520009456,0.14084059793538922,200 +Logistic,Logistic,0.9,0.8521428571428571,0.40381464298983716,0.10742954627392248,200 +Logistic,Logistic,0.95,0.9207142857142857,0.4811748253592969,0.10742954627392248,200 diff --git a/results/irm/pq_Y1_coverage.csv b/results/irm/pq_Y1_coverage.csv new file mode 100644 index 0000000..3cb5336 --- /dev/null +++ b/results/irm/pq_Y1_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +LGBM Clas.,LGBM Clas.,0.9,0.9114285714285714,0.25322340912372693,0.05863607300930684,200 +LGBM Clas.,LGBM Clas.,0.95,0.9514285714285714,0.3017343025499488,0.05863607300930684,200 +LGBM Clas.,Logistic,0.9,0.9028571428571429,0.23575937348166215,0.057047735482004806,200 +LGBM Clas.,Logistic,0.95,0.9507142857142857,0.2809246205683309,0.057047735482004806,200 +Logistic,LGBM Clas.,0.9,0.9178571428571429,0.2536257290831553,0.0584307589677001,200 +Logistic,LGBM Clas.,0.95,0.9607142857142857,0.3022136963499933,0.0584307589677001,200 +Logistic,Logistic,0.9,0.8971428571428571,0.2359931637120258,0.05685852446847999,200 +Logistic,Logistic,0.95,0.9407142857142857,0.28120319881015254,0.05685852446847999,200 diff --git a/results/irm/pq_config.yml b/results/irm/pq_config.yml new file mode 100644 index 0000000..e106878 --- /dev/null +++ b/results/irm/pq_config.yml @@ -0,0 +1,50 @@ +simulation_parameters: + repetitions: 200 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 5000 + dim_x: + - 5 +learner_definitions: + logit: &id001 + name: Logistic + lgbmc: &id002 + name: LGBM Clas. + params: + n_estimators: 200 + learning_rate: 0.05 + num_leaves: 15 + max_depth: 5 + min_child_samples: 10 + subsample: 0.9 + colsample_bytree: 0.9 + reg_alpha: 0.0 + reg_lambda: 0.1 + random_state: 42 +dml_parameters: + tau_vec: + - - 0.2 + - 0.3 + - 0.4 + - 0.5 + - 0.6 + - 0.7 + - 0.8 + trimming_threshold: + - 0.01 + learners: + - ml_g: *id001 + ml_m: *id001 + - ml_g: *id002 + ml_m: *id002 + - ml_g: *id002 + ml_m: *id001 + - ml_g: *id001 + ml_m: *id002 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/pq_effect_coverage.csv b/results/irm/pq_effect_coverage.csv new file mode 100644 index 0000000..710de75 --- /dev/null +++ b/results/irm/pq_effect_coverage.csv @@ -0,0 +1,9 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Clas.,LGBM Clas.,0.9,0.8857142857142857,0.6119091533344946,0.15466123618513827,0.83,0.87431953800783,200 +LGBM Clas.,LGBM Clas.,0.95,0.9392857142857143,0.7291347282790106,0.15466123618513827,0.895,0.9724716396352252,200 +LGBM Clas.,Logistic,0.9,0.8414285714285714,0.44675957755153617,0.12877194950989979,0.725,0.6418465368141598,200 +LGBM Clas.,Logistic,0.95,0.9028571428571429,0.5323468711147352,0.12877194950989979,0.835,0.7132416415357535,200 +Logistic,LGBM Clas.,0.9,0.89,0.6129040235808955,0.15303926661598033,0.83,0.8645516927626398,200 +Logistic,LGBM Clas.,0.95,0.94,0.7303201892952897,0.15303926661598033,0.88,0.9644960349693998,200 +Logistic,Logistic,0.9,0.8592857142857143,0.45040188642196527,0.12179153217631539,0.785,0.6382066580620731,200 +Logistic,Logistic,0.95,0.925,0.536686949824257,0.12179153217631539,0.865,0.7120923308029136,200 diff --git a/results/irm/pq_metadata.csv b/results/irm/pq_metadata.csv new file mode 100644 index 0000000..bf12575 --- /dev/null +++ b/results/irm/pq_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,PQCoverageSimulation,2025-06-05 14:33,117.12256911595662,3.12.3,scripts/irm/pq_config.yml From 8f56dbfc24cad1317e26f27a6e83dcdd3c7159f5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 14:42:27 +0000 Subject: [PATCH 18/35] Update results from script: scripts/irm/irm_ate.py --- results/irm/irm_ate_config.yml | 61 ++++++++++++++++++++++++++++++++ results/irm/irm_ate_coverage.csv | 15 ++++++++ results/irm/irm_ate_metadata.csv | 2 ++ 3 files changed, 78 insertions(+) create mode 100644 results/irm/irm_ate_config.yml create mode 100644 results/irm/irm_ate_coverage.csv create mode 100644 results/irm/irm_ate_metadata.csv diff --git a/results/irm/irm_ate_config.yml b/results/irm/irm_ate_config.yml new file mode 100644 index 0000000..d19a50a --- /dev/null +++ b/results/irm/irm_ate_config.yml @@ -0,0 +1,61 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 0.5 + n_obs: + - 500 + dim_x: + - 20 +learner_definitions: + lasso: &id001 + name: LassoCV + logit: &id002 + name: Logistic + rfr: &id003 + name: RF Regr. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + rfc: &id004 + name: RF Clas. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + lgbmr: &id005 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + lgbmc: &id006 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id001 + ml_m: *id004 + - ml_g: *id003 + ml_m: *id002 + - ml_g: *id005 + ml_m: *id006 + - ml_g: *id005 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id006 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/irm_ate_coverage.csv b/results/irm/irm_ate_coverage.csv new file mode 100644 index 0000000..46ebf4c --- /dev/null +++ b/results/irm/irm_ate_coverage.csv @@ -0,0 +1,15 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +LGBM Regr.,LGBM Clas.,0.9,0.928,1.1983115160037485,0.2840471834602478,1000 +LGBM Regr.,LGBM Clas.,0.95,0.98,1.4278762408664047,0.2840471834602478,1000 +LGBM Regr.,Logistic,0.9,0.928,0.771069826261061,0.1773323727171827,1000 +LGBM Regr.,Logistic,0.95,0.97,0.9187863675372636,0.1773323727171827,1000 +LassoCV,LGBM Clas.,0.9,0.943,1.0988039710069317,0.25576093311987325,1000 +LassoCV,LGBM Clas.,0.95,0.979,1.3093056877253173,0.25576093311987325,1000 +LassoCV,Logistic,0.9,0.927,0.6575776853999991,0.1495642781049785,1000 +LassoCV,Logistic,0.95,0.968,0.7835521406302206,0.1495642781049785,1000 +LassoCV,RF Clas.,0.9,0.926,0.5837441390355065,0.13723792736069168,1000 +LassoCV,RF Clas.,0.95,0.962,0.6955740437624297,0.13723792736069168,1000 +RF Regr.,Logistic,0.9,0.918,0.743232966143666,0.1705153153049291,1000 +RF Regr.,Logistic,0.95,0.968,0.8856167028456445,0.1705153153049291,1000 +RF Regr.,RF Clas.,0.9,0.905,0.6164614614548363,0.14356423385388378,1000 +RF Regr.,RF Clas.,0.95,0.951,0.7345591379749272,0.14356423385388378,1000 diff --git a/results/irm/irm_ate_metadata.csv b/results/irm/irm_ate_metadata.csv new file mode 100644 index 0000000..03c5799 --- /dev/null +++ b/results/irm/irm_ate_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,IRMATECoverageSimulation,2025-06-05 14:42,125.66061746279398,3.12.3,scripts/irm/irm_ate_config.yml From 4900fb023186978f57cf71daa54aac1604210b40 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 14:42:47 +0000 Subject: [PATCH 19/35] Update results from script: scripts/irm/irm_atte.py --- results/irm/irm_atte_config.yml | 61 +++++++++++++++++++++++++++++++ results/irm/irm_atte_coverage.csv | 15 ++++++++ results/irm/irm_atte_metadata.csv | 2 + 3 files changed, 78 insertions(+) create mode 100644 results/irm/irm_atte_config.yml create mode 100644 results/irm/irm_atte_coverage.csv create mode 100644 results/irm/irm_atte_metadata.csv diff --git a/results/irm/irm_atte_config.yml b/results/irm/irm_atte_config.yml new file mode 100644 index 0000000..2d3c69a --- /dev/null +++ b/results/irm/irm_atte_config.yml @@ -0,0 +1,61 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 0.5 + n_obs: + - 500 + dim_x: + - 20 +learner_definitions: + lasso: &id001 + name: LassoCV + logit: &id002 + name: Logistic + rfr: &id003 + name: RF Regr. + params: + n_estimators: 200 + max_features: 20 + max_depth: 20 + min_samples_leaf: 2 + rfc: &id004 + name: RF Clas. + params: + n_estimators: 200 + max_features: 20 + max_depth: 20 + min_samples_leaf: 20 + lgbmr: &id005 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + lgbmc: &id006 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id001 + ml_m: *id004 + - ml_g: *id003 + ml_m: *id002 + - ml_g: *id005 + ml_m: *id006 + - ml_g: *id005 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id006 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/irm/irm_atte_coverage.csv b/results/irm/irm_atte_coverage.csv new file mode 100644 index 0000000..5b68231 --- /dev/null +++ b/results/irm/irm_atte_coverage.csv @@ -0,0 +1,15 @@ +Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +LGBM Regr.,LGBM Clas.,0.9,0.927,1.5064451215730035,0.34563899658615477,1000 +LGBM Regr.,LGBM Clas.,0.95,0.974,1.7950400780897324,0.34563899658615477,1000 +LGBM Regr.,Logistic,0.9,0.926,0.853133738564191,0.2115612747662681,1000 +LGBM Regr.,Logistic,0.95,0.969,1.016571550309234,0.2115612747662681,1000 +LassoCV,LGBM Clas.,0.9,0.912,1.3899632828213013,0.3405205709305417,1000 +LassoCV,LGBM Clas.,0.95,0.977,1.6562434064190357,0.3405205709305417,1000 +LassoCV,Logistic,0.9,0.918,0.7956786618509171,0.19501674862485438,1000 +LassoCV,Logistic,0.95,0.962,0.9481096037616187,0.19501674862485438,1000 +LassoCV,RF Clas.,0.9,0.895,0.5793446092118805,0.1467183519931486,1000 +LassoCV,RF Clas.,0.95,0.945,0.6903316806354453,0.1467183519931486,1000 +RF Regr.,Logistic,0.9,0.915,0.8295563252373992,0.200468421193765,1000 +RF Regr.,Logistic,0.95,0.963,0.9884773295153919,0.200468421193765,1000 +RF Regr.,RF Clas.,0.9,0.881,0.5967830827952515,0.15670311644434254,1000 +RF Regr.,RF Clas.,0.95,0.939,0.7111109035454538,0.15670311644434254,1000 diff --git a/results/irm/irm_atte_metadata.csv b/results/irm/irm_atte_metadata.csv new file mode 100644 index 0000000..876cac0 --- /dev/null +++ b/results/irm/irm_atte_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,IRMATTECoverageSimulation,2025-06-05 14:42,126.08159985939662,3.12.3,scripts/irm/irm_atte_config.yml From ce79b56f549269978f9d2d50e36e424f092db2c5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 15:41:49 +0000 Subject: [PATCH 20/35] Update results from script: scripts/plm/plr_cate.py --- results/plm/plr_cate_config.yml | 52 +++++++++++++++++++++++++++++++ results/plm/plr_cate_coverage.csv | 29 +++++++++++++++++ results/plm/plr_cate_metadata.csv | 2 ++ 3 files changed, 83 insertions(+) create mode 100644 results/plm/plr_cate_config.yml create mode 100644 results/plm/plr_cate_coverage.csv create mode 100644 results/plm/plr_cate_metadata.csv diff --git a/results/plm/plr_cate_config.yml b/results/plm/plr_cate_config.yml new file mode 100644 index 0000000..20ce744 --- /dev/null +++ b/results/plm/plr_cate_config.yml @@ -0,0 +1,52 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 500 + p: + - 10 + support_size: + - 5 + n_x: + - 1 +learner_definitions: + lasso: &id001 + name: LassoCV + rf: &id002 + name: RF Regr. + params: + n_estimators: 200 + max_features: 10 + max_depth: 5 + min_samples_leaf: 2 + lgbm: &id003 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id001 + - ml_g: *id002 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id002 + ml_m: *id001 + - ml_g: *id003 + ml_m: *id003 + - ml_g: *id003 + ml_m: *id001 + - ml_g: *id001 + ml_m: *id003 + score: + - partialling out + - IV-type +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/plm/plr_cate_coverage.csv b/results/plm/plr_cate_coverage.csv new file mode 100644 index 0000000..c95af2f --- /dev/null +++ b/results/plm/plr_cate_coverage.csv @@ -0,0 +1,29 @@ +Learner g,Learner m,Score,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Regr.,IV-type,0.9,0.81092,0.34748895671048663,0.10460813293283802,0.981,0.8749566905579788,1000 +LGBM Regr.,LGBM Regr.,IV-type,0.95,0.87944,0.4140586305179147,0.10460813293283802,0.976,0.8752701592984206,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.9,0.74924,0.45498586012385417,0.15429490050948075,0.974,1.1431727401739051,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.95,0.83409,0.5421490913878395,0.15429490050948075,0.979,1.1448962771888775,1000 +LGBM Regr.,LassoCV,IV-type,0.9,0.88,0.36554268929918754,0.09244785218836214,0.998,0.9180115191244616,1000 +LGBM Regr.,LassoCV,IV-type,0.95,0.9358200000000001,0.43557097975105097,0.09244785218836214,1.0,0.922278240560451,1000 +LGBM Regr.,LassoCV,partialling out,0.9,0.8425499999999999,0.6463383782822439,0.17877605236642957,0.993,1.6252735422347984,1000 +LGBM Regr.,LassoCV,partialling out,0.95,0.90764,0.7701596801698865,0.17877605236642957,0.99,1.6258971712277042,1000 +LassoCV,LGBM Regr.,IV-type,0.9,0.77804,0.356611018165839,0.11531004698375871,0.98,0.8990196528636599,1000 +LassoCV,LGBM Regr.,IV-type,0.95,0.85509,0.42492823716515665,0.11531004698375871,0.973,0.897125005016581,1000 +LassoCV,LGBM Regr.,partialling out,0.9,0.11495,0.5626129349999418,0.5271527276193878,0.232,1.4094053511501499,1000 +LassoCV,LGBM Regr.,partialling out,0.95,0.17364000000000002,0.6703946611225079,0.5271527276193878,0.245,1.415448802418257,1000 +LassoCV,LassoCV,IV-type,0.9,0.8912100000000001,0.36244677298144845,0.08838657089890865,0.999,0.913576206576413,1000 +LassoCV,LassoCV,IV-type,0.95,0.94274,0.43188196792501726,0.08838657089890865,0.998,0.912240620292473,1000 +LassoCV,LassoCV,partialling out,0.9,0.88858,0.3775763713434813,0.09330414285601538,0.997,0.9491441464568747,1000 +LassoCV,LassoCV,partialling out,0.95,0.94064,0.4499099963187045,0.09330414285601538,0.997,0.9487107676660669,1000 +LassoCV,RF Regr.,IV-type,0.9,0.89254,0.3599305351929274,0.08850837997692952,1.0,0.9044665568509359,1000 +LassoCV,RF Regr.,IV-type,0.95,0.94188,0.4288836856698476,0.08850837997692952,0.999,0.9044733402499352,1000 +LassoCV,RF Regr.,partialling out,0.9,0.77416,0.43217879712767876,0.1405937817588716,0.981,1.090157100332438,1000 +LassoCV,RF Regr.,partialling out,0.95,0.85737,0.5149727996295947,0.1405937817588716,0.978,1.087530105804212,1000 +RF Regr.,LassoCV,IV-type,0.9,0.8807699999999999,0.3475079221052468,0.08785236408566467,0.996,0.8749646348632354,1000 +RF Regr.,LassoCV,IV-type,0.95,0.93665,0.4140812291796275,0.08785236408566467,0.998,0.8759773436970753,1000 +RF Regr.,LassoCV,partialling out,0.9,0.8651,0.44409447433118815,0.11793812231956644,0.995,1.113461220582107,1000 +RF Regr.,LassoCV,partialling out,0.95,0.9245099999999999,0.5291712047567231,0.11793812231956644,0.995,1.1193035806604223,1000 +RF Regr.,RF Regr.,IV-type,0.9,0.8769600000000001,0.3430202960561061,0.08782119424850063,0.997,0.8607856773588846,1000 +RF Regr.,RF Regr.,IV-type,0.95,0.9322,0.4087338929253366,0.08782119424850063,0.998,0.8634371330487173,1000 +RF Regr.,RF Regr.,partialling out,0.9,0.88322,0.3831582275710224,0.09673737638816682,0.996,0.9645065365685301,1000 +RF Regr.,RF Regr.,partialling out,0.95,0.9354,0.45656118825068054,0.09673737638816682,0.998,0.9640177197875869,1000 diff --git a/results/plm/plr_cate_metadata.csv b/results/plm/plr_cate_metadata.csv new file mode 100644 index 0000000..be41517 --- /dev/null +++ b/results/plm/plr_cate_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,PLRCATECoverageSimulation,2025-06-05 15:41,185.28740434646608,3.12.3,scripts/plm/plr_cate_config.yml From e7d2beec474db257a6a2e1e71a4891fa52af12c9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 15:40:49 +0000 Subject: [PATCH 21/35] Update results from script: scripts/plm/plr_gate.py --- results/plm/plr_gate_config.yml | 52 +++++++++++++++++++++++++++++++ results/plm/plr_gate_coverage.csv | 29 +++++++++++++++++ results/plm/plr_gate_metadata.csv | 2 ++ 3 files changed, 83 insertions(+) create mode 100644 results/plm/plr_gate_config.yml create mode 100644 results/plm/plr_gate_coverage.csv create mode 100644 results/plm/plr_gate_metadata.csv diff --git a/results/plm/plr_gate_config.yml b/results/plm/plr_gate_config.yml new file mode 100644 index 0000000..20ce744 --- /dev/null +++ b/results/plm/plr_gate_config.yml @@ -0,0 +1,52 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 500 + p: + - 10 + support_size: + - 5 + n_x: + - 1 +learner_definitions: + lasso: &id001 + name: LassoCV + rf: &id002 + name: RF Regr. + params: + n_estimators: 200 + max_features: 10 + max_depth: 5 + min_samples_leaf: 2 + lgbm: &id003 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id001 + - ml_g: *id002 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id002 + ml_m: *id001 + - ml_g: *id003 + ml_m: *id003 + - ml_g: *id003 + ml_m: *id001 + - ml_g: *id001 + ml_m: *id003 + score: + - partialling out + - IV-type +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/plm/plr_gate_coverage.csv b/results/plm/plr_gate_coverage.csv new file mode 100644 index 0000000..df0c44b --- /dev/null +++ b/results/plm/plr_gate_coverage.csv @@ -0,0 +1,29 @@ +Learner g,Learner m,Score,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Regr.,IV-type,0.9,0.8023333333333333,0.340614107591605,0.10669475143783842,0.987,0.7987268267983838,1000 +LGBM Regr.,LGBM Regr.,IV-type,0.95,0.8706666666666666,0.40586674252777905,0.10669475143783842,0.988,0.7989724883932184,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.9,0.725,0.41206326461441856,0.14116055364172336,0.982,0.9688824136316435,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.95,0.816,0.4910036642549748,0.14116055364172336,0.978,0.963097819307053,1000 +LGBM Regr.,LassoCV,IV-type,0.9,0.884,0.3584385924494663,0.09037899338673383,0.999,0.8422006093284867,1000 +LGBM Regr.,LassoCV,IV-type,0.95,0.94,0.4271059262411261,0.09037899338673383,0.999,0.8410103142728071,1000 +LGBM Regr.,LassoCV,partialling out,0.9,0.846,0.5546020564560807,0.15058630900692344,0.995,1.3030240395599795,1000 +LGBM Regr.,LassoCV,partialling out,0.95,0.9063333333333333,0.6608491105803649,0.15058630900692344,0.998,1.3052579182429735,1000 +LassoCV,LGBM Regr.,IV-type,0.9,0.7443333333333334,0.3533683685919372,0.12308426099091321,0.984,0.8290587683546056,1000 +LassoCV,LGBM Regr.,IV-type,0.95,0.828,0.4210643818802881,0.12308426099091321,0.986,0.8297177472370508,1000 +LassoCV,LGBM Regr.,partialling out,0.9,0.12766666666666665,0.4805861056396863,0.48492787025671996,0.166,1.1277373422329622,1000 +LassoCV,LGBM Regr.,partialling out,0.95,0.18,0.5726536654023722,0.48492787025671996,0.163,1.1286363645960298,1000 +LassoCV,LassoCV,IV-type,0.9,0.908,0.35675825943241785,0.08468553801157398,0.998,0.8347654007058711,1000 +LassoCV,LassoCV,IV-type,0.95,0.9536666666666667,0.42510368595573833,0.08468553801157398,1.0,0.8406082357109622,1000 +LassoCV,LassoCV,partialling out,0.9,0.897,0.3685816198393858,0.08926222502259333,0.999,0.8634981533000406,1000 +LassoCV,LassoCV,partialling out,0.95,0.9493333333333334,0.43919208883499217,0.08926222502259333,0.998,0.8629765470291304,1000 +LassoCV,RF Regr.,IV-type,0.9,0.9046666666666666,0.35535128701248625,0.08564429580896525,0.998,0.8309506852608253,1000 +LassoCV,RF Regr.,IV-type,0.95,0.9526666666666667,0.423427174912371,0.08564429580896525,0.997,0.8339321583590988,1000 +LassoCV,RF Regr.,partialling out,0.9,0.7333333333333334,0.4028059779258174,0.13583622582602936,0.98,0.9463483076426671,1000 +LassoCV,RF Regr.,partialling out,0.95,0.8286666666666667,0.4799729268039788,0.13583622582602936,0.988,0.9464873702847479,1000 +RF Regr.,LassoCV,IV-type,0.9,0.8856666666666666,0.34695511339781726,0.0872292667085423,0.999,0.8159480135750993,1000 +RF Regr.,LassoCV,IV-type,0.95,0.936,0.41342251697621396,0.0872292667085423,0.998,0.8161295955897384,1000 +RF Regr.,LassoCV,partialling out,0.9,0.86,0.4138648906001244,0.108596480808861,0.999,0.97698823470601,1000 +RF Regr.,LassoCV,partialling out,0.95,0.9236666666666666,0.4931504340269086,0.108596480808861,0.999,0.9729165052534585,1000 +RF Regr.,RF Regr.,IV-type,0.9,0.8836666666666666,0.34359869293370354,0.08702820649024633,1.0,0.8068440871102968,1000 +RF Regr.,RF Regr.,IV-type,0.95,0.9393333333333334,0.4094230953141002,0.08702820649024633,0.998,0.8071122999062327,1000 +RF Regr.,RF Regr.,partialling out,0.9,0.8806666666666666,0.3685754520029418,0.09347367483176777,1.0,0.8690762276450229,1000 +RF Regr.,RF Regr.,partialling out,0.95,0.9383333333333334,0.4391847394045658,0.09347367483176777,1.0,0.8688884478341746,1000 diff --git a/results/plm/plr_gate_metadata.csv b/results/plm/plr_gate_metadata.csv new file mode 100644 index 0000000..c820b6d --- /dev/null +++ b/results/plm/plr_gate_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,PLRGATECoverageSimulation,2025-06-05 15:40,184.26536533435186,3.12.3,scripts/plm/plr_gate_config.yml From 65b80f00591bb90684adbed3614b2309f6e0c106 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 15:50:42 +0000 Subject: [PATCH 22/35] Update results from script: scripts/plm/plr_ate.py --- results/plm/plr_ate_config.yml | 50 ++++++++++++++++++++++++++++++++ results/plm/plr_ate_coverage.csv | 29 ++++++++++++++++++ results/plm/plr_ate_metadata.csv | 2 ++ 3 files changed, 81 insertions(+) create mode 100644 results/plm/plr_ate_config.yml create mode 100644 results/plm/plr_ate_coverage.csv create mode 100644 results/plm/plr_ate_metadata.csv diff --git a/results/plm/plr_ate_config.yml b/results/plm/plr_ate_config.yml new file mode 100644 index 0000000..d504ba6 --- /dev/null +++ b/results/plm/plr_ate_config.yml @@ -0,0 +1,50 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 0.5 + n_obs: + - 500 + dim_x: + - 20 +learner_definitions: + lasso: &id001 + name: LassoCV + rf: &id002 + name: RF Regr. + params: + n_estimators: 200 + max_features: 10 + max_depth: 5 + min_samples_leaf: 20 + lgbm: &id003 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id001 + - ml_g: *id002 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id002 + ml_m: *id001 + - ml_g: *id003 + ml_m: *id003 + - ml_g: *id003 + ml_m: *id001 + - ml_g: *id001 + ml_m: *id003 + score: + - partialling out + - IV-type +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/plm/plr_ate_coverage.csv b/results/plm/plr_ate_coverage.csv new file mode 100644 index 0000000..751fcac --- /dev/null +++ b/results/plm/plr_ate_coverage.csv @@ -0,0 +1,29 @@ +Learner g,Learner m,Score,level,Coverage,CI Length,Bias,repetition +LGBM Regr.,LGBM Regr.,IV-type,0.9,0.885,0.15983740265821775,0.041136505215158464,1000 +LGBM Regr.,LGBM Regr.,IV-type,0.95,0.935,0.19045801246956545,0.041136505215158464,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.9,0.824,0.14652019534658833,0.04246199234429185,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.95,0.893,0.17458958121357435,0.04246199234429185,1000 +LGBM Regr.,LassoCV,IV-type,0.9,0.883,0.14837154235030955,0.037547727902696455,1000 +LGBM Regr.,LassoCV,IV-type,0.95,0.937,0.17679559723270477,0.037547727902696455,1000 +LGBM Regr.,LassoCV,partialling out,0.9,0.887,0.15933257799041745,0.04021756501464064,1000 +LGBM Regr.,LassoCV,partialling out,0.95,0.941,0.1898564767759428,0.04021756501464064,1000 +LassoCV,LGBM Regr.,IV-type,0.9,0.874,0.1504138026201527,0.0384034628252421,1000 +LassoCV,LGBM Regr.,IV-type,0.95,0.945,0.17922910043953308,0.0384034628252421,1000 +LassoCV,LGBM Regr.,partialling out,0.9,0.521,0.13901722228563204,0.06873936210074709,1000 +LassoCV,LGBM Regr.,partialling out,0.95,0.64,0.16564923738267465,0.06873936210074709,1000 +LassoCV,LassoCV,IV-type,0.9,0.876,0.13984950388376818,0.03654175128573881,1000 +LassoCV,LassoCV,IV-type,0.95,0.934,0.16664096207514204,0.03654175128573881,1000 +LassoCV,LassoCV,partialling out,0.9,0.9,0.1468437970720089,0.03588220373374918,1000 +LassoCV,LassoCV,partialling out,0.95,0.946,0.17497517645242536,0.03588220373374918,1000 +LassoCV,RF Regr.,IV-type,0.9,0.837,0.13013644240026234,0.036636608615855826,1000 +LassoCV,RF Regr.,IV-type,0.95,0.907,0.1550671354589842,0.036636608615855826,1000 +LassoCV,RF Regr.,partialling out,0.9,0.773,0.14296223702800953,0.046042984436838075,1000 +LassoCV,RF Regr.,partialling out,0.95,0.859,0.17035001238590083,0.046042984436838075,1000 +RF Regr.,LassoCV,IV-type,0.9,0.884,0.141016616428934,0.03611493633659719,1000 +RF Regr.,LassoCV,IV-type,0.95,0.929,0.168031662449296,0.03611493633659719,1000 +RF Regr.,LassoCV,partialling out,0.9,0.885,0.15062723475769935,0.037683080056869614,1000 +RF Regr.,LassoCV,partialling out,0.95,0.943,0.1794834205175513,0.037683080056869614,1000 +RF Regr.,RF Regr.,IV-type,0.9,0.841,0.1314513341669066,0.037780418069974564,1000 +RF Regr.,RF Regr.,IV-type,0.95,0.9,0.15663392563651957,0.037780418069974564,1000 +RF Regr.,RF Regr.,partialling out,0.9,0.876,0.14238380316163346,0.0364464310437898,1000 +RF Regr.,RF Regr.,partialling out,0.95,0.929,0.16966076592228904,0.0364464310437898,1000 diff --git a/results/plm/plr_ate_metadata.csv b/results/plm/plr_ate_metadata.csv new file mode 100644 index 0000000..c6aa9c1 --- /dev/null +++ b/results/plm/plr_ate_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,PLRATECoverageSimulation,2025-06-05 15:50,194.21264092922212,3.12.3,scripts/plm/plr_ate_config.yml From dd729b666dad7a356f0a2cf310ab88da81f08f08 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 16:23:49 +0000 Subject: [PATCH 23/35] Update results from script: scripts/plm/plr_ate_sensitivity.py --- results/plm/plr_ate_sensitivity_config.yml | 49 ++++++++++++++++++++ results/plm/plr_ate_sensitivity_coverage.csv | 29 ++++++++++++ results/plm/plr_ate_sensitivity_metadata.csv | 2 + 3 files changed, 80 insertions(+) create mode 100644 results/plm/plr_ate_sensitivity_config.yml create mode 100644 results/plm/plr_ate_sensitivity_coverage.csv create mode 100644 results/plm/plr_ate_sensitivity_metadata.csv diff --git a/results/plm/plr_ate_sensitivity_config.yml b/results/plm/plr_ate_sensitivity_config.yml new file mode 100644 index 0000000..f575860 --- /dev/null +++ b/results/plm/plr_ate_sensitivity_config.yml @@ -0,0 +1,49 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 0.5 + n_obs: + - 1000 +learner_definitions: + lasso: &id001 + name: LassoCV + rf: &id002 + name: RF Regr. + params: + n_estimators: 200 + max_features: 10 + max_depth: 5 + min_samples_leaf: 2 + lgbm: &id003 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.05 + min_child_samples: 5 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id001 + - ml_g: *id002 + ml_m: *id002 + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id002 + ml_m: *id001 + - ml_g: *id003 + ml_m: *id003 + - ml_g: *id003 + ml_m: *id001 + - ml_g: *id001 + ml_m: *id003 + score: + - partialling out + - IV-type +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/plm/plr_ate_sensitivity_coverage.csv b/results/plm/plr_ate_sensitivity_coverage.csv new file mode 100644 index 0000000..cd2031e --- /dev/null +++ b/results/plm/plr_ate_sensitivity_coverage.csv @@ -0,0 +1,29 @@ +Learner g,Learner m,Score,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition +LGBM Regr.,LGBM Regr.,IV-type,0.9,0.388,1.4093565939481743,0.7570214454341684,1.0,0.992,0.10345660369135473,0.03229286770316017,1.4538012313284263,0.2824074286090704,1000 +LGBM Regr.,LGBM Regr.,IV-type,0.95,0.577,1.6793519619323294,0.7570214454341684,1.0,1.0,0.10345660369135473,0.018230572464796476,1.4538012313284263,0.2824074286090704,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.9,0.205,1.10817064601718,0.74843530520864,1.0,0.973,0.10230834250567276,0.04378686178903142,1.4469171977836348,0.26847190174256996,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.95,0.328,1.3204667694010184,0.74843530520864,1.0,0.995,0.10230834250567276,0.02998497094011609,1.4469171977836348,0.26847190174256996,1000 +LGBM Regr.,LassoCV,IV-type,0.9,0.02,1.521119145022205,1.4608769656478595,1.0,0.359,0.1858831134654844,0.10989835156011962,2.1959840483158897,0.7297672680739162,1000 +LGBM Regr.,LassoCV,IV-type,0.95,0.045,1.8125252554924385,1.4608769656478595,1.0,0.58,0.1858831134654844,0.08960405454511797,2.1959840483158897,0.7297672680739162,1000 +LGBM Regr.,LassoCV,partialling out,0.9,0.021,1.5093401960317487,1.3250609990655347,1.0,0.537,0.17222024016738868,0.09586170007661886,2.053549895430398,0.6003221955363113,1000 +LGBM Regr.,LassoCV,partialling out,0.95,0.079,1.7984897720799624,1.3250609990655347,1.0,0.765,0.17222024016738868,0.07549935380647549,2.053549895430398,0.6003221955363113,1000 +LassoCV,LGBM Regr.,IV-type,0.9,0.748,2.5133664291353095,1.0223715401643154,1.0,1.0,0.06817434769512887,0.010427835528562208,2.534255845067475,0.5860957184805163,1000 +LassoCV,LGBM Regr.,IV-type,0.95,0.926,2.9948608194317967,1.0223715401643154,1.0,1.0,0.06817434769512887,0.0031611418652559196,2.534255845067475,0.5860957184805163,1000 +LassoCV,LGBM Regr.,partialling out,0.9,0.605,1.9815232621302428,0.9121069506452848,1.0,1.0,0.06053882318785803,0.012365997991750076,2.4395495885220395,0.6451231506011218,1000 +LassoCV,LGBM Regr.,partialling out,0.95,0.833,2.361130598290116,0.9121069506452848,1.0,1.0,0.06053882318785803,0.004652476916621015,2.4395495885220395,0.6451231506011218,1000 +LassoCV,LassoCV,IV-type,0.9,0.0,2.5877087930438964,4.872772215540649,1.0,0.0,0.28270722007024496,0.22413939259803953,6.407549231579281,3.3379951995020174,1000 +LassoCV,LassoCV,IV-type,0.95,0.001,3.0834452097987697,4.872772215540649,1.0,0.001,0.28270722007024496,0.20760827505991444,6.407549231579281,3.3379951995020174,1000 +LassoCV,LassoCV,partialling out,0.9,0.0,2.6022738054015564,4.872971572776508,1.0,0.0,0.2826615836284171,0.22383756375128439,6.408028359909112,3.337914785643903,1000 +LassoCV,LassoCV,partialling out,0.95,0.001,3.1008004924741663,4.872971572776508,1.0,0.001,0.2826615836284171,0.2072284307839625,6.408028359909112,3.337914785643903,1000 +LassoCV,RF Regr.,IV-type,0.9,0.03,2.230177042929521,1.7208906933423658,1.0,0.996,0.10321605320265151,0.050929864370534886,3.379395087094497,0.32699487018563966,1000 +LassoCV,RF Regr.,IV-type,0.95,0.104,2.6574198528480117,1.7208906933423658,1.0,0.999,0.10321605320265151,0.036611667122436437,3.379395087094497,0.32699487018563966,1000 +LassoCV,RF Regr.,partialling out,0.9,0.035,2.2613273518524517,1.6654740653451128,1.0,1.0,0.09838693440861622,0.046224192363929564,3.352306820686065,0.30396867960100216,1000 +LassoCV,RF Regr.,partialling out,0.95,0.126,2.6945377353123594,1.6654740653451128,1.0,1.0,0.09838693440861622,0.031946038970425784,3.352306820686065,0.30396867960100216,1000 +RF Regr.,LassoCV,IV-type,0.9,0.001,1.9755980491263099,2.4911901944187225,1.0,0.146,0.18765120739490848,0.13125708709313408,3.74493409869544,1.2384844937405288,1000 +RF Regr.,LassoCV,IV-type,0.95,0.004,2.354070271524165,2.4911901944187225,1.0,0.305,0.18765120739490848,0.11522730290243077,3.74493409869544,1.2384844937405288,1000 +RF Regr.,LassoCV,partialling out,0.9,0.003,1.9489582706745046,2.190059714864294,1.0,0.342,0.16663434910102978,0.11054051768693487,3.4484420814862635,0.9343987618760414,1000 +RF Regr.,LassoCV,partialling out,0.95,0.006,2.3223270176162565,2.190059714864294,1.0,0.58,0.16663434910102978,0.09457408564648179,3.4484420814862635,0.9343987618760414,1000 +RF Regr.,RF Regr.,IV-type,0.9,0.016,1.7671562935719778,1.6053531172501359,1.0,0.908,0.11827277682192193,0.06796316869150489,2.9370932976098705,0.3875377103244566,1000 +RF Regr.,RF Regr.,IV-type,0.95,0.047,2.1056966004164406,1.6053531172501359,1.0,0.972,0.11827277682192193,0.05390037915368153,2.9370932976098705,0.3875377103244566,1000 +RF Regr.,RF Regr.,partialling out,0.9,0.016,1.7741971479960048,1.5898420246719582,1.0,0.93,0.11671620294392733,0.06643566871448575,2.9271556769700973,0.3806480611364335,1000 +RF Regr.,RF Regr.,partialling out,0.95,0.057,2.114086295928167,1.5898420246719582,1.0,0.98,0.11671620294392733,0.052344557926308814,2.9271556769700973,0.3806480611364335,1000 diff --git a/results/plm/plr_ate_sensitivity_metadata.csv b/results/plm/plr_ate_sensitivity_metadata.csv new file mode 100644 index 0000000..24030fe --- /dev/null +++ b/results/plm/plr_ate_sensitivity_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,PLRATESensitivityCoverageSimulation,2025-06-05 16:23,227.22630832592645,3.12.3,scripts/plm/plr_ate_sensitivity_config.yml From 67afc5e7c1700960a708a9ace282c3274a742664 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Thu, 5 Jun 2025 19:04:03 +0200 Subject: [PATCH 24/35] fix ssm workflow --- .github/workflows/ssm_sim.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ssm_sim.yml b/.github/workflows/ssm_sim.yml index cdef61c..071a556 100644 --- a/.github/workflows/ssm_sim.yml +++ b/.github/workflows/ssm_sim.yml @@ -48,20 +48,27 @@ jobs: with: ref: ${{ env.TARGET_BRANCH }} + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.7.8" + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version-file: "monte-cover/pyproject.toml" - - name: Install dependencies + - name: Install Monte-Cover run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + cd monte-cover + uv venv + uv sync - name: Install DoubleML from correct branch run: | - pip uninstall -y doubleml - pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" + source monte-cover/.venv/bin/activate + uv pip uninstall doubleml + uv pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}" - name: Set up Git configuration run: | @@ -69,7 +76,9 @@ jobs: git config --global user.email 'github-actions@github.com' - name: Run scripts - run: python ${{ matrix.script }} + run: | + source monte-cover/.venv/bin/activate + uv run ${{ matrix.script }} - name: Commit any existing changes run: | From 110af8507e412668ab93c9a04bf01b2831ae85ee Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 17:58:35 +0000 Subject: [PATCH 25/35] Update results from script: scripts/rdd/rdd_sharp.py --- results/rdd/rdd_sharp_config.yml | 41 ++++++++++++++++++++++++++++++ results/rdd/rdd_sharp_coverage.csv | 27 ++++++++++++++++++++ results/rdd/rdd_sharp_metadata.csv | 2 ++ 3 files changed, 70 insertions(+) create mode 100644 results/rdd/rdd_sharp_config.yml create mode 100644 results/rdd/rdd_sharp_coverage.csv create mode 100644 results/rdd/rdd_sharp_metadata.csv diff --git a/results/rdd/rdd_sharp_config.yml b/results/rdd/rdd_sharp_config.yml new file mode 100644 index 0000000..57d0a43 --- /dev/null +++ b/results/rdd/rdd_sharp_config.yml @@ -0,0 +1,41 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 1000 + fuzzy: + - false + cutoff: + - 0.0 +learner_definitions: + lgbmr: &id001 + name: LGBM Regr. + params: + n_estimators: 100 + learning_rate: 0.05 + global_linear: &id002 + name: Global Linear + local_linear: &id003 + name: Linear + stacked_reg: &id004 + name: Stacked Regr. + params: + n_estimators: 100 + learning_rate: 0.05 +dml_parameters: + fs_specification: + - cutoff + - cutoff and score + - interacted cutoff and score + learners: + - ml_g: *id001 + - ml_g: *id002 + - ml_g: *id003 + - ml_g: *id004 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/rdd/rdd_sharp_coverage.csv b/results/rdd/rdd_sharp_coverage.csv new file mode 100644 index 0000000..e6fe699 --- /dev/null +++ b/results/rdd/rdd_sharp_coverage.csv @@ -0,0 +1,27 @@ +Method,fs_specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +RDFlex,cutoff,Global Linear,N/A,0.9,0.8693333333333334,1.9777420628015956,0.5363309802914388,1000 +RDFlex,cutoff,Global Linear,N/A,0.95,0.9296666666666666,2.356625021391914,0.5363309802914388,1000 +RDFlex,cutoff,LGBM Regr.,N/A,0.9,0.8756666666666666,0.5745846597977408,0.15277974067846242,1000 +RDFlex,cutoff,LGBM Regr.,N/A,0.95,0.9296666666666666,0.6846598510774335,0.15277974067846242,1000 +RDFlex,cutoff,Linear,N/A,0.9,0.8666666666666666,1.991920387201889,0.5404127325625548,1000 +RDFlex,cutoff,Linear,N/A,0.95,0.9286666666666666,2.3735195369465925,0.5404127325625548,1000 +RDFlex,cutoff,Stacked Regr.,N/A,0.9,0.8813333333333334,0.5670086678915763,0.1471163883503396,1000 +RDFlex,cutoff,Stacked Regr.,N/A,0.95,0.9423333333333334,0.6756324999259701,0.1471163883503396,1000 +RDFlex,cutoff and score,Global Linear,N/A,0.9,0.868,1.9777819118149618,0.5362223371501555,1000 +RDFlex,cutoff and score,Global Linear,N/A,0.95,0.9283333333333333,2.3566725044200316,0.5362223371501555,1000 +RDFlex,cutoff and score,LGBM Regr.,N/A,0.9,0.8663333333333334,0.6047557690566959,0.16255491915792666,1000 +RDFlex,cutoff and score,LGBM Regr.,N/A,0.95,0.934,0.7206109451761669,0.16255491915792666,1000 +RDFlex,cutoff and score,Linear,N/A,0.9,0.869,1.99069360970139,0.5359824339657442,1000 +RDFlex,cutoff and score,Linear,N/A,0.95,0.932,2.372057741393101,0.5359824339657442,1000 +RDFlex,cutoff and score,Stacked Regr.,N/A,0.9,0.8926666666666666,0.5869731443946519,0.15208518895862003,1000 +RDFlex,cutoff and score,Stacked Regr.,N/A,0.95,0.9443333333333334,0.6994216409626386,0.15208518895862003,1000 +RDFlex,interacted cutoff and score,Global Linear,N/A,0.9,0.8666666666666666,1.9803466175426516,0.5369353192144897,1000 +RDFlex,interacted cutoff and score,Global Linear,N/A,0.95,0.928,2.359728539786857,0.5369353192144897,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,N/A,0.9,0.884,0.6090642078915169,0.16109292443371379,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,N/A,0.95,0.94,0.725744766695285,0.16109292443371379,1000 +RDFlex,interacted cutoff and score,Linear,N/A,0.9,0.8683333333333334,2.000108510613013,0.5391094961686808,1000 +RDFlex,interacted cutoff and score,Linear,N/A,0.95,0.9276666666666666,2.3832762877746387,0.5391094961686808,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,N/A,0.9,0.8766666666666666,0.5858746023498046,0.15217399239455562,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,N/A,0.95,0.934,0.6981126473791827,0.15217399239455562,1000 +rdrobust,cutoff,Linear,Logistic,0.9,0.888,2.18636563211321,0.5631032433381369,1000 +rdrobust,cutoff,Linear,Logistic,0.95,0.94,2.605215336953788,0.5631032433381369,1000 diff --git a/results/rdd/rdd_sharp_metadata.csv b/results/rdd/rdd_sharp_metadata.csv new file mode 100644 index 0000000..4c44c96 --- /dev/null +++ b/results/rdd/rdd_sharp_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,RDDCoverageSimulation,2025-06-05 17:58,65.60530270735423,3.12.3,scripts/rdd/rdd_sharp_config.yml From 011e27fc3442257a093864fdd49e1828ef01314d Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 18:09:37 +0000 Subject: [PATCH 26/35] Update results from script: scripts/plm/pliv_late.py --- results/plm/pliv_late_config.yml | 57 ++++++++++++++++++++++++++++++ results/plm/pliv_late_coverage.csv | 33 +++++++++++++++++ results/plm/pliv_late_metadata.csv | 2 ++ 3 files changed, 92 insertions(+) create mode 100644 results/plm/pliv_late_config.yml create mode 100644 results/plm/pliv_late_coverage.csv create mode 100644 results/plm/pliv_late_metadata.csv diff --git a/results/plm/pliv_late_config.yml b/results/plm/pliv_late_config.yml new file mode 100644 index 0000000..9863dcf --- /dev/null +++ b/results/plm/pliv_late_config.yml @@ -0,0 +1,57 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 0.5 + n_obs: + - 500 + dim_x: + - 20 + dim_z: + - 1 +learner_definitions: + lasso: &id001 + name: LassoCV + rf: &id002 + name: RF Regr. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id001 + ml_r: *id001 + - ml_g: *id002 + ml_m: *id002 + ml_r: *id002 + - ml_g: *id001 + ml_m: *id002 + ml_r: *id002 + - ml_g: *id002 + ml_m: *id001 + ml_r: *id002 + - ml_g: *id002 + ml_m: *id002 + ml_r: *id001 + - ml_g: *id001 + ml_m: *id001 + ml_r: *id002 + - ml_g: *id002 + ml_m: *id001 + ml_r: *id001 + - ml_g: *id001 + ml_m: *id002 + ml_r: *id001 + score: + - partialling out + - IV-type +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/plm/pliv_late_coverage.csv b/results/plm/pliv_late_coverage.csv new file mode 100644 index 0000000..1a876b3 --- /dev/null +++ b/results/plm/pliv_late_coverage.csv @@ -0,0 +1,33 @@ +Learner g,Learner m,Learner r,Score,level,Coverage,CI Length,Bias,repetition +LassoCV,LassoCV,LassoCV,IV-type,0.9,0.7993019197207679,0.23180266726620893,0.07317965760520444,573 +LassoCV,LassoCV,LassoCV,IV-type,0.95,0.8603839441535777,0.27620991431567365,0.07317965760520444,573 +LassoCV,LassoCV,LassoCV,partialling out,0.9,0.8795811518324608,0.3005968104977072,0.07254739125260629,573 +LassoCV,LassoCV,LassoCV,partialling out,0.95,0.9493891797556719,0.3581831919810696,0.07254739125260629,573 +LassoCV,LassoCV,RF Regr.,IV-type,0.9,0.806282722513089,0.23262465528448026,0.07326064389971966,573 +LassoCV,LassoCV,RF Regr.,IV-type,0.95,0.8638743455497382,0.27718937345120853,0.07326064389971966,573 +LassoCV,LassoCV,RF Regr.,partialling out,0.9,0.8848167539267016,0.3081195890287047,0.07427328983504944,573 +LassoCV,LassoCV,RF Regr.,partialling out,0.95,0.9581151832460733,0.3671471354851205,0.07427328983504944,573 +LassoCV,RF Regr.,LassoCV,IV-type,0.9,0.8132635253054101,0.2659516317640923,0.07903719894266115,573 +LassoCV,RF Regr.,LassoCV,IV-type,0.95,0.8900523560209425,0.31690091528287584,0.07903719894266115,573 +LassoCV,RF Regr.,LassoCV,partialling out,0.9,0.8830715532286213,0.3186854967173057,0.07809285377225214,573 +LassoCV,RF Regr.,LassoCV,partialling out,0.95,0.9476439790575916,0.379737191034327,0.07809285377225214,573 +LassoCV,RF Regr.,RF Regr.,IV-type,0.9,0.8254799301919721,0.2668640644386955,0.07668266567351469,573 +LassoCV,RF Regr.,RF Regr.,IV-type,0.95,0.893542757417103,0.31798814587363317,0.07668266567351469,573 +LassoCV,RF Regr.,RF Regr.,partialling out,0.9,0.8656195462478184,0.30294245283479354,0.08542062639988371,573 +LassoCV,RF Regr.,RF Regr.,partialling out,0.95,0.93717277486911,0.36097819721799285,0.08542062639988371,573 +RF Regr.,LassoCV,LassoCV,IV-type,0.9,0.7818499127399651,0.24228998675825072,0.07584491443329881,573 +RF Regr.,LassoCV,LassoCV,IV-type,0.95,0.8586387434554974,0.28870632625286374,0.07584491443329881,573 +RF Regr.,LassoCV,LassoCV,partialling out,0.9,0.8970331588132635,0.3318879094683072,0.0806654890480541,573 +RF Regr.,LassoCV,LassoCV,partialling out,0.95,0.9406631762652705,0.3954688361345379,0.0806654890480541,573 +RF Regr.,LassoCV,RF Regr.,IV-type,0.9,0.8027923211169284,0.241628366516599,0.07544233872660495,573 +RF Regr.,LassoCV,RF Regr.,IV-type,0.95,0.8673647469458988,0.28791795710935314,0.07544233872660495,573 +RF Regr.,LassoCV,RF Regr.,partialling out,0.9,0.8952879581151832,0.32006834738062506,0.07708638562807664,573 +RF Regr.,LassoCV,RF Regr.,partialling out,0.95,0.9476439790575916,0.3813849592318696,0.07708638562807664,573 +RF Regr.,RF Regr.,LassoCV,IV-type,0.9,0.8184991273996509,0.2791009597646858,0.08075894727233103,573 +RF Regr.,RF Regr.,LassoCV,IV-type,0.95,0.8848167539267016,0.3325693060015278,0.08075894727233103,573 +RF Regr.,RF Regr.,LassoCV,partialling out,0.9,0.806282722513089,0.35157995852822205,0.10562690570040405,573 +RF Regr.,RF Regr.,LassoCV,partialling out,0.95,0.8726003490401396,0.4189333598507066,0.10562690570040405,573 +RF Regr.,RF Regr.,RF Regr.,IV-type,0.9,0.8359511343804538,0.2769681338448809,0.07840964131824099,573 +RF Regr.,RF Regr.,RF Regr.,IV-type,0.95,0.8830715532286213,0.33002788716667447,0.07840964131824099,573 +RF Regr.,RF Regr.,RF Regr.,partialling out,0.9,0.8534031413612565,0.304252712059565,0.07977048559824199,573 +RF Regr.,RF Regr.,RF Regr.,partialling out,0.95,0.9179755671902269,0.36253946738141946,0.07977048559824199,573 diff --git a/results/plm/pliv_late_metadata.csv b/results/plm/pliv_late_metadata.csv new file mode 100644 index 0000000..18f9cba --- /dev/null +++ b/results/plm/pliv_late_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,PLIVLATECoverageSimulation,2025-06-05 18:09,333.49471075932183,3.12.3,scripts/plm/pliv_late_config.yml From 26c6b50926bed600968a0e16963d7346060190bf Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 19:37:10 +0000 Subject: [PATCH 27/35] Update results from script: scripts/ssm/ssm_nonig_ate.py --- results/ssm/ssm_nonig_ate_config.yml | 74 ++++++++++++++++++++++++++ results/ssm/ssm_nonig_ate_coverage.csv | 19 +++++++ results/ssm/ssm_nonig_ate_metadata.csv | 2 + 3 files changed, 95 insertions(+) create mode 100644 results/ssm/ssm_nonig_ate_config.yml create mode 100644 results/ssm/ssm_nonig_ate_coverage.csv create mode 100644 results/ssm/ssm_nonig_ate_metadata.csv diff --git a/results/ssm/ssm_nonig_ate_config.yml b/results/ssm/ssm_nonig_ate_config.yml new file mode 100644 index 0000000..6c5f926 --- /dev/null +++ b/results/ssm/ssm_nonig_ate_config.yml @@ -0,0 +1,74 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 1.0 + n_obs: + - 500 + dim_x: + - 20 +learner_definitions: + lasso: &id001 + name: LassoCV + logit: &id002 + name: Logistic + rfr: &id003 + name: RF Regr. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + rfc: &id004 + name: RF Clas. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + lgbmr: &id005 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + lgbmc: &id006 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + ml_pi: *id002 + - ml_g: *id003 + ml_m: *id004 + ml_pi: *id004 + - ml_g: *id001 + ml_m: *id004 + ml_pi: *id004 + - ml_g: *id003 + ml_m: *id002 + ml_pi: *id004 + - ml_g: *id003 + ml_m: *id004 + ml_pi: *id002 + - ml_g: *id005 + ml_m: *id006 + ml_pi: *id006 + - ml_g: *id001 + ml_m: *id006 + ml_pi: *id006 + - ml_g: *id005 + ml_m: *id002 + ml_pi: *id006 + - ml_g: *id005 + ml_m: *id006 + ml_pi: *id002 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/ssm/ssm_nonig_ate_coverage.csv b/results/ssm/ssm_nonig_ate_coverage.csv new file mode 100644 index 0000000..9a3d225 --- /dev/null +++ b/results/ssm/ssm_nonig_ate_coverage.csv @@ -0,0 +1,19 @@ +Learner g,Learner m,Learner pi,level,Coverage,CI Length,Bias,repetition +LGBM Regr.,LGBM Clas.,LGBM Clas.,0.9,0.89,1.5301470087049076,0.3770578639809072,1000 +LGBM Regr.,LGBM Clas.,LGBM Clas.,0.95,0.942,1.823282618570531,0.3770578639809072,1000 +LGBM Regr.,LGBM Clas.,Logistic,0.9,0.929,2.4676110419059616,0.6723444365149791,1000 +LGBM Regr.,LGBM Clas.,Logistic,0.95,0.969,2.9403399127694723,0.6723444365149791,1000 +LGBM Regr.,Logistic,LGBM Clas.,0.9,0.809,1.0997736728076188,0.32081226177381494,1000 +LGBM Regr.,Logistic,LGBM Clas.,0.95,0.895,1.310461158688781,0.32081226177381494,1000 +LassoCV,LGBM Clas.,LGBM Clas.,0.9,0.902,1.4984436991476344,0.3690107638339736,1000 +LassoCV,LGBM Clas.,LGBM Clas.,0.95,0.961,1.785505795207747,0.3690107638339736,1000 +LassoCV,Logistic,Logistic,0.9,0.84,3.803087791117463,1.1219550707748396,1000 +LassoCV,Logistic,Logistic,0.95,0.916,4.531658609920874,1.1219550707748396,1000 +LassoCV,RF Clas.,RF Clas.,0.9,0.76,0.6487741040070446,0.20425601204335075,1000 +LassoCV,RF Clas.,RF Clas.,0.95,0.854,0.773062026383923,0.20425601204335075,1000 +RF Regr.,Logistic,RF Clas.,0.9,0.711,0.7424019979703704,0.26224742985370675,1000 +RF Regr.,Logistic,RF Clas.,0.95,0.816,0.8846265431954046,0.26224742985370675,1000 +RF Regr.,RF Clas.,Logistic,0.9,0.898,1.5259930175436052,0.4120022228090046,1000 +RF Regr.,RF Clas.,Logistic,0.95,0.958,1.818332832805496,0.4120022228090046,1000 +RF Regr.,RF Clas.,RF Clas.,0.9,0.759,0.6647246082851119,0.21200753632780625,1000 +RF Regr.,RF Clas.,RF Clas.,0.95,0.835,0.7920682245088012,0.21200753632780625,1000 diff --git a/results/ssm/ssm_nonig_ate_metadata.csv b/results/ssm/ssm_nonig_ate_metadata.csv new file mode 100644 index 0000000..0eab540 --- /dev/null +++ b/results/ssm/ssm_nonig_ate_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,SSMNonIgnorableATECoverageSimulation,2025-06-05 19:37,152.50586200555165,3.12.3,scripts/ssm/ssm_nonig_ate_config.yml From 3d37f89cb88533b41f0e5cf6c267523886dee8c1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 20:21:50 +0000 Subject: [PATCH 28/35] Update results from script: scripts/rdd/rdd_fuzzy.py --- results/rdd/rdd_fuzzy_config.yml | 63 ++++++++++++++++++++++++++++++ results/rdd/rdd_fuzzy_coverage.csv | 27 +++++++++++++ results/rdd/rdd_fuzzy_metadata.csv | 2 + 3 files changed, 92 insertions(+) create mode 100644 results/rdd/rdd_fuzzy_config.yml create mode 100644 results/rdd/rdd_fuzzy_coverage.csv create mode 100644 results/rdd/rdd_fuzzy_metadata.csv diff --git a/results/rdd/rdd_fuzzy_config.yml b/results/rdd/rdd_fuzzy_config.yml new file mode 100644 index 0000000..1c010bd --- /dev/null +++ b/results/rdd/rdd_fuzzy_config.yml @@ -0,0 +1,63 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + n_obs: + - 2000 + fuzzy: + - true + cutoff: + - 0.0 +learner_definitions: + lgbmr: &id001 + name: LGBM Regr. + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + lgbmc: &id002 + name: LGBM Clas. + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + global_linear: &id003 + name: Global Linear + global_logistic: &id004 + name: Global Logistic + local_linear: &id005 + name: Linear + local_logistic: &id006 + name: Logistic + stacked_reg: &id007 + name: Stacked Regr. + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 + stacked_cls: &id008 + name: Stacked Clas. + params: + n_estimators: 200 + learning_rate: 0.02 + max_depth: 5 +dml_parameters: + fs_specification: + - cutoff + - cutoff and score + - interacted cutoff and score + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + - ml_g: *id005 + ml_m: *id006 + - ml_g: *id007 + ml_m: *id008 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/rdd/rdd_fuzzy_coverage.csv b/results/rdd/rdd_fuzzy_coverage.csv new file mode 100644 index 0000000..a6c0a42 --- /dev/null +++ b/results/rdd/rdd_fuzzy_coverage.csv @@ -0,0 +1,27 @@ +Method,fs_specification,Learner g,Learner m,level,Coverage,CI Length,Bias,repetition +RDFlex,cutoff,Global Linear,Global Logistic,0.9,0.914,159.78774916508115,16.371821196962774,1000 +RDFlex,cutoff,Global Linear,Global Logistic,0.95,0.9616666666666667,190.39884668322458,16.371821196962774,1000 +RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.9,0.911,21.894892395588858,2.054842519054099,1000 +RDFlex,cutoff,LGBM Regr.,LGBM Clas.,0.95,0.964,26.089373447939103,2.054842519054099,1000 +RDFlex,cutoff,Linear,Logistic,0.9,0.912,24.13350016941463,3.970042561943364,1000 +RDFlex,cutoff,Linear,Logistic,0.95,0.9606666666666667,28.756839136264052,3.970042561943364,1000 +RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.9,0.9203333333333333,3.9519734545980754,0.647703696648072,1000 +RDFlex,cutoff,Stacked Regr.,Stacked Clas.,0.95,0.9736666666666667,4.709066820265513,0.647703696648072,1000 +RDFlex,cutoff and score,Global Linear,Global Logistic,0.9,0.9136666666666666,40.229554938489734,4.971844764740165,1000 +RDFlex,cutoff and score,Global Linear,Global Logistic,0.95,0.961,47.9364713683679,4.971844764740165,1000 +RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.9146666666666666,16.86998628777336,1.5860785538090398,1000 +RDFlex,cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9656666666666667,20.101828516499268,1.5860785538090398,1000 +RDFlex,cutoff and score,Linear,Logistic,0.9,0.914,97.14155501358066,9.810720713145727,1000 +RDFlex,cutoff and score,Linear,Logistic,0.95,0.963,115.75130218833239,9.810720713145727,1000 +RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.923,2.1887222402404873,0.5251851443672533,1000 +RDFlex,cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.9683333333333334,2.6080234087356517,0.5251851443672533,1000 +RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.9,0.916,39.62681032308488,5.974362235508999,1000 +RDFlex,interacted cutoff and score,Global Linear,Global Logistic,0.95,0.9616666666666667,47.218256860577036,5.974362235508999,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.9,0.9126666666666666,207.48089868228553,16.46141812561768,1000 +RDFlex,interacted cutoff and score,LGBM Regr.,LGBM Clas.,0.95,0.9626666666666667,247.22873952679137,16.46141812561768,1000 +RDFlex,interacted cutoff and score,Linear,Logistic,0.9,0.9143333333333333,1703.658231476743,157.0745025974768,1000 +RDFlex,interacted cutoff and score,Linear,Logistic,0.95,0.9626666666666667,2030.033992658808,157.0745025974768,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.9,0.9093333333333333,2.7930762710442028,0.5786138381787103,1000 +RDFlex,interacted cutoff and score,Stacked Regr.,Stacked Clas.,0.95,0.9663333333333334,3.328155653257759,0.5786138381787103,1000 +rdrobust,cutoff,Linear,Logistic,0.9,0.935,16.18988307541303,3.355681291457316,1000 +rdrobust,cutoff,Linear,Logistic,0.95,0.976,19.291435554988908,3.355681291457316,1000 diff --git a/results/rdd/rdd_fuzzy_metadata.csv b/results/rdd/rdd_fuzzy_metadata.csv new file mode 100644 index 0000000..ca7af26 --- /dev/null +++ b/results/rdd/rdd_fuzzy_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,RDDCoverageSimulation,2025-06-05 20:21,208.91020024220148,3.12.3,scripts/rdd/rdd_fuzzy_config.yml From af46e5a39db2e9f1f96646cec88037c9e4f34522 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 21:15:55 +0000 Subject: [PATCH 29/35] Update results from script: scripts/ssm/ssm_mar_ate.py --- results/ssm/ssm_mar_ate_config.yml | 74 ++++++++++++++++++++++++++++ results/ssm/ssm_mar_ate_coverage.csv | 19 +++++++ results/ssm/ssm_mar_ate_metadata.csv | 2 + 3 files changed, 95 insertions(+) create mode 100644 results/ssm/ssm_mar_ate_config.yml create mode 100644 results/ssm/ssm_mar_ate_coverage.csv create mode 100644 results/ssm/ssm_mar_ate_metadata.csv diff --git a/results/ssm/ssm_mar_ate_config.yml b/results/ssm/ssm_mar_ate_config.yml new file mode 100644 index 0000000..6c5f926 --- /dev/null +++ b/results/ssm/ssm_mar_ate_config.yml @@ -0,0 +1,74 @@ +simulation_parameters: + repetitions: 1000 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 1.0 + n_obs: + - 500 + dim_x: + - 20 +learner_definitions: + lasso: &id001 + name: LassoCV + logit: &id002 + name: Logistic + rfr: &id003 + name: RF Regr. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + rfc: &id004 + name: RF Clas. + params: + n_estimators: 200 + max_features: 20 + max_depth: 5 + min_samples_leaf: 2 + lgbmr: &id005 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.01 + lgbmc: &id006 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.01 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + ml_pi: *id002 + - ml_g: *id003 + ml_m: *id004 + ml_pi: *id004 + - ml_g: *id001 + ml_m: *id004 + ml_pi: *id004 + - ml_g: *id003 + ml_m: *id002 + ml_pi: *id004 + - ml_g: *id003 + ml_m: *id004 + ml_pi: *id002 + - ml_g: *id005 + ml_m: *id006 + ml_pi: *id006 + - ml_g: *id001 + ml_m: *id006 + ml_pi: *id006 + - ml_g: *id005 + ml_m: *id002 + ml_pi: *id006 + - ml_g: *id005 + ml_m: *id006 + ml_pi: *id002 +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/ssm/ssm_mar_ate_coverage.csv b/results/ssm/ssm_mar_ate_coverage.csv new file mode 100644 index 0000000..aa8edb2 --- /dev/null +++ b/results/ssm/ssm_mar_ate_coverage.csv @@ -0,0 +1,19 @@ +Learner g,Learner m,Learner pi,level,Coverage,CI Length,Bias,repetition +LGBM Regr.,LGBM Clas.,LGBM Clas.,0.9,0.934,1.0713352028098442,0.24591485806272242,1000 +LGBM Regr.,LGBM Clas.,LGBM Clas.,0.95,0.981,1.2765746316095508,0.24591485806272242,1000 +LGBM Regr.,LGBM Clas.,Logistic,0.9,0.939,0.9131848507685725,0.21223789879924182,1000 +LGBM Regr.,LGBM Clas.,Logistic,0.95,0.972,1.088126863939359,0.21223789879924182,1000 +LGBM Regr.,Logistic,LGBM Clas.,0.9,0.933,0.7703469411630964,0.17142872246581048,1000 +LGBM Regr.,Logistic,LGBM Clas.,0.95,0.972,0.9179249968148138,0.17142872246581048,1000 +LassoCV,LGBM Clas.,LGBM Clas.,0.9,0.947,1.0364590345690332,0.2359720559564468,1000 +LassoCV,LGBM Clas.,LGBM Clas.,0.95,0.982,1.2350171139370278,0.2359720559564468,1000 +LassoCV,Logistic,Logistic,0.9,0.926,0.5826714123685559,0.12863481417003114,1000 +LassoCV,Logistic,Logistic,0.95,0.965,0.6942958110990313,0.12863481417003114,1000 +LassoCV,RF Clas.,RF Clas.,0.9,0.919,0.5111034002250002,0.11799184761111325,1000 +LassoCV,RF Clas.,RF Clas.,0.95,0.956,0.6090172647602495,0.11799184761111325,1000 +RF Regr.,Logistic,RF Clas.,0.9,0.923,0.5773836889150485,0.13144778185362027,1000 +RF Regr.,Logistic,RF Clas.,0.95,0.963,0.687995099984517,0.13144778185362027,1000 +RF Regr.,RF Clas.,Logistic,0.9,0.923,0.5549423867573083,0.1256504508256171,1000 +RF Regr.,RF Clas.,Logistic,0.95,0.958,0.6612546391467519,0.1256504508256171,1000 +RF Regr.,RF Clas.,RF Clas.,0.9,0.922,0.5213838221703648,0.12121755103534768,1000 +RF Regr.,RF Clas.,RF Clas.,0.95,0.961,0.6212671430647002,0.12121755103534768,1000 diff --git a/results/ssm/ssm_mar_ate_metadata.csv b/results/ssm/ssm_mar_ate_metadata.csv new file mode 100644 index 0000000..b659c07 --- /dev/null +++ b/results/ssm/ssm_mar_ate_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.dev0,SSMMarATECoverageSimulation,2025-06-05 21:15,251.25987704992295,3.12.3,scripts/ssm/ssm_mar_ate_config.yml From 235a797f6f3fe0fac3f0520ea644056f771c6ea2 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Fri, 6 Jun 2025 07:07:58 +0200 Subject: [PATCH 30/35] small did multi test run --- results/did/did_multi_detailed.csv | 49 ++++++++++++++++++++++++++++ results/did/did_multi_eventstudy.csv | 49 ++++++++++++++++++++++++++++ results/did/did_multi_group.csv | 49 ++++++++++++++++++++++++++++ results/did/did_multi_metadata.csv | 2 ++ results/did/did_multi_time.csv | 49 ++++++++++++++++++++++++++++ results/did/did_pa_multi_config.yml | 43 ++++++++++++++++++++++++ 6 files changed, 241 insertions(+) create mode 100644 results/did/did_multi_detailed.csv create mode 100644 results/did/did_multi_eventstudy.csv create mode 100644 results/did/did_multi_group.csv create mode 100644 results/did/did_multi_metadata.csv create mode 100644 results/did/did_multi_time.csv create mode 100644 results/did/did_pa_multi_config.yml diff --git a/results/did/did_multi_detailed.csv b/results/did/did_multi_detailed.csv new file mode 100644 index 0000000..57940d8 --- /dev/null +++ b/results/did/did_multi_detailed.csv @@ -0,0 +1,49 @@ +Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.475,0.6631981918824853,0.4245293917047067,0.1,0.9932181765069161,10 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.5416666666666666,0.7902493871815527,0.4245293917047067,0.2,1.1015948175949941,10 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.5416666666666667,0.5869258060896296,0.32764889570233524,0.1,0.8999313982334242,10 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.625,0.6993652338930899,0.32764889570233524,0.2,0.9905529011972265,10 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8916666666666666,0.5732380857479841,0.1296008781804815,0.9,0.8863780150212672,10 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9583333333333333,0.6830553091310898,0.1296008781804815,0.9,0.9768868984728167,10 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.4333333333333333,0.6644058230082063,0.4310666769580376,0.1,0.9956876207839418,10 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.525,0.7916883684223393,0.4310666769580376,0.2,1.1010287059662813,10 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.5333333333333333,0.5867446374588254,0.3252236559819743,0.1,0.9009653298965914,10 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.625,0.6991493581545523,0.3252236559819743,0.2,0.9906567705361429,10 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9166666666666666,0.5744375253283261,0.1271887141133709,0.9,0.886140673979245,10 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.95,0.6844845295435221,0.1271887141133709,0.9,0.9769555531920947,10 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9166666666666666,2.664236516104595,0.737702874596228,1.0,4.15532260850043,10 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9583333333333334,3.174633616207804,0.737702874596228,1.0,4.551877666833338,10 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9,3.5057037569343903,1.0460340720779053,0.8,5.373199518974397,10 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.95,4.177303677040729,1.0460340720779053,1.0,5.908190710461086,10 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9,2.281268632456931,0.5618099356228352,0.9,3.5680571904376768,10 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.95,2.7182992367310748,0.5618099356228352,1.0,3.9096210173277703,10 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9166666666666666,1.1409695517478806,0.2729230152948616,1.0,1.7661171504579372,10 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9583333333333334,1.3595490761249551,0.2729230152948616,1.0,1.9422779024965977,10 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.95,1.300679100736105,0.30657137533712187,0.9,2.0078045899921433,10 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9666666666666666,1.5498547415501558,0.30657137533712187,1.0,2.20728741408156,10 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9,0.9958269378502731,0.25005822501578195,1.0,1.562210388164876,10 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9666666666666666,1.1866009844527796,0.25005822501578195,1.0,1.7164594299889349,10 +Linear,Logistic,experimental,False,1,0.9,0.8666666666666666,0.29402341515497066,0.08910609381990196,0.7,0.4563945612014034,10 +Linear,Logistic,experimental,False,1,0.95,0.925,0.35035050832046627,0.08910609381990196,0.7,0.5008231628131503,10 +Linear,Logistic,experimental,False,4,0.9,0.33333333333333337,0.977397258784985,0.7610350745966782,0.1,1.4094995674731499,10 +Linear,Logistic,experimental,False,4,0.95,0.4083333333333333,1.1646406673627152,0.7610350745966782,0.3,1.5699905716332707,10 +Linear,Logistic,experimental,False,6,0.9,0.8416666666666666,0.9663512842635651,0.2905316800247778,0.9,1.3942167275591268,10 +Linear,Logistic,experimental,False,6,0.95,0.925,1.1514785768998363,0.2905316800247778,0.9,1.5608208532578387,10 +Linear,Logistic,experimental,True,1,0.9,0.875,0.29394309706642396,0.08863490209339472,0.7,0.4568938014814658,10 +Linear,Logistic,experimental,True,1,0.95,0.925,0.3502548034150089,0.08863490209339472,0.7,0.501950368270883,10 +Linear,Logistic,experimental,True,4,0.9,0.325,0.9782224880817683,0.7615993346262713,0.0,1.4186481260559465,10 +Linear,Logistic,experimental,True,4,0.95,0.425,1.165623988719814,0.7615993346262713,0.3,1.5858023453294943,10 +Linear,Logistic,experimental,True,6,0.9,0.8416666666666666,0.9666491673336306,0.2924338853282872,0.9,1.396461066885858,10 +Linear,Logistic,experimental,True,6,0.95,0.9083333333333334,1.151833526470647,0.2924338853282872,0.9,1.562384080954277,10 +Linear,Logistic,observational,False,1,0.9,0.85,0.31425203663015777,0.08367653043797604,0.9,0.4895007276578588,10 +Linear,Logistic,observational,False,1,0.95,0.925,0.37445439750466164,0.08367653043797604,1.0,0.5393388520558242,10 +Linear,Logistic,observational,False,4,0.9,0.4833333333333334,1.2561035808707968,0.7066371770929671,0.3,1.778385188880911,10 +Linear,Logistic,observational,False,4,0.95,0.6,1.496739733566086,0.7066371770929671,0.4,2.000876501405903,10 +Linear,Logistic,observational,False,6,0.9,0.8833333333333332,0.9938097946227102,0.27383988384601515,0.9,1.44354873326288,10 +Linear,Logistic,observational,False,6,0.95,0.9416666666666668,1.184197410047798,0.27383988384601515,0.9,1.6062137591033303,10 +Linear,Logistic,observational,True,1,0.9,0.875,0.3129076258919823,0.08545450451730417,0.9,0.48745486803071925,10 +Linear,Logistic,observational,True,1,0.95,0.9333333333333333,0.37285243330305884,0.08545450451730417,1.0,0.5382885976877344,10 +Linear,Logistic,observational,True,4,0.9,0.4583333333333333,1.2506620298584346,0.7103356872029178,0.2,1.7866328735215717,10 +Linear,Logistic,observational,True,4,0.95,0.5916666666666666,1.4902557256096856,0.7103356872029178,0.5,2.006300527306606,10 +Linear,Logistic,observational,True,6,0.9,0.8833333333333332,0.9882996807860529,0.2762054991697468,0.9,1.437888190932466,10 +Linear,Logistic,observational,True,6,0.95,0.925,1.1776317044472453,0.2762054991697468,0.9,1.608395843078576,10 diff --git a/results/did/did_multi_eventstudy.csv b/results/did/did_multi_eventstudy.csv new file mode 100644 index 0000000..3777945 --- /dev/null +++ b/results/did/did_multi_eventstudy.csv @@ -0,0 +1,49 @@ +Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.31666666666666665,0.6537151341141119,0.49066639902375886,0.1,0.8523540998501415,10 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.5166666666666667,0.7789496268960298,0.49066639902375886,0.3,0.9770428100782877,10 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.36666666666666664,0.5452858065123898,0.3778386818154926,0.1,0.743163176914931,10 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4333333333333333,0.6497481140774426,0.3778386818154926,0.2,0.8296678442930695,10 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.95,0.5315426696186827,0.12186390955805393,0.9,0.7244710155084734,10 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9666666666666668,0.6333721564208382,0.12186390955805393,0.9,0.8204261481453766,10 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.2833333333333333,0.6558959400557026,0.49088494137273975,0.1,0.8651478545918568,10 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.45,0.7815482174531182,0.49088494137273975,0.2,0.977876743090663,10 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.41666666666666663,0.5445922521271076,0.36496793788889453,0.1,0.7376622142543021,10 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4666666666666666,0.648921693054806,0.36496793788889453,0.1,0.8399324262943078,10 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9166666666666667,0.5333510002926822,0.1267921483923527,0.9,0.7262289685879977,10 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.95,0.635526915321633,0.1267921483923527,0.9,0.8201526076321896,10 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9333333333333333,2.5552992411975888,0.7608777281615188,1.0,3.5528245647198546,10 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9666666666666668,3.0448268468435344,0.7608777281615188,1.0,3.9835090886742615,10 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9,3.5766594600203696,1.1151310261364602,0.9,4.849978691856272,10 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.95,4.261852612135938,1.1151310261364602,0.9,5.4843854314489615,10 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9666666666666668,2.0715338989490997,0.4576593539811947,1.0,2.8905463706130057,10 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,1.0,2.4683848873647265,0.4576593539811947,1.0,3.226733174299394,10 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.95,1.1035468154353276,0.22988973515771507,1.0,1.549789512023247,10 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.95,1.3149571354356897,0.22988973515771507,1.0,1.7100374044594258,10 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9333333333333333,1.2615162998420864,0.2921735103221472,0.9,1.7349291303889927,10 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9666666666666666,1.5031893860265462,0.2921735103221472,0.9,1.9482906291010973,10 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9166666666666667,0.9020852664274039,0.223288825691725,1.0,1.268826579092233,10 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9666666666666668,1.0749008934362119,0.223288825691725,1.0,1.412850138548627,10 +Linear,Logistic,experimental,False,1,0.9,0.8166666666666667,0.20956843058873878,0.07506477801543116,0.8,0.3000288302842385,10 +Linear,Logistic,experimental,False,1,0.95,0.9,0.24971618721586605,0.07506477801543116,0.9,0.33172401119841494,10 +Linear,Logistic,experimental,False,4,0.9,0.26666666666666666,0.9758192029894814,0.8863553189507677,0.0,1.2590578271307205,10 +Linear,Logistic,experimental,False,4,0.95,0.35,1.1627602979036324,0.8863553189507677,0.3,1.425570646998339,10 +Linear,Logistic,experimental,False,6,0.9,0.8166666666666667,0.9629875332601576,0.2908069953332307,0.9,1.2404344667371887,10 +Linear,Logistic,experimental,False,6,0.95,0.9333333333333333,1.1474704203613983,0.2908069953332307,0.9,1.4105058132057242,10 +Linear,Logistic,experimental,True,1,0.9,0.8333333333333334,0.20947111193940754,0.07428343024326965,0.8,0.2979299079446137,10 +Linear,Logistic,experimental,True,1,0.95,0.9,0.24960022489278275,0.07428343024326965,0.9,0.3330048127441146,10 +Linear,Logistic,experimental,True,4,0.9,0.3,0.9769037190176618,0.8858975253049832,0.0,1.245923467397466,10 +Linear,Logistic,experimental,True,4,0.95,0.35,1.1640525784573916,0.8858975253049832,0.3,1.4322400852668717,10 +Linear,Logistic,experimental,True,6,0.9,0.8166666666666667,0.9636908575425596,0.2931962751755147,0.9,1.2538856223646675,10 +Linear,Logistic,experimental,True,6,0.95,0.9333333333333333,1.1483084829344887,0.2931962751755147,0.9,1.4163679478599076,10 +Linear,Logistic,observational,False,1,0.9,0.8666666666666666,0.22523328177594734,0.060145661276170145,0.9,0.32202147548595106,10 +Linear,Logistic,observational,False,1,0.95,0.9166666666666667,0.26838200868899714,0.060145661276170145,1.0,0.35718168239326376,10 +Linear,Logistic,observational,False,4,0.9,0.41666666666666663,1.3135619894646324,0.8111606055413789,0.3,1.671840343983336,10 +Linear,Logistic,observational,False,4,0.95,0.5666666666666667,1.5652056502942664,0.8111606055413789,0.4,1.8973728801243435,10 +Linear,Logistic,observational,False,6,0.9,0.9,0.9938846466929837,0.27188590324660405,0.9,1.2869823819918522,10 +Linear,Logistic,observational,False,6,0.95,0.9333333333333333,1.1842866017907592,0.27188590324660405,0.9,1.452249753513595,10 +Linear,Logistic,observational,True,1,0.9,0.85,0.22362395181955633,0.06146781325453693,0.9,0.31927096315703024,10 +Linear,Logistic,observational,True,1,0.95,0.9166666666666667,0.2664643737687314,0.06146781325453693,1.0,0.3549910024680031,10 +Linear,Logistic,observational,True,4,0.9,0.4,1.3111465991606206,0.8189855976253808,0.4,1.6704469678803615,10 +Linear,Logistic,observational,True,4,0.95,0.5666666666666667,1.5623275352286452,0.8189855976253808,0.5,1.894016910355654,10 +Linear,Logistic,observational,True,6,0.9,0.8833333333333334,0.987754294207036,0.2775290083947557,0.9,1.2747731988103106,10 +Linear,Logistic,observational,True,6,0.95,0.95,1.1769818362553224,0.2775290083947557,0.9,1.4482252219187701,10 diff --git a/results/did/did_multi_group.csv b/results/did/did_multi_group.csv new file mode 100644 index 0000000..759e756 --- /dev/null +++ b/results/did/did_multi_group.csv @@ -0,0 +1,49 @@ +Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.4,0.7038778757281438,0.47194973287162156,0.2,0.8758913214185977,10 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.5,0.8387222202249021,0.47194973287162156,0.2,1.001010069655882,10 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.5,0.6149881253656317,0.3604337033572076,0.2,0.7810219664812267,10 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.5666666666666667,0.732803549742243,0.3604337033572076,0.2,0.8919457124055196,10 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9333333333333333,0.5996559313467016,0.12668533432786627,0.9,0.7620919434781398,10 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9666666666666666,0.7145341137336549,0.12668533432786627,0.9,0.8630472730843429,10 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.4,0.7053152806494579,0.4746991771029457,0.0,0.8810869883220048,10 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4333333333333333,0.8404349938302946,0.4746991771029457,0.2,1.0105973604651746,10 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.4666666666666666,0.6137523693083393,0.35662870553179526,0.2,0.7787017019204658,10 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.5666666666666667,0.7313310555784558,0.35662870553179526,0.2,0.8859891714764163,10 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9666666666666666,0.6016187358287552,0.12413016996963395,0.9,0.7675776583241858,10 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9666666666666666,0.7168729395297528,0.12413016996963395,0.9,0.8672944117057485,10 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9,2.564117397311814,0.6748075074539025,1.0,3.211572092656728,10 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,1.0,3.0553343279413907,0.6748075074539025,1.0,3.6685301941243447,10 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9333333333333333,3.6314497401813917,1.135290510403337,0.9,4.591536068751408,10 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9666666666666666,4.327139257743116,1.135290510403337,0.9,5.164689966870119,10 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9,2.311485371433312,0.6025651661909022,1.0,2.944256005463674,10 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,1.0,2.7543047019916633,0.6025651661909022,1.0,3.32115409107021,10 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,1.0,1.159774095087729,0.2556180857092131,1.0,1.4568864048385959,10 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,1.0,1.381956071548696,0.2556180857092131,1.0,1.6519752896650126,10 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9333333333333333,1.3002684796757527,0.3050015094584157,1.0,1.6348037247808438,10 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9666666666666666,1.5493654563782737,0.3050015094584157,1.0,1.8717284730559123,10 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9666666666666666,1.0225970849005845,0.2444184989857893,1.0,1.305407047359292,10 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,1.0,1.2184995821272093,0.2444184989857893,1.0,1.4842057522002559,10 +Linear,Logistic,experimental,False,1,0.9,0.8666666666666666,0.2632736921387274,0.08586190204900797,0.8,0.3413446240666038,10 +Linear,Logistic,experimental,False,1,0.95,0.9333333333333333,0.31370995340487845,0.08586190204900797,0.9,0.38529167082027993,10 +Linear,Logistic,experimental,False,4,0.9,0.3,1.0817308916902049,0.8593353424866175,0.1,1.3746939016404482,10 +Linear,Logistic,experimental,False,4,0.95,0.4,1.2889618589385583,0.8593353424866175,0.3,1.5579590830548598,10 +Linear,Logistic,experimental,False,6,0.9,0.8333333333333333,1.0683268803526378,0.3323660779211622,0.9,1.3490656766283455,10 +Linear,Logistic,experimental,False,6,0.95,0.9333333333333333,1.2729899943060263,0.3323660779211622,0.9,1.5404615209866068,10 +Linear,Logistic,experimental,True,1,0.9,0.9,0.2631245503202201,0.08406984540734988,0.8,0.3387436337194215,10 +Linear,Logistic,experimental,True,1,0.95,0.9333333333333333,0.31353223996699353,0.08406984540734988,0.9,0.38588941069606797,10 +Linear,Logistic,experimental,True,4,0.9,0.3,1.081869575626785,0.8607114497457358,0.1,1.3624602314700855,10 +Linear,Logistic,experimental,True,4,0.95,0.4666666666666666,1.2891271110415283,0.8607114497457358,0.3,1.5716251157153072,10 +Linear,Logistic,experimental,True,6,0.9,0.8,1.0679763763477195,0.3311340138013665,0.9,1.3471096697431735,10 +Linear,Logistic,experimental,True,6,0.95,0.9333333333333333,1.2725723430239788,0.3311340138013665,0.9,1.5417334084201677,10 +Linear,Logistic,observational,False,1,0.9,0.8333333333333333,0.2816649382994895,0.07271564074321726,0.9,0.36083833235314666,10 +Linear,Logistic,observational,False,1,0.95,0.9666666666666666,0.3356244748645851,0.07271564074321726,0.9,0.4079591535467177,10 +Linear,Logistic,observational,False,4,0.9,0.4333333333333333,1.3994251351987763,0.8150159195179807,0.3,1.741311134100631,10 +Linear,Logistic,observational,False,4,0.95,0.6,1.6675178989228194,0.8150159195179807,0.5,2.0039775780893656,10 +Linear,Logistic,observational,False,6,0.9,0.9,1.0956980944083736,0.3101338106517739,0.9,1.386667166945419,10 +Linear,Logistic,observational,False,6,0.95,0.9333333333333333,1.3056048074926598,0.3101338106517739,0.9,1.558225874803193,10 +Linear,Logistic,observational,True,1,0.9,0.9666666666666666,0.2798937856932111,0.07210043374255667,0.9,0.36057975492164257,10 +Linear,Logistic,observational,True,1,0.95,0.9666666666666666,0.3335140163638711,0.07210043374255667,0.9,0.40431144423970544,10 +Linear,Logistic,observational,True,4,0.9,0.4333333333333333,1.3878865436390964,0.8157724009785868,0.4,1.734158677188527,10 +Linear,Logistic,observational,True,4,0.95,0.6,1.6537688190541107,0.8157724009785868,0.5,2.0031223112830108,10 +Linear,Logistic,observational,True,6,0.9,0.8666666666666666,1.0918235063839146,0.3093097916806968,0.9,1.3809527770014425,10 +Linear,Logistic,observational,True,6,0.95,0.9333333333333333,1.300987951099824,0.3093097916806968,0.9,1.5734346850225365,10 diff --git a/results/did/did_multi_metadata.csv b/results/did/did_multi_metadata.csv new file mode 100644 index 0000000..6ec764b --- /dev/null +++ b/results/did/did_multi_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.10.0,DIDMultiCoverageSimulation,2025-06-06 07:07,2.84502333799998,3.12.9,scripts/did/did_pa_multi_config.yml diff --git a/results/did/did_multi_time.csv b/results/did/did_multi_time.csv new file mode 100644 index 0000000..b56b019 --- /dev/null +++ b/results/did/did_multi_time.csv @@ -0,0 +1,49 @@ +Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.2333333333333333,0.6625987616638548,0.5213456662078564,0.2,0.7853107113133788,10 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.3,0.7895351220211091,0.5213456662078564,0.2,0.9197962066654052,10 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.26666666666666666,0.5493976514472162,0.4185989168586719,0.3,0.6682227683696846,10 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.36666666666666664,0.654647679516107,0.4185989168586719,0.3,0.7654036063784714,10 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9,0.5332788843896268,0.10688967347011427,0.9,0.6590174118187636,10 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9333333333333333,0.6354409839230054,0.10688967347011427,0.9,0.7462631615057108,10 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.2333333333333333,0.6634007679597379,0.5331593697938268,0.1,0.7887760040753716,10 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.3,0.7904907714658677,0.5331593697938268,0.2,0.9122615403961426,10 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.26666666666666666,0.5479992332361258,0.4111745820322453,0.3,0.6599184723785617,10 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.36666666666666664,0.652981361441263,0.4111745820322453,0.3,0.7613711498758222,10 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9,0.5346760598484942,0.11070063279136115,0.9,0.6528004508473261,10 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9,0.6371058211672402,0.11070063279136115,0.9,0.7449493953115034,10 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9666666666666666,2.724444869693017,0.8243030230260644,1.0,3.3909690969119404,10 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,1.0,3.246376294503422,0.8243030230260644,1.0,3.8976431441316266,10 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9,3.6042175187597345,1.1266225099616716,0.8,4.424909178964977,10 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9333333333333333,4.294690064495208,1.1266225099616716,0.9,4.989501158590644,10 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9333333333333332,2.049861926914148,0.497363662933101,1.0,2.586659888950317,10 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9666666666666666,2.442561139909952,0.497363662933101,1.0,2.931429368444929,10 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9333333333333333,1.1329976823562682,0.2659616385892486,0.9,1.3893901574468819,10 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9666666666666666,1.3500500078546822,0.2659616385892486,1.0,1.5944049463735803,10 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9333333333333333,1.251058212717813,0.3007003480415746,0.9,1.529240506606829,10 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9666666666666666,1.4907278066039762,0.3007003480415746,0.9,1.7668714749531325,10 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9333333333333332,0.8948452833589853,0.204625060665209,1.0,1.1255012536126696,10 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9666666666666666,1.0662739215097916,0.204625060665209,1.0,1.2696773979939644,10 +Linear,Logistic,experimental,False,1,0.9,0.8,0.24368352652491732,0.07973745587306341,0.7,0.3140608732531678,10 +Linear,Logistic,experimental,False,1,0.95,0.8333333333333334,0.29036683130263713,0.07973745587306341,0.9,0.353078827935582,10 +Linear,Logistic,experimental,False,4,0.9,0.06666666666666667,0.9696652351208508,1.0073141648234347,0.1,1.1209858064157003,10 +Linear,Logistic,experimental,False,4,0.95,0.2,1.1554273929041232,1.0073141648234347,0.2,1.2941513352794645,10 +Linear,Logistic,experimental,False,6,0.9,0.8333333333333333,0.9438277624968293,0.29488640962019724,0.9,1.0892187696065885,10 +Linear,Logistic,experimental,False,6,0.95,0.9333333333333333,1.1246401453552475,0.29488640962019724,0.9,1.2670156104265275,10 +Linear,Logistic,experimental,True,1,0.9,0.8,0.2436357920939464,0.0791259916382179,0.8,0.31171105219819967,10 +Linear,Logistic,experimental,True,1,0.95,0.8333333333333334,0.2903099522199074,0.0791259916382179,0.9,0.3520619816250074,10 +Linear,Logistic,experimental,True,4,0.9,0.03333333333333333,0.9695009673483899,1.0075438566095531,0.1,1.115461156732828,10 +Linear,Logistic,experimental,True,4,0.95,0.2,1.1552316557804254,1.0075438566095531,0.2,1.2789195615169306,10 +Linear,Logistic,experimental,True,6,0.9,0.8,0.9443978779018088,0.293922986691963,0.8,1.0881146173894785,10 +Linear,Logistic,experimental,True,6,0.95,0.9,1.1253194797607422,0.293922986691963,0.9,1.265074198410168,10 +Linear,Logistic,observational,False,1,0.9,0.9,0.26927852812297753,0.07828736376144599,1.0,0.3461250687337986,10 +Linear,Logistic,observational,False,1,0.95,0.9333333333333333,0.32086515680374456,0.07828736376144599,1.0,0.39040958691611627,10 +Linear,Logistic,observational,False,4,0.9,0.2333333333333333,1.3841812843946113,0.9247903996730132,0.2,1.5543899701191308,10 +Linear,Logistic,observational,False,4,0.95,0.4,1.6493537303473822,0.9247903996730132,0.2,1.822670006501189,10 +Linear,Logistic,observational,False,6,0.9,0.9,0.974459490524002,0.2757337897896041,0.9,1.1210193921596707,10 +Linear,Logistic,observational,False,6,0.95,0.9333333333333333,1.1611401005693511,0.2757337897896041,0.9,1.3140753581598792,10 +Linear,Logistic,observational,True,1,0.9,0.9,0.2685978168282629,0.07797281860557405,0.9,0.3454345116230722,10 +Linear,Logistic,observational,True,1,0.95,0.9666666666666666,0.32005403926741827,0.07797281860557405,1.0,0.38818579953504795,10 +Linear,Logistic,observational,True,4,0.9,0.2333333333333333,1.3696358280898424,0.9230010520428433,0.2,1.537629262768635,10 +Linear,Logistic,observational,True,4,0.95,0.3333333333333333,1.6320217501462715,0.9230010520428433,0.2,1.7984131238646603,10 +Linear,Logistic,observational,True,6,0.9,0.8666666666666666,0.9691273134777948,0.27660893376700224,0.8,1.1165632239349048,10 +Linear,Logistic,observational,True,6,0.95,0.9,1.1547864197320314,0.27660893376700224,0.9,1.2865502355295269,10 diff --git a/results/did/did_pa_multi_config.yml b/results/did/did_pa_multi_config.yml new file mode 100644 index 0000000..72961df --- /dev/null +++ b/results/did/did_pa_multi_config.yml @@ -0,0 +1,43 @@ +simulation_parameters: + repetitions: 10 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + DGP: + - 1 + - 4 + - 6 + n_obs: + - 2000 +learner_definitions: + linear: &id001 + name: Linear + logistic: &id002 + name: Logistic + lgbmr: &id003 + name: LGBM Regr. + params: + n_estimators: 500 + learning_rate: 0.02 + lgbmc: &id004 + name: LGBM Clas. + params: + n_estimators: 500 + learning_rate: 0.02 +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id002 + - ml_g: *id003 + ml_m: *id004 + score: + - observational + - experimental + in_sample_normalization: + - true + - false +confidence_parameters: + level: + - 0.95 + - 0.9 From 459f5de76d39581ff60aba16aee84d46201cdd21 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 6 Jun 2025 08:42:14 +0000 Subject: [PATCH 31/35] Update results from script: scripts/did/did_pa_atte_coverage.py --- results/did/did_pa_atte_coverage.csv | 49 +++++++++++++++++++ results/did/did_pa_atte_coverage_metadata.csv | 2 + 2 files changed, 51 insertions(+) create mode 100644 results/did/did_pa_atte_coverage.csv create mode 100644 results/did/did_pa_atte_coverage_metadata.csv diff --git a/results/did/did_pa_atte_coverage.csv b/results/did/did_pa_atte_coverage.csv new file mode 100644 index 0000000..9119e34 --- /dev/null +++ b/results/did/did_pa_atte_coverage.csv @@ -0,0 +1,49 @@ +Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,repetition +LGBM,LGBM,experimental,False,1,0.9,0.051,2.1624651397785515,2.1697505905302976,1000 +LGBM,LGBM,experimental,False,1,0.95,0.094,2.576736143777478,2.1697505905302976,1000 +LGBM,LGBM,experimental,False,2,0.9,0.378,2.1294819852681166,1.3213007503499328,1000 +LGBM,LGBM,experimental,False,2,0.95,0.475,2.537434291091179,1.3213007503499328,1000 +LGBM,LGBM,experimental,False,3,0.9,0.47,1.8796188347464011,1.0014962844281858,1000 +LGBM,LGBM,experimental,False,3,0.95,0.583,2.239703983626731,1.0014962844281858,1000 +LGBM,LGBM,experimental,False,4,0.9,0.078,1.8830006248174618,1.9173419242712182,1000 +LGBM,LGBM,experimental,False,4,0.95,0.122,2.243733635040054,1.9173419242712182,1000 +LGBM,LGBM,experimental,False,5,0.9,0.889,2.0642395778601936,0.5210006870170777,1000 +LGBM,LGBM,experimental,False,5,0.95,0.951,2.459693167693339,0.5210006870170777,1000 +LGBM,LGBM,experimental,False,6,0.9,0.908,1.8086231900602612,0.4337431389507495,1000 +LGBM,LGBM,experimental,False,6,0.95,0.948,2.1551074551794365,0.4337431389507495,1000 +LGBM,LGBM,experimental,True,1,0.9,0.049,2.159654648685151,2.17137681706751,1000 +LGBM,LGBM,experimental,True,1,0.95,0.099,2.573387237083486,2.17137681706751,1000 +LGBM,LGBM,experimental,True,2,0.9,0.373,2.129643042411216,1.319771507009933,1000 +LGBM,LGBM,experimental,True,2,0.95,0.47,2.537626202514028,1.319771507009933,1000 +LGBM,LGBM,experimental,True,3,0.9,0.477,1.8791031704354346,1.0000542552199656,1000 +LGBM,LGBM,experimental,True,3,0.95,0.598,2.23908953170162,1.0000542552199656,1000 +LGBM,LGBM,experimental,True,4,0.9,0.084,1.8855456999420728,1.9240618121414252,1000 +LGBM,LGBM,experimental,True,4,0.95,0.127,2.2467662791005663,1.9240618121414252,1000 +LGBM,LGBM,experimental,True,5,0.9,0.891,2.063666189492902,0.5200351889316859,1000 +LGBM,LGBM,experimental,True,5,0.95,0.947,2.459009933312703,0.5200351889316859,1000 +LGBM,LGBM,experimental,True,6,0.9,0.897,1.8093537325759845,0.43669034939483387,1000 +LGBM,LGBM,experimental,True,6,0.95,0.944,2.1559779502779253,0.43669034939483387,1000 +LGBM,LGBM,observational,False,1,0.9,0.893,12.590652453419517,3.3787304616783773,1000 +LGBM,LGBM,observational,False,1,0.95,0.953,15.002687744501154,3.3787304616783773,1000 +LGBM,LGBM,observational,False,2,0.9,0.914,14.716645515368727,3.622038823656133,1000 +LGBM,LGBM,observational,False,2,0.95,0.966,17.535964727040476,3.622038823656133,1000 +LGBM,LGBM,observational,False,3,0.9,0.933,14.387879061625718,3.413516510279929,1000 +LGBM,LGBM,observational,False,3,0.95,0.977,17.14421533481309,3.413516510279929,1000 +LGBM,LGBM,observational,False,4,0.9,0.843,18.129751335736472,5.765050835726839,1000 +LGBM,LGBM,observational,False,4,0.95,0.932,21.602931157204278,5.765050835726839,1000 +LGBM,LGBM,observational,False,5,0.9,0.917,7.704465948378402,1.901185439754437,1000 +LGBM,LGBM,observational,False,5,0.95,0.96,9.180437414922883,1.901185439754437,1000 +LGBM,LGBM,observational,False,6,0.9,0.922,7.569553123736534,1.7987428999886972,1000 +LGBM,LGBM,observational,False,6,0.95,0.971,9.019678868984235,1.7987428999886972,1000 +LGBM,LGBM,observational,True,1,0.9,0.906,4.1677645580041025,1.0303464962408189,1000 +LGBM,LGBM,observational,True,1,0.95,0.967,4.966197779476664,1.0303464962408189,1000 +LGBM,LGBM,observational,True,2,0.9,0.917,5.020279353776863,1.2235227285733399,1000 +LGBM,LGBM,observational,True,2,0.95,0.963,5.982031813960893,1.2235227285733399,1000 +LGBM,LGBM,observational,True,3,0.9,0.921,4.904759478369554,1.1326648179240986,1000 +LGBM,LGBM,observational,True,3,0.95,0.969,5.84438139231425,1.1326648179240986,1000 +LGBM,LGBM,observational,True,4,0.9,0.925,5.958922268906894,1.370042866021869,1000 +LGBM,LGBM,observational,True,4,0.95,0.971,7.1004938326197875,1.370042866021869,1000 +LGBM,LGBM,observational,True,5,0.9,0.92,3.449425982629519,0.8277123141255975,1000 +LGBM,LGBM,observational,True,5,0.95,0.965,4.110244572838216,0.8277123141255975,1000 +LGBM,LGBM,observational,True,6,0.9,0.925,3.40159548571924,0.8004114911308209,1000 +LGBM,LGBM,observational,True,6,0.95,0.97,4.053251020481494,0.8004114911308209,1000 diff --git a/results/did/did_pa_atte_coverage_metadata.csv b/results/did/did_pa_atte_coverage_metadata.csv new file mode 100644 index 0000000..1f17571 --- /dev/null +++ b/results/did/did_pa_atte_coverage_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version +0.11.dev0,did_pa_atte_coverage.py,2025-06-06 08:42:11,11024.603029727936,3.12.3 From 7d9eeb40148aaedcac0139dd4a94c6d10a52a862 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 6 Jun 2025 09:10:05 +0000 Subject: [PATCH 32/35] Update results from script: scripts/did/did_cs_atte_coverage.py --- results/did/did_cs_atte_coverage.csv | 49 +++++++++++++++++++ results/did/did_cs_atte_coverage_metadata.csv | 2 + 2 files changed, 51 insertions(+) create mode 100644 results/did/did_cs_atte_coverage.csv create mode 100644 results/did/did_cs_atte_coverage_metadata.csv diff --git a/results/did/did_cs_atte_coverage.csv b/results/did/did_cs_atte_coverage.csv new file mode 100644 index 0000000..53cf347 --- /dev/null +++ b/results/did/did_cs_atte_coverage.csv @@ -0,0 +1,49 @@ +Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,repetition +LGBM,LGBM,experimental,False,1,0.9,0.714,10.421426989395727,3.905582429463871,1000 +LGBM,LGBM,experimental,False,1,0.95,0.796,12.417896177537456,3.905582429463871,1000 +LGBM,LGBM,experimental,False,2,0.9,0.747,11.153655901082034,3.716762206006709,1000 +LGBM,LGBM,experimental,False,2,0.95,0.83,13.290400740757454,3.716762206006709,1000 +LGBM,LGBM,experimental,False,3,0.9,0.826,10.129500599367143,2.9806910089316485,1000 +LGBM,LGBM,experimental,False,3,0.95,0.9,12.070044428775317,2.9806910089316485,1000 +LGBM,LGBM,experimental,False,4,0.9,0.709,10.248410164509226,3.911148948219506,1000 +LGBM,LGBM,experimental,False,4,0.95,0.788,12.211733914865182,3.911148948219506,1000 +LGBM,LGBM,experimental,False,5,0.9,0.897,11.953436462004694,2.904688869350282,1000 +LGBM,LGBM,experimental,False,5,0.95,0.95,14.243398058730905,2.904688869350282,1000 +LGBM,LGBM,experimental,False,6,0.9,0.901,10.409876930645252,2.475898589693061,1000 +LGBM,LGBM,experimental,False,6,0.95,0.951,12.40413343366814,2.475898589693061,1000 +LGBM,LGBM,experimental,True,1,0.9,0.695,10.441642571924747,3.98549935766534,1000 +LGBM,LGBM,experimental,True,1,0.95,0.774,12.441984529859,3.98549935766534,1000 +LGBM,LGBM,experimental,True,2,0.9,0.769,11.14737947150305,3.7228962496196263,1000 +LGBM,LGBM,experimental,True,2,0.95,0.832,13.282921913629773,3.7228962496196263,1000 +LGBM,LGBM,experimental,True,3,0.9,0.822,10.139719900553855,2.993567832780566,1000 +LGBM,LGBM,experimental,True,3,0.95,0.896,12.082221477203783,2.993567832780566,1000 +LGBM,LGBM,experimental,True,4,0.9,0.707,10.258140130646604,3.9509918046955974,1000 +LGBM,LGBM,experimental,True,4,0.95,0.782,12.223327884618827,3.9509918046955974,1000 +LGBM,LGBM,experimental,True,5,0.9,0.894,11.981860540543671,2.9439981898378322,1000 +LGBM,LGBM,experimental,True,5,0.95,0.949,14.277267437329282,2.9439981898378322,1000 +LGBM,LGBM,experimental,True,6,0.9,0.894,10.42424549288115,2.562430198965583,1000 +LGBM,LGBM,experimental,True,6,0.95,0.955,12.421254631585413,2.562430198965583,1000 +LGBM,LGBM,observational,False,1,0.9,0.94,50.01837238134115,11.670635965681225,1000 +LGBM,LGBM,observational,False,1,0.95,0.973,59.600566777747616,11.670635965681225,1000 +LGBM,LGBM,observational,False,2,0.9,0.929,59.19235508827008,13.470175038952636,1000 +LGBM,LGBM,observational,False,2,0.95,0.977,70.53204141217991,13.470175038952636,1000 +LGBM,LGBM,observational,False,3,0.9,0.945,56.62260255116421,12.634763113659828,1000 +LGBM,LGBM,observational,False,3,0.95,0.989,67.46999240102099,12.634763113659828,1000 +LGBM,LGBM,observational,False,4,0.9,0.945,70.02798665966547,16.708878014378698,1000 +LGBM,LGBM,observational,False,4,0.95,0.982,83.44349279101235,16.708878014378698,1000 +LGBM,LGBM,observational,False,5,0.9,0.932,32.68395008367948,7.535531362351606,1000 +LGBM,LGBM,observational,False,5,0.95,0.973,38.945328621880215,7.535531362351606,1000 +LGBM,LGBM,observational,False,6,0.9,0.922,31.254676611393744,7.328062886784694,1000 +LGBM,LGBM,observational,False,6,0.95,0.96,37.24224423562365,7.328062886784694,1000 +LGBM,LGBM,observational,True,1,0.9,0.903,17.911052050251026,4.470376853620159,1000 +LGBM,LGBM,observational,True,1,0.95,0.954,21.342334885309523,4.470376853620159,1000 +LGBM,LGBM,observational,True,2,0.9,0.928,20.466840035852762,4.861276719991755,1000 +LGBM,LGBM,observational,True,2,0.95,0.965,24.387744107030723,4.861276719991755,1000 +LGBM,LGBM,observational,True,3,0.9,0.916,20.087760624155962,4.7945055222830755,1000 +LGBM,LGBM,observational,True,3,0.95,0.958,23.93604312766554,4.7945055222830755,1000 +LGBM,LGBM,observational,True,4,0.9,0.913,23.82669034521118,5.634290832938362,1000 +LGBM,LGBM,observational,True,4,0.95,0.956,28.391252681829,5.634290832938362,1000 +LGBM,LGBM,observational,True,5,0.9,0.89,16.373740873746755,4.117691170125128,1000 +LGBM,LGBM,observational,True,5,0.95,0.943,19.51051563427767,4.117691170125128,1000 +LGBM,LGBM,observational,True,6,0.9,0.891,14.987930087831806,3.7336563054022593,1000 +LGBM,LGBM,observational,True,6,0.95,0.957,17.85922023310908,3.7336563054022593,1000 diff --git a/results/did/did_cs_atte_coverage_metadata.csv b/results/did/did_cs_atte_coverage_metadata.csv new file mode 100644 index 0000000..08604a2 --- /dev/null +++ b/results/did/did_cs_atte_coverage_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (seconds),Python Version +0.11.dev0,did_cs_atte_coverage.py,2025-06-06 09:10:00,12688.770802021027,3.12.3 From 35d2a867c518cf1ed33c45c00347f23f90797a6e Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 6 Jun 2025 11:16:22 +0000 Subject: [PATCH 33/35] Update results from script: scripts/did/did_pa_multi.py --- results/did/did_multi_detailed.csv | 96 ++++++++++++++-------------- results/did/did_multi_eventstudy.csv | 96 ++++++++++++++-------------- results/did/did_multi_group.csv | 96 ++++++++++++++-------------- results/did/did_multi_metadata.csv | 2 +- results/did/did_multi_time.csv | 96 ++++++++++++++-------------- results/did/did_pa_multi_config.yml | 2 +- 6 files changed, 194 insertions(+), 194 deletions(-) diff --git a/results/did/did_multi_detailed.csv b/results/did/did_multi_detailed.csv index 57940d8..924bb00 100644 --- a/results/did/did_multi_detailed.csv +++ b/results/did/did_multi_detailed.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.475,0.6631981918824853,0.4245293917047067,0.1,0.9932181765069161,10 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.5416666666666666,0.7902493871815527,0.4245293917047067,0.2,1.1015948175949941,10 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.5416666666666667,0.5869258060896296,0.32764889570233524,0.1,0.8999313982334242,10 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.625,0.6993652338930899,0.32764889570233524,0.2,0.9905529011972265,10 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8916666666666666,0.5732380857479841,0.1296008781804815,0.9,0.8863780150212672,10 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9583333333333333,0.6830553091310898,0.1296008781804815,0.9,0.9768868984728167,10 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.4333333333333333,0.6644058230082063,0.4310666769580376,0.1,0.9956876207839418,10 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.525,0.7916883684223393,0.4310666769580376,0.2,1.1010287059662813,10 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.5333333333333333,0.5867446374588254,0.3252236559819743,0.1,0.9009653298965914,10 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.625,0.6991493581545523,0.3252236559819743,0.2,0.9906567705361429,10 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9166666666666666,0.5744375253283261,0.1271887141133709,0.9,0.886140673979245,10 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.95,0.6844845295435221,0.1271887141133709,0.9,0.9769555531920947,10 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9166666666666666,2.664236516104595,0.737702874596228,1.0,4.15532260850043,10 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9583333333333334,3.174633616207804,0.737702874596228,1.0,4.551877666833338,10 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9,3.5057037569343903,1.0460340720779053,0.8,5.373199518974397,10 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.95,4.177303677040729,1.0460340720779053,1.0,5.908190710461086,10 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9,2.281268632456931,0.5618099356228352,0.9,3.5680571904376768,10 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.95,2.7182992367310748,0.5618099356228352,1.0,3.9096210173277703,10 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9166666666666666,1.1409695517478806,0.2729230152948616,1.0,1.7661171504579372,10 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9583333333333334,1.3595490761249551,0.2729230152948616,1.0,1.9422779024965977,10 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.95,1.300679100736105,0.30657137533712187,0.9,2.0078045899921433,10 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9666666666666666,1.5498547415501558,0.30657137533712187,1.0,2.20728741408156,10 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9,0.9958269378502731,0.25005822501578195,1.0,1.562210388164876,10 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9666666666666666,1.1866009844527796,0.25005822501578195,1.0,1.7164594299889349,10 -Linear,Logistic,experimental,False,1,0.9,0.8666666666666666,0.29402341515497066,0.08910609381990196,0.7,0.4563945612014034,10 -Linear,Logistic,experimental,False,1,0.95,0.925,0.35035050832046627,0.08910609381990196,0.7,0.5008231628131503,10 -Linear,Logistic,experimental,False,4,0.9,0.33333333333333337,0.977397258784985,0.7610350745966782,0.1,1.4094995674731499,10 -Linear,Logistic,experimental,False,4,0.95,0.4083333333333333,1.1646406673627152,0.7610350745966782,0.3,1.5699905716332707,10 -Linear,Logistic,experimental,False,6,0.9,0.8416666666666666,0.9663512842635651,0.2905316800247778,0.9,1.3942167275591268,10 -Linear,Logistic,experimental,False,6,0.95,0.925,1.1514785768998363,0.2905316800247778,0.9,1.5608208532578387,10 -Linear,Logistic,experimental,True,1,0.9,0.875,0.29394309706642396,0.08863490209339472,0.7,0.4568938014814658,10 -Linear,Logistic,experimental,True,1,0.95,0.925,0.3502548034150089,0.08863490209339472,0.7,0.501950368270883,10 -Linear,Logistic,experimental,True,4,0.9,0.325,0.9782224880817683,0.7615993346262713,0.0,1.4186481260559465,10 -Linear,Logistic,experimental,True,4,0.95,0.425,1.165623988719814,0.7615993346262713,0.3,1.5858023453294943,10 -Linear,Logistic,experimental,True,6,0.9,0.8416666666666666,0.9666491673336306,0.2924338853282872,0.9,1.396461066885858,10 -Linear,Logistic,experimental,True,6,0.95,0.9083333333333334,1.151833526470647,0.2924338853282872,0.9,1.562384080954277,10 -Linear,Logistic,observational,False,1,0.9,0.85,0.31425203663015777,0.08367653043797604,0.9,0.4895007276578588,10 -Linear,Logistic,observational,False,1,0.95,0.925,0.37445439750466164,0.08367653043797604,1.0,0.5393388520558242,10 -Linear,Logistic,observational,False,4,0.9,0.4833333333333334,1.2561035808707968,0.7066371770929671,0.3,1.778385188880911,10 -Linear,Logistic,observational,False,4,0.95,0.6,1.496739733566086,0.7066371770929671,0.4,2.000876501405903,10 -Linear,Logistic,observational,False,6,0.9,0.8833333333333332,0.9938097946227102,0.27383988384601515,0.9,1.44354873326288,10 -Linear,Logistic,observational,False,6,0.95,0.9416666666666668,1.184197410047798,0.27383988384601515,0.9,1.6062137591033303,10 -Linear,Logistic,observational,True,1,0.9,0.875,0.3129076258919823,0.08545450451730417,0.9,0.48745486803071925,10 -Linear,Logistic,observational,True,1,0.95,0.9333333333333333,0.37285243330305884,0.08545450451730417,1.0,0.5382885976877344,10 -Linear,Logistic,observational,True,4,0.9,0.4583333333333333,1.2506620298584346,0.7103356872029178,0.2,1.7866328735215717,10 -Linear,Logistic,observational,True,4,0.95,0.5916666666666666,1.4902557256096856,0.7103356872029178,0.5,2.006300527306606,10 -Linear,Logistic,observational,True,6,0.9,0.8833333333333332,0.9882996807860529,0.2762054991697468,0.9,1.437888190932466,10 -Linear,Logistic,observational,True,6,0.95,0.925,1.1776317044472453,0.2762054991697468,0.9,1.608395843078576,10 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.3998435054773083,0.6774122808203658,0.45546224863660545,0.11737089201877934,1.012132591144609,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.4996087636932708,0.80718651881127,0.45546224863660545,0.16901408450704225,1.1223411583790253,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.5140845070422535,0.5825338629194045,0.338495362915356,0.1784037558685446,0.8960068193289008,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.610719874804382,0.6941319108212115,0.338495362915356,0.2535211267605634,0.9864382959365845,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8986697965571204,0.5797420679850099,0.14540866383510231,0.8826291079812206,0.8926096426690106,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9479655712050078,0.6908052819747433,0.14540866383510231,0.9342723004694836,0.9822524232973114,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.3916275430359938,0.6775298981684246,0.45592163092849536,0.09859154929577464,1.0135080886880636,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.48669796557120504,0.807326668525734,0.45592163092849536,0.1784037558685446,1.124193118709099,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.5172143974960877,0.5826574073228631,0.3365628793076158,0.16901408450704225,0.8969567851338134,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.6091549295774648,0.6942791230577917,0.3365628793076158,0.24413145539906103,0.9871059533494473,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8869327073552425,0.5798047605704594,0.14806948542850354,0.863849765258216,0.8925487708481121,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9448356807511737,0.690879984797879,0.14806948542850354,0.9154929577464789,0.983624165795513,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.8959311424100157,2.7633926433144422,0.7280607911432032,0.9624413145539906,4.285605520531536,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9636150234741784,3.2927854292283714,0.7280607911432032,0.9953051643192489,4.706108077772218,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9213615023474179,3.515668749299535,0.9392311486884903,0.9812206572769953,5.420434398399545,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9737871674491393,4.189177697817945,0.9392311486884903,0.9906103286384976,5.965771640360436,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.935054773082942,2.1578720027595883,0.496041638455299,0.9859154929577465,3.3700943427527528,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9737871674491393,2.5712630834481467,0.496041638455299,0.9953051643192489,3.698989752638499,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9205790297339593,1.1465042775820924,0.27713945258525985,0.9389671361502347,1.7867222903995619,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9604851330203443,1.3661441087294448,0.27713945258525985,0.9671361502347418,1.9593600449966166,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9194053208137716,1.4124733764235744,0.3231443456256321,0.9154929577464789,2.1833254527657897,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.965962441314554,1.6830658373187681,0.3231443456256321,0.9577464788732394,2.401459731447086,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9064945226917058,1.0226495617680404,0.25036302518907455,0.92018779342723,1.5996891640220663,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9557902973395932,1.2185621121715549,0.25036302518907455,0.9812206572769953,1.75215700962208,213 +Linear,Logistic,experimental,False,1,0.9,0.8599374021909233,0.2942329381007584,0.07885862091152207,0.8215962441314554,0.45900161300159426,213 +Linear,Logistic,experimental,False,1,0.95,0.9182316118935838,0.35060017030919877,0.07885862091152207,0.892018779342723,0.504127351002567,213 +Linear,Logistic,experimental,False,4,0.9,0.32629107981220656,0.9776211802565642,0.8124180807452879,0.051643192488262914,1.415728891229315,213 +Linear,Logistic,experimental,False,4,0.95,0.40375586854460094,1.1649074862531439,0.8124180807452879,0.07042253521126761,1.5791872279233108,213 +Linear,Logistic,experimental,False,6,0.9,0.877151799687011,0.9835799789631973,0.25364255139431,0.892018779342723,1.4193976201818594,213 +Linear,Logistic,experimental,False,6,0.95,0.9374021909233177,1.1720078328522332,0.25364255139431,0.9389671361502347,1.582242445231874,213 +Linear,Logistic,experimental,True,1,0.9,0.8583724569640063,0.29425152842277663,0.07893506849681084,0.8309859154929577,0.4588389524655604,213 +Linear,Logistic,experimental,True,1,0.95,0.9174491392801252,0.3506223220441737,0.07893506849681084,0.892018779342723,0.5038096551733684,213 +Linear,Logistic,experimental,True,4,0.9,0.32511737089201875,0.977667670038589,0.8122120271542994,0.051643192488262914,1.415351456768969,213 +Linear,Logistic,experimental,True,4,0.95,0.40375586854460094,1.1649628822451792,0.8122120271542994,0.07042253521126761,1.5770157392547421,213 +Linear,Logistic,experimental,True,6,0.9,0.8744131455399061,0.983350088492923,0.25389098394203446,0.8732394366197183,1.4200093317947935,213 +Linear,Logistic,experimental,True,6,0.95,0.9393583724569641,1.1717339014611698,0.25389098394203446,0.9295774647887324,1.5838096355577131,213 +Linear,Logistic,observational,False,1,0.9,0.9076682316118936,0.3175553687272376,0.07561516930339814,0.9248826291079812,0.4938850661149503,213 +Linear,Logistic,observational,False,1,0.95,0.954225352112676,0.3783905604757408,0.07561516930339814,0.971830985915493,0.5429993998740388,213 +Linear,Logistic,observational,False,4,0.9,0.430359937402191,1.2397563880413096,0.7953463265166806,0.19248826291079812,1.7693165100102601,213 +Linear,Logistic,observational,False,4,0.95,0.5246478873239436,1.4772608518776844,0.7953463265166806,0.29577464788732394,1.9799630660898182,213 +Linear,Logistic,observational,False,6,0.9,0.8802816901408451,1.0215638849527298,0.2657697771880829,0.8591549295774648,1.4750781011819736,213 +Linear,Logistic,observational,False,6,0.95,0.9358372456964007,1.2172684484546181,0.2657697771880829,0.9530516431924883,1.6461025177498927,213 +Linear,Logistic,observational,True,1,0.9,0.9080594679186228,0.3156057782031101,0.07507216274302907,0.9107981220657277,0.49086788828196987,213 +Linear,Logistic,observational,True,1,0.95,0.9534428794992176,0.37606748008166807,0.07507216274302907,0.9671361502347418,0.5391557012552065,213 +Linear,Logistic,observational,True,4,0.9,0.42801251956181535,1.240156742199843,0.7925778982914784,0.19248826291079812,1.770662270313202,213 +Linear,Logistic,observational,True,4,0.95,0.5230829420970267,1.4777379032814864,0.7925778982914784,0.28169014084507044,1.977822853534819,213 +Linear,Logistic,observational,True,6,0.9,0.8697183098591549,1.0162174373615258,0.2678133905875943,0.8591549295774648,1.467059987564026,213 +Linear,Logistic,observational,True,6,0.95,0.9362284820031298,1.2108977632141256,0.2678133905875943,0.9107981220657277,1.6385166854484863,213 diff --git a/results/did/did_multi_eventstudy.csv b/results/did/did_multi_eventstudy.csv index 3777945..d0cc9a1 100644 --- a/results/did/did_multi_eventstudy.csv +++ b/results/did/did_multi_eventstudy.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.31666666666666665,0.6537151341141119,0.49066639902375886,0.1,0.8523540998501415,10 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.5166666666666667,0.7789496268960298,0.49066639902375886,0.3,0.9770428100782877,10 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.36666666666666664,0.5452858065123898,0.3778386818154926,0.1,0.743163176914931,10 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4333333333333333,0.6497481140774426,0.3778386818154926,0.2,0.8296678442930695,10 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.95,0.5315426696186827,0.12186390955805393,0.9,0.7244710155084734,10 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9666666666666668,0.6333721564208382,0.12186390955805393,0.9,0.8204261481453766,10 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.2833333333333333,0.6558959400557026,0.49088494137273975,0.1,0.8651478545918568,10 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.45,0.7815482174531182,0.49088494137273975,0.2,0.977876743090663,10 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.41666666666666663,0.5445922521271076,0.36496793788889453,0.1,0.7376622142543021,10 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4666666666666666,0.648921693054806,0.36496793788889453,0.1,0.8399324262943078,10 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9166666666666667,0.5333510002926822,0.1267921483923527,0.9,0.7262289685879977,10 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.95,0.635526915321633,0.1267921483923527,0.9,0.8201526076321896,10 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9333333333333333,2.5552992411975888,0.7608777281615188,1.0,3.5528245647198546,10 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9666666666666668,3.0448268468435344,0.7608777281615188,1.0,3.9835090886742615,10 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9,3.5766594600203696,1.1151310261364602,0.9,4.849978691856272,10 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.95,4.261852612135938,1.1151310261364602,0.9,5.4843854314489615,10 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9666666666666668,2.0715338989490997,0.4576593539811947,1.0,2.8905463706130057,10 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,1.0,2.4683848873647265,0.4576593539811947,1.0,3.226733174299394,10 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.95,1.1035468154353276,0.22988973515771507,1.0,1.549789512023247,10 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.95,1.3149571354356897,0.22988973515771507,1.0,1.7100374044594258,10 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9333333333333333,1.2615162998420864,0.2921735103221472,0.9,1.7349291303889927,10 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9666666666666666,1.5031893860265462,0.2921735103221472,0.9,1.9482906291010973,10 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9166666666666667,0.9020852664274039,0.223288825691725,1.0,1.268826579092233,10 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9666666666666668,1.0749008934362119,0.223288825691725,1.0,1.412850138548627,10 -Linear,Logistic,experimental,False,1,0.9,0.8166666666666667,0.20956843058873878,0.07506477801543116,0.8,0.3000288302842385,10 -Linear,Logistic,experimental,False,1,0.95,0.9,0.24971618721586605,0.07506477801543116,0.9,0.33172401119841494,10 -Linear,Logistic,experimental,False,4,0.9,0.26666666666666666,0.9758192029894814,0.8863553189507677,0.0,1.2590578271307205,10 -Linear,Logistic,experimental,False,4,0.95,0.35,1.1627602979036324,0.8863553189507677,0.3,1.425570646998339,10 -Linear,Logistic,experimental,False,6,0.9,0.8166666666666667,0.9629875332601576,0.2908069953332307,0.9,1.2404344667371887,10 -Linear,Logistic,experimental,False,6,0.95,0.9333333333333333,1.1474704203613983,0.2908069953332307,0.9,1.4105058132057242,10 -Linear,Logistic,experimental,True,1,0.9,0.8333333333333334,0.20947111193940754,0.07428343024326965,0.8,0.2979299079446137,10 -Linear,Logistic,experimental,True,1,0.95,0.9,0.24960022489278275,0.07428343024326965,0.9,0.3330048127441146,10 -Linear,Logistic,experimental,True,4,0.9,0.3,0.9769037190176618,0.8858975253049832,0.0,1.245923467397466,10 -Linear,Logistic,experimental,True,4,0.95,0.35,1.1640525784573916,0.8858975253049832,0.3,1.4322400852668717,10 -Linear,Logistic,experimental,True,6,0.9,0.8166666666666667,0.9636908575425596,0.2931962751755147,0.9,1.2538856223646675,10 -Linear,Logistic,experimental,True,6,0.95,0.9333333333333333,1.1483084829344887,0.2931962751755147,0.9,1.4163679478599076,10 -Linear,Logistic,observational,False,1,0.9,0.8666666666666666,0.22523328177594734,0.060145661276170145,0.9,0.32202147548595106,10 -Linear,Logistic,observational,False,1,0.95,0.9166666666666667,0.26838200868899714,0.060145661276170145,1.0,0.35718168239326376,10 -Linear,Logistic,observational,False,4,0.9,0.41666666666666663,1.3135619894646324,0.8111606055413789,0.3,1.671840343983336,10 -Linear,Logistic,observational,False,4,0.95,0.5666666666666667,1.5652056502942664,0.8111606055413789,0.4,1.8973728801243435,10 -Linear,Logistic,observational,False,6,0.9,0.9,0.9938846466929837,0.27188590324660405,0.9,1.2869823819918522,10 -Linear,Logistic,observational,False,6,0.95,0.9333333333333333,1.1842866017907592,0.27188590324660405,0.9,1.452249753513595,10 -Linear,Logistic,observational,True,1,0.9,0.85,0.22362395181955633,0.06146781325453693,0.9,0.31927096315703024,10 -Linear,Logistic,observational,True,1,0.95,0.9166666666666667,0.2664643737687314,0.06146781325453693,1.0,0.3549910024680031,10 -Linear,Logistic,observational,True,4,0.9,0.4,1.3111465991606206,0.8189855976253808,0.4,1.6704469678803615,10 -Linear,Logistic,observational,True,4,0.95,0.5666666666666667,1.5623275352286452,0.8189855976253808,0.5,1.894016910355654,10 -Linear,Logistic,observational,True,6,0.9,0.8833333333333334,0.987754294207036,0.2775290083947557,0.9,1.2747731988103106,10 -Linear,Logistic,observational,True,6,0.95,0.95,1.1769818362553224,0.2775290083947557,0.9,1.4482252219187701,10 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.2652582159624413,0.6738426353849756,0.5274293504796588,0.09859154929577464,0.8819137878378639,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.35758998435054773,0.8029330239249745,0.5274293504796588,0.1596244131455399,1.0020805794352288,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.36619718309859156,0.5417379559244159,0.38469874476660076,0.1784037558685446,0.7377899281180879,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4561815336463224,0.6455205893536495,0.38469874476660076,0.2535211267605634,0.8314719135575573,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8935837245696401,0.539297489044855,0.1350874946339115,0.8779342723004695,0.733879976113594,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.94679186228482,0.6426125937052664,0.1350874946339115,0.9577464788732394,0.8249185552960796,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.2566510172143975,0.6741169305030346,0.5271254414071088,0.07981220657276995,0.8840733793865773,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.352112676056338,0.8032598667173798,0.5271254414071088,0.14084507042253522,1.0029350081401456,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.3669796557120501,0.5416448273714012,0.38289493428958665,0.15492957746478872,0.7375321995926126,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4663536776212832,0.6454096198382773,0.38289493428958665,0.2347417840375587,0.8302756334680312,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8857589984350547,0.5394652594848104,0.13836010080710312,0.8497652582159625,0.7318025579590295,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9475743348982786,0.6428125045147116,0.13836010080710312,0.92018779342723,0.82720613326034,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.8896713615023474,2.6890092512865227,0.7200494209782025,0.92018779342723,3.7251683690668043,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9553990610328639,3.2041521508418644,0.7200494209782025,0.9624413145539906,4.1740665666747265,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9209702660406885,3.5465803488647247,0.9687050535779654,0.9765258215962441,4.86821272823518,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9733959311424101,4.226011140538671,0.9687050535779654,0.9906103286384976,5.460451299700878,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9397496087636932,2.016844429290545,0.447496108983214,0.9624413145539906,2.808950740031606,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9757433489827857,2.403218364880276,0.447496108983214,0.9906103286384976,3.1542968976841452,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9209702660406885,1.0931141425868476,0.2586324201487641,0.9295774647887324,1.5164351424033533,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9687010954616588,1.302525838990542,0.2586324201487641,0.9671361502347418,1.6959957409224138,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9420970266040688,1.3861070534921676,0.29683891288883923,0.9295774647887324,1.9076412479291383,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9726134585289514,1.6516484257609465,0.29683891288883923,0.971830985915493,2.1381249471795822,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9053208137715181,0.9470898644218079,0.22867843675954816,0.8873239436619719,1.3244262222504375,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9616588419405321,1.1285271795460685,0.22867843675954816,0.9671361502347418,1.483381487672127,213 +Linear,Logistic,experimental,False,1,0.9,0.8395931142410016,0.20961349451394826,0.06166699387236784,0.7323943661971831,0.2990383792423722,213 +Linear,Logistic,experimental,False,1,0.95,0.9006259780907668,0.24976988419471297,0.06166699387236784,0.8262910798122066,0.3323039576385979,213 +Linear,Logistic,experimental,False,4,0.9,0.2136150234741784,0.9759976682205357,0.9530895482664571,0.046948356807511735,1.257805513435992,213 +Linear,Logistic,experimental,False,4,0.95,0.2683881064162754,1.1629729523426826,0.9530895482664571,0.06103286384976526,1.433457694555575,213 +Linear,Logistic,experimental,False,6,0.9,0.888888888888889,0.984101871756088,0.2529040307014624,0.8544600938967136,1.2672750485116713,213 +Linear,Logistic,experimental,False,6,0.95,0.9311424100156495,1.172629706471318,0.2529040307014624,0.9389671361502347,1.4397703898140197,213 +Linear,Logistic,experimental,True,1,0.9,0.838810641627543,0.20962019115150052,0.06163754557151296,0.7370892018779343,0.2991822645495156,213 +Linear,Logistic,experimental,True,1,0.95,0.9006259780907668,0.24977786373052385,0.06163754557151296,0.8309859154929577,0.33245309146308927,213 +Linear,Logistic,experimental,True,4,0.9,0.2167449139280125,0.9760619360265147,0.952817028222668,0.046948356807511735,1.2609463601610145,213 +Linear,Logistic,experimental,True,4,0.95,0.27230046948356806,1.1630495321568497,0.952817028222668,0.06572769953051644,1.4322602770975743,213 +Linear,Logistic,experimental,True,6,0.9,0.8834115805946792,0.983811761600466,0.2527027570120268,0.8497652582159625,1.265876521888396,213 +Linear,Logistic,experimental,True,6,0.95,0.9358372456964007,1.172284018899335,0.2527027570120268,0.9389671361502347,1.4415267063053059,213 +Linear,Logistic,observational,False,1,0.9,0.9100156494522692,0.2250881504094304,0.0527385366052549,0.9389671361502347,0.32114042199578385,213 +Linear,Logistic,observational,False,1,0.95,0.960093896713615,0.26820907399940563,0.0527385366052549,0.9624413145539906,0.3565208721390806,213 +Linear,Logistic,observational,False,4,0.9,0.3317683881064163,1.2904625746951797,0.929438804907565,0.18779342723004694,1.6422845737610774,213 +Linear,Logistic,observational,False,4,0.95,0.42018779342723006,1.5376809999118553,0.929438804907565,0.27699530516431925,1.8716546886192966,213 +Linear,Logistic,observational,False,6,0.9,0.8818466353677622,1.0252410978950457,0.26737451665728457,0.8544600938967136,1.317569321483734,213 +Linear,Logistic,observational,False,6,0.95,0.9327073552425664,1.2216501179310573,0.26737451665728457,0.92018779342723,1.5020723454442884,213 +Linear,Logistic,observational,True,1,0.9,0.9123630672926448,0.22389841311304975,0.052297945992543005,0.9248826291079812,0.31905149827680773,213 +Linear,Logistic,observational,True,1,0.95,0.9585289514866979,0.2667914145713799,0.052297945992543005,0.9765258215962441,0.3542427529311309,213 +Linear,Logistic,observational,True,4,0.9,0.3270735524256651,1.2929917075582165,0.9243991887988946,0.20187793427230047,1.64687629811095,213 +Linear,Logistic,observational,True,4,0.95,0.41784037558685444,1.54069464759603,0.9243991887988946,0.26291079812206575,1.8776567708863852,213 +Linear,Logistic,observational,True,6,0.9,0.8755868544600939,1.019446010780053,0.2700327354611733,0.8497652582159625,1.308225574447524,213 +Linear,Logistic,observational,True,6,0.95,0.9280125195618153,1.214744845725342,0.2700327354611733,0.92018779342723,1.4918494896646508,213 diff --git a/results/did/did_multi_group.csv b/results/did/did_multi_group.csv index 759e756..93cf706 100644 --- a/results/did/did_multi_group.csv +++ b/results/did/did_multi_group.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.4,0.7038778757281438,0.47194973287162156,0.2,0.8758913214185977,10 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.5,0.8387222202249021,0.47194973287162156,0.2,1.001010069655882,10 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.5,0.6149881253656317,0.3604337033572076,0.2,0.7810219664812267,10 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.5666666666666667,0.732803549742243,0.3604337033572076,0.2,0.8919457124055196,10 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9333333333333333,0.5996559313467016,0.12668533432786627,0.9,0.7620919434781398,10 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9666666666666666,0.7145341137336549,0.12668533432786627,0.9,0.8630472730843429,10 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.4,0.7053152806494579,0.4746991771029457,0.0,0.8810869883220048,10 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4333333333333333,0.8404349938302946,0.4746991771029457,0.2,1.0105973604651746,10 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.4666666666666666,0.6137523693083393,0.35662870553179526,0.2,0.7787017019204658,10 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.5666666666666667,0.7313310555784558,0.35662870553179526,0.2,0.8859891714764163,10 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9666666666666666,0.6016187358287552,0.12413016996963395,0.9,0.7675776583241858,10 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9666666666666666,0.7168729395297528,0.12413016996963395,0.9,0.8672944117057485,10 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9,2.564117397311814,0.6748075074539025,1.0,3.211572092656728,10 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,1.0,3.0553343279413907,0.6748075074539025,1.0,3.6685301941243447,10 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9333333333333333,3.6314497401813917,1.135290510403337,0.9,4.591536068751408,10 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9666666666666666,4.327139257743116,1.135290510403337,0.9,5.164689966870119,10 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9,2.311485371433312,0.6025651661909022,1.0,2.944256005463674,10 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,1.0,2.7543047019916633,0.6025651661909022,1.0,3.32115409107021,10 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,1.0,1.159774095087729,0.2556180857092131,1.0,1.4568864048385959,10 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,1.0,1.381956071548696,0.2556180857092131,1.0,1.6519752896650126,10 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9333333333333333,1.3002684796757527,0.3050015094584157,1.0,1.6348037247808438,10 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9666666666666666,1.5493654563782737,0.3050015094584157,1.0,1.8717284730559123,10 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9666666666666666,1.0225970849005845,0.2444184989857893,1.0,1.305407047359292,10 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,1.0,1.2184995821272093,0.2444184989857893,1.0,1.4842057522002559,10 -Linear,Logistic,experimental,False,1,0.9,0.8666666666666666,0.2632736921387274,0.08586190204900797,0.8,0.3413446240666038,10 -Linear,Logistic,experimental,False,1,0.95,0.9333333333333333,0.31370995340487845,0.08586190204900797,0.9,0.38529167082027993,10 -Linear,Logistic,experimental,False,4,0.9,0.3,1.0817308916902049,0.8593353424866175,0.1,1.3746939016404482,10 -Linear,Logistic,experimental,False,4,0.95,0.4,1.2889618589385583,0.8593353424866175,0.3,1.5579590830548598,10 -Linear,Logistic,experimental,False,6,0.9,0.8333333333333333,1.0683268803526378,0.3323660779211622,0.9,1.3490656766283455,10 -Linear,Logistic,experimental,False,6,0.95,0.9333333333333333,1.2729899943060263,0.3323660779211622,0.9,1.5404615209866068,10 -Linear,Logistic,experimental,True,1,0.9,0.9,0.2631245503202201,0.08406984540734988,0.8,0.3387436337194215,10 -Linear,Logistic,experimental,True,1,0.95,0.9333333333333333,0.31353223996699353,0.08406984540734988,0.9,0.38588941069606797,10 -Linear,Logistic,experimental,True,4,0.9,0.3,1.081869575626785,0.8607114497457358,0.1,1.3624602314700855,10 -Linear,Logistic,experimental,True,4,0.95,0.4666666666666666,1.2891271110415283,0.8607114497457358,0.3,1.5716251157153072,10 -Linear,Logistic,experimental,True,6,0.9,0.8,1.0679763763477195,0.3311340138013665,0.9,1.3471096697431735,10 -Linear,Logistic,experimental,True,6,0.95,0.9333333333333333,1.2725723430239788,0.3311340138013665,0.9,1.5417334084201677,10 -Linear,Logistic,observational,False,1,0.9,0.8333333333333333,0.2816649382994895,0.07271564074321726,0.9,0.36083833235314666,10 -Linear,Logistic,observational,False,1,0.95,0.9666666666666666,0.3356244748645851,0.07271564074321726,0.9,0.4079591535467177,10 -Linear,Logistic,observational,False,4,0.9,0.4333333333333333,1.3994251351987763,0.8150159195179807,0.3,1.741311134100631,10 -Linear,Logistic,observational,False,4,0.95,0.6,1.6675178989228194,0.8150159195179807,0.5,2.0039775780893656,10 -Linear,Logistic,observational,False,6,0.9,0.9,1.0956980944083736,0.3101338106517739,0.9,1.386667166945419,10 -Linear,Logistic,observational,False,6,0.95,0.9333333333333333,1.3056048074926598,0.3101338106517739,0.9,1.558225874803193,10 -Linear,Logistic,observational,True,1,0.9,0.9666666666666666,0.2798937856932111,0.07210043374255667,0.9,0.36057975492164257,10 -Linear,Logistic,observational,True,1,0.95,0.9666666666666666,0.3335140163638711,0.07210043374255667,0.9,0.40431144423970544,10 -Linear,Logistic,observational,True,4,0.9,0.4333333333333333,1.3878865436390964,0.8157724009785868,0.4,1.734158677188527,10 -Linear,Logistic,observational,True,4,0.95,0.6,1.6537688190541107,0.8157724009785868,0.5,2.0031223112830108,10 -Linear,Logistic,observational,True,6,0.9,0.8666666666666666,1.0918235063839146,0.3093097916806968,0.9,1.3809527770014425,10 -Linear,Logistic,observational,True,6,0.95,0.9333333333333333,1.300987951099824,0.3093097916806968,0.9,1.5734346850225365,10 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.3615023474178404,0.7202622844804144,0.5142431175686811,0.09389671361502347,0.8960068730292109,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.4522691705790297,0.8582454474204736,0.5142431175686811,0.16901408450704225,1.0218690905313046,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.4585289514866979,0.6080965137285453,0.37995101909660545,0.1784037558685446,0.7737228418008084,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.568075117370892,0.72459168797971,0.37995101909660545,0.23943661971830985,0.8761317466502612,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9107981220657277,0.6041693800921251,0.1487229003064066,0.892018779342723,0.768532913195283,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9577464788732394,0.7199122196283984,0.1487229003064066,0.9389671361502347,0.8712620903971866,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.3599374021909233,0.7202694109877334,0.5144563506346828,0.09859154929577464,0.896154594874867,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.43974960876369323,0.8582539391777047,0.5144563506346828,0.13615023474178403,1.0209397722371372,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.4726134585289514,0.6080786219858845,0.3781886928978884,0.18309859154929578,0.7737589269691583,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.5758998435054773,0.7245703686533813,0.3781886928978884,0.2347417840375587,0.8776184255890188,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8967136150234741,0.6043541483976993,0.15369943111934065,0.9061032863849765,0.7688492385342756,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9483568075117371,0.7201323846439818,0.15369943111934065,0.9436619718309859,0.8710978723491863,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9014084507042254,2.7211728776223616,0.7210652400496165,0.9389671361502347,3.4325568651778737,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9640062597809077,3.2424774754771537,0.7210652400496165,0.9765258215962441,3.90620301995042,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9280125195618153,3.5331758180056396,0.9482156709536594,0.9530516431924883,4.456779474115103,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9812206572769953,4.210038656858068,0.9482156709536594,0.9906103286384976,5.050979601042688,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9593114241001566,2.1134207943166583,0.4426633274641162,0.9671361502347418,2.68416063766941,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9890453834115805,2.518296201660071,0.4426633274641162,1.0,3.0441389382496644,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9311424100156495,1.1368385210095147,0.2642736374124477,0.9577464788732394,1.445045473376009,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.971830985915493,1.3546266493913184,0.2642736374124477,0.9906103286384976,1.6350299884984196,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9358372456964007,1.4316645897561893,0.3075262798142372,0.9295774647887324,1.812614988818496,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9687010954616588,1.7059335784570864,0.3075262798142372,0.9530516431924883,2.0542997322706174,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.92018779342723,1.008702982773344,0.23956099456087157,0.9295774647887324,1.2878304500066007,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9655712050078247,1.2019437382997051,0.23956099456087157,0.9671361502347418,1.4556619642167887,213 +Linear,Logistic,experimental,False,1,0.9,0.8466353677621283,0.26379623317984124,0.07367574399457054,0.812206572769953,0.33886769227656965,213 +Linear,Logistic,experimental,False,1,0.95,0.917057902973396,0.3143325994593638,0.07367574399457054,0.8873239436619719,0.3826093895409892,213 +Linear,Logistic,experimental,False,4,0.9,0.31768388106416273,1.081000965905216,0.9209342612568459,0.046948356807511735,1.3612059586767777,213 +Linear,Logistic,experimental,False,4,0.95,0.406885758998435,1.2880920987200657,0.9209342612568459,0.07042253521126761,1.5495796948573959,213 +Linear,Logistic,experimental,False,6,0.9,0.8841940532081377,1.0864109362400156,0.2793222944837174,0.8826291079812206,1.3646379214251372,213 +Linear,Logistic,experimental,False,6,0.95,0.9420970266040688,1.2945384759781382,0.2793222944837174,0.9436619718309859,1.5555664326647642,213 +Linear,Logistic,experimental,True,1,0.9,0.8482003129890453,0.26383019814224784,0.07370017470086833,0.7981220657276995,0.33938445004816475,213 +Linear,Logistic,experimental,True,1,0.95,0.917057902973396,0.314373071208316,0.07370017470086833,0.8779342723004695,0.38255405924208785,213 +Linear,Logistic,experimental,True,4,0.9,0.3145539906103286,1.0811145595227114,0.9206314437246235,0.046948356807511735,1.3629136590606885,213 +Linear,Logistic,experimental,True,4,0.95,0.406885758998435,1.288227453863841,0.9206314437246235,0.07042253521126761,1.5491777387798102,213 +Linear,Logistic,experimental,True,6,0.9,0.8763693270735524,1.0860190707615656,0.2799723827071057,0.8732394366197183,1.3678468139137758,213 +Linear,Logistic,experimental,True,6,0.95,0.9436619718309859,1.2940715394605282,0.2799723827071057,0.9436619718309859,1.5543990432787684,213 +Linear,Logistic,observational,False,1,0.9,0.9233176838810642,0.28345104003446464,0.06577425283822312,0.9154929577464789,0.36397364538854327,213 +Linear,Logistic,observational,False,1,0.95,0.9640062597809077,0.3377527463508229,0.06577425283822312,0.9577464788732394,0.41153782693535657,213 +Linear,Logistic,observational,False,4,0.9,0.41001564945226915,1.3821730458167794,0.9084755331600975,0.22065727699530516,1.7301013537520684,213 +Linear,Logistic,observational,False,4,0.95,0.5320813771517997,1.6469607664869994,0.9084755331600975,0.3145539906103286,1.9685722460940824,213 +Linear,Logistic,observational,False,6,0.9,0.8904538341158059,1.1242898926223517,0.2899194075787034,0.9061032863849765,1.4121558358228579,213 +Linear,Logistic,observational,False,6,0.95,0.9436619718309859,1.3396740364103084,0.2899194075787034,0.9436619718309859,1.6083520009872296,213 +Linear,Logistic,observational,True,1,0.9,0.9264475743348983,0.28168330654961965,0.06531180649498958,0.92018779342723,0.36242423041318955,213 +Linear,Logistic,observational,True,1,0.95,0.9640062597809077,0.3356463619845842,0.06531180649498958,0.9624413145539906,0.4088457694027527,213 +Linear,Logistic,observational,True,4,0.9,0.41471048513302033,1.382459197707508,0.905500217221854,0.20187793427230047,1.7291824258918072,213 +Linear,Logistic,observational,True,4,0.95,0.5164319248826291,1.6473017374954506,0.905500217221854,0.29577464788732394,1.9692425330349306,213 +Linear,Logistic,observational,True,6,0.9,0.8763693270735524,1.1216619887490014,0.29425840904927425,0.8967136150234741,1.405222302167041,213 +Linear,Logistic,observational,True,6,0.95,0.9405320813771518,1.336542695808199,0.29425840904927425,0.9389671361502347,1.599301420149991,213 diff --git a/results/did/did_multi_metadata.csv b/results/did/did_multi_metadata.csv index 6ec764b..eddd880 100644 --- a/results/did/did_multi_metadata.csv +++ b/results/did/did_multi_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.10.0,DIDMultiCoverageSimulation,2025-06-06 07:07,2.84502333799998,3.12.9,scripts/did/did_pa_multi_config.yml +0.11.dev0,DIDMultiCoverageSimulation,2025-06-06 11:16,337.9197948932648,3.12.3,scripts/did/did_pa_multi_config.yml diff --git a/results/did/did_multi_time.csv b/results/did/did_multi_time.csv index b56b019..c2b076e 100644 --- a/results/did/did_multi_time.csv +++ b/results/did/did_multi_time.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.2333333333333333,0.6625987616638548,0.5213456662078564,0.2,0.7853107113133788,10 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.3,0.7895351220211091,0.5213456662078564,0.2,0.9197962066654052,10 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.26666666666666666,0.5493976514472162,0.4185989168586719,0.3,0.6682227683696846,10 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.36666666666666664,0.654647679516107,0.4185989168586719,0.3,0.7654036063784714,10 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9,0.5332788843896268,0.10688967347011427,0.9,0.6590174118187636,10 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9333333333333333,0.6354409839230054,0.10688967347011427,0.9,0.7462631615057108,10 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.2333333333333333,0.6634007679597379,0.5331593697938268,0.1,0.7887760040753716,10 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.3,0.7904907714658677,0.5331593697938268,0.2,0.9122615403961426,10 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.26666666666666666,0.5479992332361258,0.4111745820322453,0.3,0.6599184723785617,10 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.36666666666666664,0.652981361441263,0.4111745820322453,0.3,0.7613711498758222,10 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9,0.5346760598484942,0.11070063279136115,0.9,0.6528004508473261,10 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9,0.6371058211672402,0.11070063279136115,0.9,0.7449493953115034,10 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9666666666666666,2.724444869693017,0.8243030230260644,1.0,3.3909690969119404,10 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,1.0,3.246376294503422,0.8243030230260644,1.0,3.8976431441316266,10 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9,3.6042175187597345,1.1266225099616716,0.8,4.424909178964977,10 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9333333333333333,4.294690064495208,1.1266225099616716,0.9,4.989501158590644,10 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9333333333333332,2.049861926914148,0.497363662933101,1.0,2.586659888950317,10 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9666666666666666,2.442561139909952,0.497363662933101,1.0,2.931429368444929,10 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9333333333333333,1.1329976823562682,0.2659616385892486,0.9,1.3893901574468819,10 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9666666666666666,1.3500500078546822,0.2659616385892486,1.0,1.5944049463735803,10 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9333333333333333,1.251058212717813,0.3007003480415746,0.9,1.529240506606829,10 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9666666666666666,1.4907278066039762,0.3007003480415746,0.9,1.7668714749531325,10 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9333333333333332,0.8948452833589853,0.204625060665209,1.0,1.1255012536126696,10 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9666666666666666,1.0662739215097916,0.204625060665209,1.0,1.2696773979939644,10 -Linear,Logistic,experimental,False,1,0.9,0.8,0.24368352652491732,0.07973745587306341,0.7,0.3140608732531678,10 -Linear,Logistic,experimental,False,1,0.95,0.8333333333333334,0.29036683130263713,0.07973745587306341,0.9,0.353078827935582,10 -Linear,Logistic,experimental,False,4,0.9,0.06666666666666667,0.9696652351208508,1.0073141648234347,0.1,1.1209858064157003,10 -Linear,Logistic,experimental,False,4,0.95,0.2,1.1554273929041232,1.0073141648234347,0.2,1.2941513352794645,10 -Linear,Logistic,experimental,False,6,0.9,0.8333333333333333,0.9438277624968293,0.29488640962019724,0.9,1.0892187696065885,10 -Linear,Logistic,experimental,False,6,0.95,0.9333333333333333,1.1246401453552475,0.29488640962019724,0.9,1.2670156104265275,10 -Linear,Logistic,experimental,True,1,0.9,0.8,0.2436357920939464,0.0791259916382179,0.8,0.31171105219819967,10 -Linear,Logistic,experimental,True,1,0.95,0.8333333333333334,0.2903099522199074,0.0791259916382179,0.9,0.3520619816250074,10 -Linear,Logistic,experimental,True,4,0.9,0.03333333333333333,0.9695009673483899,1.0075438566095531,0.1,1.115461156732828,10 -Linear,Logistic,experimental,True,4,0.95,0.2,1.1552316557804254,1.0075438566095531,0.2,1.2789195615169306,10 -Linear,Logistic,experimental,True,6,0.9,0.8,0.9443978779018088,0.293922986691963,0.8,1.0881146173894785,10 -Linear,Logistic,experimental,True,6,0.95,0.9,1.1253194797607422,0.293922986691963,0.9,1.265074198410168,10 -Linear,Logistic,observational,False,1,0.9,0.9,0.26927852812297753,0.07828736376144599,1.0,0.3461250687337986,10 -Linear,Logistic,observational,False,1,0.95,0.9333333333333333,0.32086515680374456,0.07828736376144599,1.0,0.39040958691611627,10 -Linear,Logistic,observational,False,4,0.9,0.2333333333333333,1.3841812843946113,0.9247903996730132,0.2,1.5543899701191308,10 -Linear,Logistic,observational,False,4,0.95,0.4,1.6493537303473822,0.9247903996730132,0.2,1.822670006501189,10 -Linear,Logistic,observational,False,6,0.9,0.9,0.974459490524002,0.2757337897896041,0.9,1.1210193921596707,10 -Linear,Logistic,observational,False,6,0.95,0.9333333333333333,1.1611401005693511,0.2757337897896041,0.9,1.3140753581598792,10 -Linear,Logistic,observational,True,1,0.9,0.9,0.2685978168282629,0.07797281860557405,0.9,0.3454345116230722,10 -Linear,Logistic,observational,True,1,0.95,0.9666666666666666,0.32005403926741827,0.07797281860557405,1.0,0.38818579953504795,10 -Linear,Logistic,observational,True,4,0.9,0.2333333333333333,1.3696358280898424,0.9230010520428433,0.2,1.537629262768635,10 -Linear,Logistic,observational,True,4,0.95,0.3333333333333333,1.6320217501462715,0.9230010520428433,0.2,1.7984131238646603,10 -Linear,Logistic,observational,True,6,0.9,0.8666666666666666,0.9691273134777948,0.27660893376700224,0.8,1.1165632239349048,10 -Linear,Logistic,observational,True,6,0.95,0.9,1.1547864197320314,0.27660893376700224,0.9,1.2865502355295269,10 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.1111111111111111,0.6848879510567563,0.5814212808822293,0.06572769953051644,0.8112788259677559,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.19092331768388104,0.8160943293200865,0.5814212808822293,0.1596244131455399,0.9363347602930319,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.2190923317683881,0.5442904416410593,0.42804406962185293,0.15023474178403756,0.6610129830707445,213 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.3114241001564945,0.6485620636792077,0.42804406962185293,0.2112676056338028,0.7594820835871338,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9076682316118936,0.540281441005603,0.13166144694063336,0.8967136150234741,0.6574061190218973,213 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9530516431924883,0.6437850447817539,0.13166144694063336,0.9436619718309859,0.755440248127496,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.10641627543035993,0.6849270317496503,0.5816182941099456,0.07042253521126761,0.810739883430405,213 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.17527386541471046,0.8161408968378937,0.5816182941099456,0.13145539906103287,0.9369536425675097,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.2190923317683881,0.5441327590803698,0.4271332871560829,0.15492957746478872,0.659742375492321,213 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.29733959311424096,0.6483741733193132,0.4271332871560829,0.20657276995305165,0.7581397106264711,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8982785602503913,0.5400972159018558,0.1320306660672212,0.892018779342723,0.65734442843056,213 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9546165884194053,0.6435655270310701,0.1320306660672212,0.9248826291079812,0.7544822430389665,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.863849765258216,2.947691714934299,0.8081717452002422,0.9014084507042254,3.6345410887622425,213 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9483568075117371,3.5123913180688033,0.8081717452002422,0.9530516431924883,4.148298978134212,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9107981220657277,3.920449689676996,1.0987394652573248,0.9342723004694836,4.803568258598088,213 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.974960876369327,4.671503937532334,1.0987394652573248,1.0,5.490527294034555,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9452269170579031,1.9822578095421914,0.41440886099461754,0.9765258215962441,2.4747132887427976,213 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9843505477308294,2.362005865516796,0.41440886099461754,0.9906103286384976,2.82160571997147,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9139280125195618,1.1523823561555302,0.27480514203439516,0.9295774647887324,1.4209065729930848,213 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9733959311424101,1.37314827135733,0.27480514203439516,0.9765258215962441,1.6207292567271263,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9295774647887324,1.4768427642563355,0.3193093946805207,0.92018779342723,1.8030473738560802,213 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9655712050078247,1.7597666937304888,0.3193093946805207,0.971830985915493,2.066811584773276,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9029733959311425,0.9274870427023983,0.22082805477342204,0.92018779342723,1.1562170046781757,213 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9593114241001566,1.1051689767638488,0.22082805477342204,0.971830985915493,1.3147550346737336,213 +Linear,Logistic,experimental,False,1,0.9,0.8231611893583725,0.24405699708437717,0.07177094325289468,0.7746478873239436,0.3128134096194603,213 +Linear,Logistic,experimental,False,1,0.95,0.895148669796557,0.290811848922341,0.07177094325289468,0.8685446009389671,0.3532819977654326,213 +Linear,Logistic,experimental,False,4,0.9,0.03912363067292645,0.9696140468983949,1.089724580728028,0.028169014084507043,1.112835689831825,213 +Linear,Logistic,experimental,False,4,0.95,0.0594679186228482,1.1553663983750042,1.089724580728028,0.056338028169014086,1.2908015410381681,213 +Linear,Logistic,experimental,False,6,0.9,0.8841940532081377,0.9637145816970751,0.24601437794174566,0.8591549295774648,1.1098958953800837,213 +Linear,Logistic,experimental,False,6,0.95,0.9420970266040688,1.1483367520081937,0.24601437794174566,0.92018779342723,1.2882569853203092,213 +Linear,Logistic,experimental,True,1,0.9,0.8294209702660407,0.24407331778379834,0.07187555926470358,0.755868544600939,0.31239740640105534,213 +Linear,Logistic,experimental,True,1,0.95,0.8904538341158059,0.29083129623518633,0.07187555926470358,0.8685446009389671,0.35340605322083257,213 +Linear,Logistic,experimental,True,4,0.9,0.03755868544600939,0.9698402059419257,1.089909708050757,0.028169014084507043,1.1118287534726101,213 +Linear,Logistic,experimental,True,4,0.95,0.06572769953051644,1.1556358834968625,1.089909708050757,0.056338028169014086,1.2947346339601935,213 +Linear,Logistic,experimental,True,6,0.9,0.8857589984350547,0.963597334445951,0.2461020735514813,0.8732394366197183,1.1092556702862066,213 +Linear,Logistic,experimental,True,6,0.95,0.9420970266040688,1.1481970432914277,0.2461020735514813,0.9295774647887324,1.288313460888964,213 +Linear,Logistic,observational,False,1,0.9,0.9092331768388106,0.2738536200757641,0.0653695842721706,0.9107981220657277,0.35111788102252456,213 +Linear,Logistic,observational,False,1,0.95,0.9577464788732394,0.32631671511051,0.0653695842721706,0.9577464788732394,0.39595800514716334,213 +Linear,Logistic,observational,False,4,0.9,0.17214397496087636,1.352533603300557,1.0665740251760334,0.15023474178403756,1.523002565481769,213 +Linear,Logistic,observational,False,4,0.95,0.26134585289514867,1.611643192386921,1.0665740251760334,0.22535211267605634,1.7732161740973758,213 +Linear,Logistic,observational,False,6,0.9,0.8857589984350547,1.0021244212459983,0.25876871605083246,0.8591549295774648,1.1510136525260573,213 +Linear,Logistic,observational,False,6,0.95,0.9389671361502347,1.1941048987504526,0.25876871605083246,0.92018779342723,1.3383058201805063,213 +Linear,Logistic,observational,True,1,0.9,0.9123630672926448,0.2715453775356287,0.0644928980540172,0.9154929577464789,0.3477637744633791,213 +Linear,Logistic,observational,True,1,0.95,0.9608763693270735,0.32356627448034075,0.0644928980540172,0.9624413145539906,0.3936582053408612,213 +Linear,Logistic,observational,True,4,0.9,0.17683881064162754,1.3541054412513556,1.0639801560397304,0.15023474178403756,1.5236529859485555,213 +Linear,Logistic,observational,True,4,0.95,0.24413145539906103,1.6135161528270596,1.0639801560397304,0.2112676056338028,1.7775778058708633,213 +Linear,Logistic,observational,True,6,0.9,0.8763693270735524,0.9971323968875062,0.2594135097814355,0.8732394366197183,1.1473737786354752,213 +Linear,Logistic,observational,True,6,0.95,0.9327073552425664,1.1881565348399659,0.2594135097814355,0.9154929577464789,1.33540506645655,213 diff --git a/results/did/did_pa_multi_config.yml b/results/did/did_pa_multi_config.yml index 72961df..ed4e23a 100644 --- a/results/did/did_pa_multi_config.yml +++ b/results/did/did_pa_multi_config.yml @@ -1,5 +1,5 @@ simulation_parameters: - repetitions: 10 + repetitions: 1000 max_runtime: 19800 random_seed: 42 n_jobs: -2 From 22b5452b8bb584daf48d47d04be80887954744c0 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Fri, 6 Jun 2025 13:50:22 +0200 Subject: [PATCH 34/35] update did multi results --- results/did/did_multi_detailed.csv | 96 ++++++++++++++-------------- results/did/did_multi_eventstudy.csv | 96 ++++++++++++++-------------- results/did/did_multi_group.csv | 96 ++++++++++++++-------------- results/did/did_multi_metadata.csv | 2 +- results/did/did_multi_time.csv | 96 ++++++++++++++-------------- results/did/did_pa_multi_config.yml | 24 +++++-- scripts/did/did_pa_multi_config.yml | 24 +++++-- 7 files changed, 233 insertions(+), 201 deletions(-) diff --git a/results/did/did_multi_detailed.csv b/results/did/did_multi_detailed.csv index 924bb00..c996df8 100644 --- a/results/did/did_multi_detailed.csv +++ b/results/did/did_multi_detailed.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.3998435054773083,0.6774122808203658,0.45546224863660545,0.11737089201877934,1.012132591144609,213 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.4996087636932708,0.80718651881127,0.45546224863660545,0.16901408450704225,1.1223411583790253,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.5140845070422535,0.5825338629194045,0.338495362915356,0.1784037558685446,0.8960068193289008,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.610719874804382,0.6941319108212115,0.338495362915356,0.2535211267605634,0.9864382959365845,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8986697965571204,0.5797420679850099,0.14540866383510231,0.8826291079812206,0.8926096426690106,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9479655712050078,0.6908052819747433,0.14540866383510231,0.9342723004694836,0.9822524232973114,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.3916275430359938,0.6775298981684246,0.45592163092849536,0.09859154929577464,1.0135080886880636,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.48669796557120504,0.807326668525734,0.45592163092849536,0.1784037558685446,1.124193118709099,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.5172143974960877,0.5826574073228631,0.3365628793076158,0.16901408450704225,0.8969567851338134,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.6091549295774648,0.6942791230577917,0.3365628793076158,0.24413145539906103,0.9871059533494473,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8869327073552425,0.5798047605704594,0.14806948542850354,0.863849765258216,0.8925487708481121,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9448356807511737,0.690879984797879,0.14806948542850354,0.9154929577464789,0.983624165795513,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.8959311424100157,2.7633926433144422,0.7280607911432032,0.9624413145539906,4.285605520531536,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9636150234741784,3.2927854292283714,0.7280607911432032,0.9953051643192489,4.706108077772218,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9213615023474179,3.515668749299535,0.9392311486884903,0.9812206572769953,5.420434398399545,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9737871674491393,4.189177697817945,0.9392311486884903,0.9906103286384976,5.965771640360436,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.935054773082942,2.1578720027595883,0.496041638455299,0.9859154929577465,3.3700943427527528,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9737871674491393,2.5712630834481467,0.496041638455299,0.9953051643192489,3.698989752638499,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9205790297339593,1.1465042775820924,0.27713945258525985,0.9389671361502347,1.7867222903995619,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9604851330203443,1.3661441087294448,0.27713945258525985,0.9671361502347418,1.9593600449966166,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9194053208137716,1.4124733764235744,0.3231443456256321,0.9154929577464789,2.1833254527657897,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.965962441314554,1.6830658373187681,0.3231443456256321,0.9577464788732394,2.401459731447086,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9064945226917058,1.0226495617680404,0.25036302518907455,0.92018779342723,1.5996891640220663,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9557902973395932,1.2185621121715549,0.25036302518907455,0.9812206572769953,1.75215700962208,213 -Linear,Logistic,experimental,False,1,0.9,0.8599374021909233,0.2942329381007584,0.07885862091152207,0.8215962441314554,0.45900161300159426,213 -Linear,Logistic,experimental,False,1,0.95,0.9182316118935838,0.35060017030919877,0.07885862091152207,0.892018779342723,0.504127351002567,213 -Linear,Logistic,experimental,False,4,0.9,0.32629107981220656,0.9776211802565642,0.8124180807452879,0.051643192488262914,1.415728891229315,213 -Linear,Logistic,experimental,False,4,0.95,0.40375586854460094,1.1649074862531439,0.8124180807452879,0.07042253521126761,1.5791872279233108,213 -Linear,Logistic,experimental,False,6,0.9,0.877151799687011,0.9835799789631973,0.25364255139431,0.892018779342723,1.4193976201818594,213 -Linear,Logistic,experimental,False,6,0.95,0.9374021909233177,1.1720078328522332,0.25364255139431,0.9389671361502347,1.582242445231874,213 -Linear,Logistic,experimental,True,1,0.9,0.8583724569640063,0.29425152842277663,0.07893506849681084,0.8309859154929577,0.4588389524655604,213 -Linear,Logistic,experimental,True,1,0.95,0.9174491392801252,0.3506223220441737,0.07893506849681084,0.892018779342723,0.5038096551733684,213 -Linear,Logistic,experimental,True,4,0.9,0.32511737089201875,0.977667670038589,0.8122120271542994,0.051643192488262914,1.415351456768969,213 -Linear,Logistic,experimental,True,4,0.95,0.40375586854460094,1.1649628822451792,0.8122120271542994,0.07042253521126761,1.5770157392547421,213 -Linear,Logistic,experimental,True,6,0.9,0.8744131455399061,0.983350088492923,0.25389098394203446,0.8732394366197183,1.4200093317947935,213 -Linear,Logistic,experimental,True,6,0.95,0.9393583724569641,1.1717339014611698,0.25389098394203446,0.9295774647887324,1.5838096355577131,213 -Linear,Logistic,observational,False,1,0.9,0.9076682316118936,0.3175553687272376,0.07561516930339814,0.9248826291079812,0.4938850661149503,213 -Linear,Logistic,observational,False,1,0.95,0.954225352112676,0.3783905604757408,0.07561516930339814,0.971830985915493,0.5429993998740388,213 -Linear,Logistic,observational,False,4,0.9,0.430359937402191,1.2397563880413096,0.7953463265166806,0.19248826291079812,1.7693165100102601,213 -Linear,Logistic,observational,False,4,0.95,0.5246478873239436,1.4772608518776844,0.7953463265166806,0.29577464788732394,1.9799630660898182,213 -Linear,Logistic,observational,False,6,0.9,0.8802816901408451,1.0215638849527298,0.2657697771880829,0.8591549295774648,1.4750781011819736,213 -Linear,Logistic,observational,False,6,0.95,0.9358372456964007,1.2172684484546181,0.2657697771880829,0.9530516431924883,1.6461025177498927,213 -Linear,Logistic,observational,True,1,0.9,0.9080594679186228,0.3156057782031101,0.07507216274302907,0.9107981220657277,0.49086788828196987,213 -Linear,Logistic,observational,True,1,0.95,0.9534428794992176,0.37606748008166807,0.07507216274302907,0.9671361502347418,0.5391557012552065,213 -Linear,Logistic,observational,True,4,0.9,0.42801251956181535,1.240156742199843,0.7925778982914784,0.19248826291079812,1.770662270313202,213 -Linear,Logistic,observational,True,4,0.95,0.5230829420970267,1.4777379032814864,0.7925778982914784,0.28169014084507044,1.977822853534819,213 -Linear,Logistic,observational,True,6,0.9,0.8697183098591549,1.0162174373615258,0.2678133905875943,0.8591549295774648,1.467059987564026,213 -Linear,Logistic,observational,True,6,0.95,0.9362284820031298,1.2108977632141256,0.2678133905875943,0.9107981220657277,1.6385166854484863,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.34725,0.6632201286426477,0.49011237580369793,0.033,0.9893937609780801,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.4386666666666667,0.7902755264435218,0.49011237580369793,0.079,1.0974315281580984,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.3925,0.6374280121970815,0.4673059259736469,0.072,0.97009187850922,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4734166666666667,0.759542324114714,0.4673059259736469,0.111,1.071000291105895,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9011666666666667,0.6326304449554325,0.1563321623926723,0.888,0.9627064358023116,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9486666666666667,0.7538256701505138,0.1563321623926723,0.955,1.0626523999436193,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.3473333333333333,0.663082675494273,0.4905683815649934,0.033,0.9889346057283442,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4349166666666667,0.7901117409151553,0.4905683815649934,0.08,1.0975864011236443,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.3943333333333333,0.6375246725440924,0.46800538303488937,0.072,0.9700098655165373,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4765833333333333,0.759657502022201,0.46800538303488937,0.118,1.0708460305597636,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8973333333333333,0.6326541125548435,0.15625780987233984,0.89,0.9621250011958815,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.947,0.7538538718346554,0.15625780987233984,0.952,1.062489694804598,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9435833333333333,0.8381616751048,0.17814054462577147,0.953,1.2839156930486022,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9778333333333333,0.9987312363300253,0.17814054462577147,0.982,1.416218776113961,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.926,1.1565439632042973,0.2473594696190042,0.931,1.738632033036807,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9675,1.3781071320120069,0.2473594696190042,0.965,1.9268759390224799,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.931,0.7998363707192514,0.17526821717897792,0.945,1.2193415272752777,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9684166666666666,0.9530638194478139,0.17526821717897792,0.978,1.3462730296391963,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9301666666666666,0.7578895509440912,0.16579983845899396,0.945,1.163855412725748,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9713333333333334,0.9030811008166856,0.16579983845899396,0.98,1.2816924647051413,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.89675,1.026901361709593,0.2402117312036738,0.865,1.5489489542811425,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.94425,1.2236284442866847,0.2402117312036738,0.922,1.7135671893558606,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9195833333333334,0.7392347349554987,0.16906220777838002,0.931,1.131310154953256,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.962,0.8808525165361331,0.16906220777838002,0.963,1.2473273228458321,1000 +Linear,Logistic,experimental,False,1,0.9,0.86275,0.2948051239305335,0.07900091146011895,0.793,0.45951022758324694,1000 +Linear,Logistic,experimental,False,1,0.95,0.92075,0.35128197177800297,0.07900091146011895,0.871,0.5047371756723105,1000 +Linear,Logistic,experimental,False,4,0.9,0.3095833333333333,0.9746459049539102,0.8123545762455107,0.041,1.4101593649841206,1000 +Linear,Logistic,experimental,False,4,0.95,0.39075,1.1613622270630595,0.8123545762455107,0.071,1.5725658487365672,1000 +Linear,Logistic,experimental,False,6,0.9,0.9036666666666666,0.9838971680839407,0.238720331830627,0.907,1.4193492655675315,1000 +Linear,Logistic,experimental,False,6,0.95,0.9551666666666666,1.172385787001319,0.238720331830627,0.96,1.5835849590579594,1000 +Linear,Logistic,experimental,True,1,0.9,0.8628333333333333,0.2948075666878801,0.07902018736146375,0.796,0.4595632294378891,1000 +Linear,Logistic,experimental,True,1,0.95,0.9198333333333334,0.3512848825029112,0.07902018736146375,0.867,0.5043781347089554,1000 +Linear,Logistic,experimental,True,4,0.9,0.309,0.9745871850206221,0.8120029975685781,0.042,1.4109703744664022,1000 +Linear,Logistic,experimental,True,4,0.95,0.3904166666666667,1.1612922579469427,0.8120029975685781,0.072,1.573528921909118,1000 +Linear,Logistic,experimental,True,6,0.9,0.9026666666666666,0.9840257361416861,0.23855494093764582,0.909,1.4210470533624664,1000 +Linear,Logistic,experimental,True,6,0.95,0.955,1.172538985291194,0.23855494093764582,0.963,1.585082469200275,1000 +Linear,Logistic,observational,False,1,0.9,0.9035,0.3185718084378124,0.0757558873307347,0.903,0.49534769392303246,1000 +Linear,Logistic,observational,False,1,0.95,0.9523333333333334,0.3796017230938245,0.0757558873307347,0.952,0.5441495204164779,1000 +Linear,Logistic,observational,False,4,0.9,0.41975,1.2508957207637406,0.7931627006785289,0.182,1.7859975260538024,1000 +Linear,Logistic,observational,False,4,0.95,0.5270833333333333,1.4905341854984016,0.7931627006785289,0.266,1.9972462513836364,1000 +Linear,Logistic,observational,False,6,0.9,0.8975833333333334,1.0291252957567814,0.2544578718137418,0.89,1.4849517269617925,1000 +Linear,Logistic,observational,False,6,0.95,0.9506666666666667,1.2262784251512802,0.2544578718137418,0.955,1.6571001913426326,1000 +Linear,Logistic,observational,True,1,0.9,0.90125,0.3166209527583751,0.07556902517081551,0.907,0.4925599895610136,1000 +Linear,Logistic,observational,True,1,0.95,0.9506666666666667,0.37727713517421785,0.07556902517081551,0.942,0.5411280924724183,1000 +Linear,Logistic,observational,True,4,0.9,0.416,1.2505883681999757,0.7923942748523936,0.194,1.7849545041218804,1000 +Linear,Logistic,observational,True,4,0.95,0.525,1.49016795232989,0.7923942748523936,0.281,1.996332789038424,1000 +Linear,Logistic,observational,True,6,0.9,0.8970833333333333,1.0227456972598339,0.2535415241660661,0.897,1.475471462269075,1000 +Linear,Logistic,observational,True,6,0.95,0.95025,1.2186766646754759,0.2535415241660661,0.955,1.6467072408407117,1000 diff --git a/results/did/did_multi_eventstudy.csv b/results/did/did_multi_eventstudy.csv index d0cc9a1..2241619 100644 --- a/results/did/did_multi_eventstudy.csv +++ b/results/did/did_multi_eventstudy.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.2652582159624413,0.6738426353849756,0.5274293504796588,0.09859154929577464,0.8819137878378639,213 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.35758998435054773,0.8029330239249745,0.5274293504796588,0.1596244131455399,1.0020805794352288,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.36619718309859156,0.5417379559244159,0.38469874476660076,0.1784037558685446,0.7377899281180879,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4561815336463224,0.6455205893536495,0.38469874476660076,0.2535211267605634,0.8314719135575573,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.8935837245696401,0.539297489044855,0.1350874946339115,0.8779342723004695,0.733879976113594,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.94679186228482,0.6426125937052664,0.1350874946339115,0.9577464788732394,0.8249185552960796,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.2566510172143975,0.6741169305030346,0.5271254414071088,0.07981220657276995,0.8840733793865773,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.352112676056338,0.8032598667173798,0.5271254414071088,0.14084507042253522,1.0029350081401456,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.3669796557120501,0.5416448273714012,0.38289493428958665,0.15492957746478872,0.7375321995926126,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4663536776212832,0.6454096198382773,0.38289493428958665,0.2347417840375587,0.8302756334680312,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8857589984350547,0.5394652594848104,0.13836010080710312,0.8497652582159625,0.7318025579590295,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9475743348982786,0.6428125045147116,0.13836010080710312,0.92018779342723,0.82720613326034,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.8896713615023474,2.6890092512865227,0.7200494209782025,0.92018779342723,3.7251683690668043,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9553990610328639,3.2041521508418644,0.7200494209782025,0.9624413145539906,4.1740665666747265,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9209702660406885,3.5465803488647247,0.9687050535779654,0.9765258215962441,4.86821272823518,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9733959311424101,4.226011140538671,0.9687050535779654,0.9906103286384976,5.460451299700878,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9397496087636932,2.016844429290545,0.447496108983214,0.9624413145539906,2.808950740031606,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9757433489827857,2.403218364880276,0.447496108983214,0.9906103286384976,3.1542968976841452,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9209702660406885,1.0931141425868476,0.2586324201487641,0.9295774647887324,1.5164351424033533,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9687010954616588,1.302525838990542,0.2586324201487641,0.9671361502347418,1.6959957409224138,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9420970266040688,1.3861070534921676,0.29683891288883923,0.9295774647887324,1.9076412479291383,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9726134585289514,1.6516484257609465,0.29683891288883923,0.971830985915493,2.1381249471795822,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9053208137715181,0.9470898644218079,0.22867843675954816,0.8873239436619719,1.3244262222504375,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9616588419405321,1.1285271795460685,0.22867843675954816,0.9671361502347418,1.483381487672127,213 -Linear,Logistic,experimental,False,1,0.9,0.8395931142410016,0.20961349451394826,0.06166699387236784,0.7323943661971831,0.2990383792423722,213 -Linear,Logistic,experimental,False,1,0.95,0.9006259780907668,0.24976988419471297,0.06166699387236784,0.8262910798122066,0.3323039576385979,213 -Linear,Logistic,experimental,False,4,0.9,0.2136150234741784,0.9759976682205357,0.9530895482664571,0.046948356807511735,1.257805513435992,213 -Linear,Logistic,experimental,False,4,0.95,0.2683881064162754,1.1629729523426826,0.9530895482664571,0.06103286384976526,1.433457694555575,213 -Linear,Logistic,experimental,False,6,0.9,0.888888888888889,0.984101871756088,0.2529040307014624,0.8544600938967136,1.2672750485116713,213 -Linear,Logistic,experimental,False,6,0.95,0.9311424100156495,1.172629706471318,0.2529040307014624,0.9389671361502347,1.4397703898140197,213 -Linear,Logistic,experimental,True,1,0.9,0.838810641627543,0.20962019115150052,0.06163754557151296,0.7370892018779343,0.2991822645495156,213 -Linear,Logistic,experimental,True,1,0.95,0.9006259780907668,0.24977786373052385,0.06163754557151296,0.8309859154929577,0.33245309146308927,213 -Linear,Logistic,experimental,True,4,0.9,0.2167449139280125,0.9760619360265147,0.952817028222668,0.046948356807511735,1.2609463601610145,213 -Linear,Logistic,experimental,True,4,0.95,0.27230046948356806,1.1630495321568497,0.952817028222668,0.06572769953051644,1.4322602770975743,213 -Linear,Logistic,experimental,True,6,0.9,0.8834115805946792,0.983811761600466,0.2527027570120268,0.8497652582159625,1.265876521888396,213 -Linear,Logistic,experimental,True,6,0.95,0.9358372456964007,1.172284018899335,0.2527027570120268,0.9389671361502347,1.4415267063053059,213 -Linear,Logistic,observational,False,1,0.9,0.9100156494522692,0.2250881504094304,0.0527385366052549,0.9389671361502347,0.32114042199578385,213 -Linear,Logistic,observational,False,1,0.95,0.960093896713615,0.26820907399940563,0.0527385366052549,0.9624413145539906,0.3565208721390806,213 -Linear,Logistic,observational,False,4,0.9,0.3317683881064163,1.2904625746951797,0.929438804907565,0.18779342723004694,1.6422845737610774,213 -Linear,Logistic,observational,False,4,0.95,0.42018779342723006,1.5376809999118553,0.929438804907565,0.27699530516431925,1.8716546886192966,213 -Linear,Logistic,observational,False,6,0.9,0.8818466353677622,1.0252410978950457,0.26737451665728457,0.8544600938967136,1.317569321483734,213 -Linear,Logistic,observational,False,6,0.95,0.9327073552425664,1.2216501179310573,0.26737451665728457,0.92018779342723,1.5020723454442884,213 -Linear,Logistic,observational,True,1,0.9,0.9123630672926448,0.22389841311304975,0.052297945992543005,0.9248826291079812,0.31905149827680773,213 -Linear,Logistic,observational,True,1,0.95,0.9585289514866979,0.2667914145713799,0.052297945992543005,0.9765258215962441,0.3542427529311309,213 -Linear,Logistic,observational,True,4,0.9,0.3270735524256651,1.2929917075582165,0.9243991887988946,0.20187793427230047,1.64687629811095,213 -Linear,Logistic,observational,True,4,0.95,0.41784037558685444,1.54069464759603,0.9243991887988946,0.26291079812206575,1.8776567708863852,213 -Linear,Logistic,observational,True,6,0.9,0.8755868544600939,1.019446010780053,0.2700327354611733,0.8497652582159625,1.308225574447524,213 -Linear,Logistic,observational,True,6,0.95,0.9280125195618153,1.214744845725342,0.2700327354611733,0.92018779342723,1.4918494896646508,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.212,0.6604863947299484,0.5680925390085144,0.03,0.8639202587233566,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.2853333333333333,0.787018081571581,0.5680925390085144,0.073,0.9797059963406656,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.24783333333333332,0.6059094348361568,0.5434696918979384,0.06,0.8130147435733623,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.32,0.7219856227407174,0.5434696918979384,0.102,0.9183163527410149,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.897,0.6023951572598649,0.14911519055090663,0.896,0.80773077576638,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9485,0.7177981027277819,0.14911519055090663,0.944,0.9122884478790374,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.216,0.6603511092788719,0.568539302268738,0.034,0.8638114270473408,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.28983333333333333,0.7868568790138594,0.568539302268738,0.073,0.9802312306197214,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.24733333333333332,0.6060215144180863,0.5437755527978768,0.064,0.8139649562039803,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.3206666666666667,0.7221191737998417,0.5437755527978768,0.105,0.918312219671443,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8961666666666667,0.6024262845726389,0.149332789476576,0.895,0.807525870642016,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9488333333333334,0.7178351932086445,0.149332789476576,0.942,0.9117009348167522,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9585,0.8182729709439571,0.16565056617253612,0.958,1.1056917522246983,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9826666666666666,0.9750323836079928,0.16565056617253612,0.983,1.2457725065304077,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.937,1.1923984653336688,0.2434226455534901,0.929,1.5772729726576005,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9705,1.4208304064151072,0.2434226455534901,0.969,1.7864645790746925,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9405,0.7715719701005695,0.16570390033287308,0.943,1.0366837950276395,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9736666666666667,0.9193847088269498,0.16570390033287308,0.975,1.1696730766341457,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9435,0.7351138303781443,0.15288401708102373,0.953,0.9974445142927548,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9776666666666667,0.8759421558675607,0.15288401708102373,0.98,1.1224659600314153,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.8981666666666667,1.048606227423576,0.24125031716387346,0.882,1.3932585328634872,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9471666666666666,1.2494913869775361,0.24125031716387346,0.934,1.5755625535278197,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9265,0.707833153098743,0.1584727972282905,0.932,0.9562892376447345,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9656666666666667,0.8434352239039034,0.1584727972282905,0.971,1.0771546668822336,1000 +Linear,Logistic,experimental,False,1,0.9,0.8333333333333334,0.21008119266299188,0.06279250993648063,0.772,0.2997763455348952,1000 +Linear,Logistic,experimental,False,1,0.95,0.898,0.25032718091263456,0.06279250993648063,0.852,0.3330657871692049,1000 +Linear,Logistic,experimental,False,4,0.9,0.186,0.9725081430802559,0.9487602256636943,0.038,1.254980282567084,1000 +Linear,Logistic,experimental,False,4,0.95,0.2495,1.1588149266764283,0.9487602256636943,0.065,1.4284992957761715,1000 +Linear,Logistic,experimental,False,6,0.9,0.9058333333333334,0.9844631950664593,0.2397840052146586,0.905,1.2664633333872184,1000 +Linear,Logistic,experimental,False,6,0.95,0.9588333333333334,1.1730602497509743,0.2397840052146586,0.958,1.4417094501447543,1000 +Linear,Logistic,experimental,True,1,0.9,0.8348333333333333,0.21008310515619139,0.06274327271440339,0.766,0.2999472120228887,1000 +Linear,Logistic,experimental,True,1,0.95,0.8965,0.2503294597888399,0.06274327271440339,0.851,0.3328788977719515,1000 +Linear,Logistic,experimental,True,4,0.9,0.1855,0.9724386030694512,0.9484794587569109,0.04,1.2546597764324676,1000 +Linear,Logistic,experimental,True,4,0.95,0.24766666666666665,1.158732064642732,0.9484794587569109,0.067,1.4275114494639556,1000 +Linear,Logistic,experimental,True,6,0.9,0.906,0.9845132322822716,0.2399237885684876,0.906,1.2667892152063878,1000 +Linear,Logistic,experimental,True,6,0.95,0.959,1.173119872771085,0.2399237885684876,0.959,1.4415012389600717,1000 +Linear,Logistic,observational,False,1,0.9,0.9121666666666667,0.2263188173897403,0.05316466170010902,0.908,0.3227959863736354,1000 +Linear,Logistic,observational,False,1,0.95,0.9536666666666667,0.26967550415394803,0.05316466170010902,0.964,0.35842709635415104,1000 +Linear,Logistic,observational,False,4,0.9,0.308,1.3046529273340777,0.9246466619438996,0.17,1.658910870414429,1000 +Linear,Logistic,observational,False,4,0.95,0.41483333333333333,1.5545898479968427,0.9246466619438996,0.26,1.8916048826034793,1000 +Linear,Logistic,observational,False,6,0.9,0.8995,1.0353589791850297,0.2578329738577894,0.909,1.3296442260169248,1000 +Linear,Logistic,observational,False,6,0.95,0.9556666666666667,1.2337063170987448,0.2578329738577894,0.954,1.5137160905920766,1000 +Linear,Logistic,observational,True,1,0.9,0.9068333333333334,0.22511549914119275,0.053178118920023186,0.913,0.32092353183996714,1000 +Linear,Logistic,observational,True,1,0.95,0.9531666666666666,0.2682416620232874,0.053178118920023186,0.962,0.3568312746703429,1000 +Linear,Logistic,observational,True,4,0.9,0.3075,1.3070131076837166,0.9238327987989788,0.175,1.658886675275069,1000 +Linear,Logistic,observational,True,4,0.95,0.4135,1.5574021763441892,0.9238327987989788,0.273,1.8945931482242002,1000 +Linear,Logistic,observational,True,6,0.9,0.8951666666666667,1.0276881926550976,0.25674529262721113,0.912,1.3196141537282695,1000 +Linear,Logistic,observational,True,6,0.95,0.9548333333333334,1.2245660111861596,0.25674529262721113,0.955,1.50455977788913,1000 diff --git a/results/did/did_multi_group.csv b/results/did/did_multi_group.csv index 93cf706..378a6a5 100644 --- a/results/did/did_multi_group.csv +++ b/results/did/did_multi_group.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.3615023474178404,0.7202622844804144,0.5142431175686811,0.09389671361502347,0.8960068730292109,213 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.4522691705790297,0.8582454474204736,0.5142431175686811,0.16901408450704225,1.0218690905313046,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.4585289514866979,0.6080965137285453,0.37995101909660545,0.1784037558685446,0.7737228418008084,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.568075117370892,0.72459168797971,0.37995101909660545,0.23943661971830985,0.8761317466502612,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9107981220657277,0.6041693800921251,0.1487229003064066,0.892018779342723,0.768532913195283,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9577464788732394,0.7199122196283984,0.1487229003064066,0.9389671361502347,0.8712620903971866,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.3599374021909233,0.7202694109877334,0.5144563506346828,0.09859154929577464,0.896154594874867,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.43974960876369323,0.8582539391777047,0.5144563506346828,0.13615023474178403,1.0209397722371372,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.4726134585289514,0.6080786219858845,0.3781886928978884,0.18309859154929578,0.7737589269691583,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.5758998435054773,0.7245703686533813,0.3781886928978884,0.2347417840375587,0.8776184255890188,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8967136150234741,0.6043541483976993,0.15369943111934065,0.9061032863849765,0.7688492385342756,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9483568075117371,0.7201323846439818,0.15369943111934065,0.9436619718309859,0.8710978723491863,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9014084507042254,2.7211728776223616,0.7210652400496165,0.9389671361502347,3.4325568651778737,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9640062597809077,3.2424774754771537,0.7210652400496165,0.9765258215962441,3.90620301995042,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9280125195618153,3.5331758180056396,0.9482156709536594,0.9530516431924883,4.456779474115103,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.9812206572769953,4.210038656858068,0.9482156709536594,0.9906103286384976,5.050979601042688,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9593114241001566,2.1134207943166583,0.4426633274641162,0.9671361502347418,2.68416063766941,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9890453834115805,2.518296201660071,0.4426633274641162,1.0,3.0441389382496644,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9311424100156495,1.1368385210095147,0.2642736374124477,0.9577464788732394,1.445045473376009,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.971830985915493,1.3546266493913184,0.2642736374124477,0.9906103286384976,1.6350299884984196,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9358372456964007,1.4316645897561893,0.3075262798142372,0.9295774647887324,1.812614988818496,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9687010954616588,1.7059335784570864,0.3075262798142372,0.9530516431924883,2.0542997322706174,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.92018779342723,1.008702982773344,0.23956099456087157,0.9295774647887324,1.2878304500066007,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9655712050078247,1.2019437382997051,0.23956099456087157,0.9671361502347418,1.4556619642167887,213 -Linear,Logistic,experimental,False,1,0.9,0.8466353677621283,0.26379623317984124,0.07367574399457054,0.812206572769953,0.33886769227656965,213 -Linear,Logistic,experimental,False,1,0.95,0.917057902973396,0.3143325994593638,0.07367574399457054,0.8873239436619719,0.3826093895409892,213 -Linear,Logistic,experimental,False,4,0.9,0.31768388106416273,1.081000965905216,0.9209342612568459,0.046948356807511735,1.3612059586767777,213 -Linear,Logistic,experimental,False,4,0.95,0.406885758998435,1.2880920987200657,0.9209342612568459,0.07042253521126761,1.5495796948573959,213 -Linear,Logistic,experimental,False,6,0.9,0.8841940532081377,1.0864109362400156,0.2793222944837174,0.8826291079812206,1.3646379214251372,213 -Linear,Logistic,experimental,False,6,0.95,0.9420970266040688,1.2945384759781382,0.2793222944837174,0.9436619718309859,1.5555664326647642,213 -Linear,Logistic,experimental,True,1,0.9,0.8482003129890453,0.26383019814224784,0.07370017470086833,0.7981220657276995,0.33938445004816475,213 -Linear,Logistic,experimental,True,1,0.95,0.917057902973396,0.314373071208316,0.07370017470086833,0.8779342723004695,0.38255405924208785,213 -Linear,Logistic,experimental,True,4,0.9,0.3145539906103286,1.0811145595227114,0.9206314437246235,0.046948356807511735,1.3629136590606885,213 -Linear,Logistic,experimental,True,4,0.95,0.406885758998435,1.288227453863841,0.9206314437246235,0.07042253521126761,1.5491777387798102,213 -Linear,Logistic,experimental,True,6,0.9,0.8763693270735524,1.0860190707615656,0.2799723827071057,0.8732394366197183,1.3678468139137758,213 -Linear,Logistic,experimental,True,6,0.95,0.9436619718309859,1.2940715394605282,0.2799723827071057,0.9436619718309859,1.5543990432787684,213 -Linear,Logistic,observational,False,1,0.9,0.9233176838810642,0.28345104003446464,0.06577425283822312,0.9154929577464789,0.36397364538854327,213 -Linear,Logistic,observational,False,1,0.95,0.9640062597809077,0.3377527463508229,0.06577425283822312,0.9577464788732394,0.41153782693535657,213 -Linear,Logistic,observational,False,4,0.9,0.41001564945226915,1.3821730458167794,0.9084755331600975,0.22065727699530516,1.7301013537520684,213 -Linear,Logistic,observational,False,4,0.95,0.5320813771517997,1.6469607664869994,0.9084755331600975,0.3145539906103286,1.9685722460940824,213 -Linear,Logistic,observational,False,6,0.9,0.8904538341158059,1.1242898926223517,0.2899194075787034,0.9061032863849765,1.4121558358228579,213 -Linear,Logistic,observational,False,6,0.95,0.9436619718309859,1.3396740364103084,0.2899194075787034,0.9436619718309859,1.6083520009872296,213 -Linear,Logistic,observational,True,1,0.9,0.9264475743348983,0.28168330654961965,0.06531180649498958,0.92018779342723,0.36242423041318955,213 -Linear,Logistic,observational,True,1,0.95,0.9640062597809077,0.3356463619845842,0.06531180649498958,0.9624413145539906,0.4088457694027527,213 -Linear,Logistic,observational,True,4,0.9,0.41471048513302033,1.382459197707508,0.905500217221854,0.20187793427230047,1.7291824258918072,213 -Linear,Logistic,observational,True,4,0.95,0.5164319248826291,1.6473017374954506,0.905500217221854,0.29577464788732394,1.9692425330349306,213 -Linear,Logistic,observational,True,6,0.9,0.8763693270735524,1.1216619887490014,0.29425840904927425,0.8967136150234741,1.405222302167041,213 -Linear,Logistic,observational,True,6,0.95,0.9405320813771518,1.336542695808199,0.29425840904927425,0.9389671361502347,1.599301420149991,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.3273333333333333,0.707420656396312,0.5512047367255486,0.036,0.8797234045148201,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.41,0.8429437037666345,0.5512047367255486,0.074,1.0041771175059788,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.374,0.6782674902962976,0.5244026247767223,0.071,0.8610132839087776,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.4453333333333333,0.8082055637552085,0.5244026247767223,0.117,0.976065830250241,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.899,0.671827470630062,0.16624920009099023,0.902,0.8525368328187772,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9473333333333334,0.8005318070155615,0.16624920009099023,0.951,0.966016106037394,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.326,0.7072412761451377,0.5510566287884651,0.038,0.8801404707275715,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.4096666666666666,0.8427299590138623,0.5510566287884651,0.07,1.002125974085047,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.36866666666666664,0.6784182431926686,0.5259144954763614,0.072,0.8610207509558818,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.4496666666666666,0.8083851969107737,0.5259144954763614,0.119,0.9763798979720331,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.894,0.6718234690242728,0.1657257952040637,0.896,0.8515844126182411,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9476666666666667,0.800527038808167,0.1657257952040637,0.95,0.9665334176962624,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.963,0.8652755981021423,0.17113142201133558,0.969,1.0889358534995506,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.984,1.031039468311053,0.17113142201133558,0.984,1.237297042745351,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9346666666666666,1.2284508258805134,0.24656685966850245,0.941,1.5361413271219306,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.972,1.463789443664172,0.24656685966850245,0.971,1.749648191744939,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9383333333333334,0.8377546803078038,0.1751631649589318,0.953,1.057109034491574,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.975,0.9982462721174428,0.1751631649589318,0.984,1.2014689940380152,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9496666666666667,0.7841947843278112,0.16086506631076952,0.956,0.9891207416675105,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.981,0.9344257196886802,0.16086506631076952,0.982,1.1249345196702958,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9063333333333333,1.0936611276605492,0.24761334359753662,0.893,1.3734409007728312,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9496666666666667,1.3031776119063627,0.24761334359753662,0.957,1.5621218569040163,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.93,0.7753642730061224,0.17208754196605122,0.938,0.9822835747368598,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.968,0.9239035164530885,0.17208754196605122,0.97,1.114292213724501,1000 +Linear,Logistic,experimental,False,1,0.9,0.8386666666666667,0.26406011408190466,0.07484251880343806,0.782,0.3391725273742781,1000 +Linear,Logistic,experimental,False,1,0.95,0.9,0.3146470329480206,0.07484251880343806,0.861,0.38312548180720674,1000 +Linear,Logistic,experimental,False,4,0.9,0.30766666666666664,1.0775081920199379,0.9180199075797375,0.043,1.358394713485805,1000 +Linear,Logistic,experimental,False,4,0.95,0.387,1.2839302019353809,0.9180199075797375,0.075,1.5436882060371362,1000 +Linear,Logistic,experimental,False,6,0.9,0.9006666666666666,1.0862296392061837,0.26426326890215074,0.901,1.3668444179732824,1000 +Linear,Logistic,experimental,False,6,0.95,0.9506666666666667,1.2943224472379558,0.26426326890215074,0.953,1.5536493498793789,1000 +Linear,Logistic,experimental,True,1,0.9,0.8393333333333334,0.26407759152816007,0.07480194343006176,0.78,0.339074181745231,1000 +Linear,Logistic,experimental,True,1,0.95,0.9026666666666666,0.3146678586097337,0.07480194343006176,0.856,0.38322152907222656,1000 +Linear,Logistic,experimental,True,4,0.9,0.30866666666666664,1.0774440156012468,0.9177015030168479,0.042,1.358470637092642,1000 +Linear,Logistic,experimental,True,4,0.95,0.3873333333333333,1.2838537310158835,0.9177015030168479,0.077,1.5429798338746872,1000 +Linear,Logistic,experimental,True,6,0.9,0.9,1.0863837801957064,0.26380863451040953,0.9,1.3672808159070202,1000 +Linear,Logistic,experimental,True,6,0.95,0.9513333333333334,1.2945061175555181,0.26380863451040953,0.954,1.5541073384842106,1000 +Linear,Logistic,observational,False,1,0.9,0.9076666666666666,0.28416293869816867,0.06719907904707288,0.895,0.3651157807139984,1000 +Linear,Logistic,observational,False,1,0.95,0.9483333333333334,0.3386010259294065,0.06719907904707288,0.945,0.41201437318485834,1000 +Linear,Logistic,observational,False,4,0.9,0.401,1.391585982365,0.904387305913916,0.19,1.740406175908275,1000 +Linear,Logistic,observational,False,4,0.95,0.5136666666666666,1.6581769721851722,0.904387305913916,0.283,1.9826656824817435,1000 +Linear,Logistic,observational,False,6,0.9,0.8973333333333333,1.1352802141544187,0.27975950279202133,0.905,1.4266992538777443,1000 +Linear,Logistic,observational,False,6,0.95,0.9503333333333334,1.3527698122461733,0.27975950279202133,0.955,1.6225397076843175,1000 +Linear,Logistic,observational,True,1,0.9,0.903,0.2824293492396861,0.06705110860806944,0.898,0.3627882621194592,1000 +Linear,Logistic,observational,True,1,0.95,0.946,0.33653532667998365,0.06705110860806944,0.942,0.40929988535575634,1000 +Linear,Logistic,observational,True,4,0.9,0.402,1.3915312960755781,0.9034692492203523,0.193,1.7414149632974412,1000 +Linear,Logistic,observational,True,4,0.95,0.5146666666666666,1.6581118094521734,0.9034692492203523,0.286,1.9845340670655314,1000 +Linear,Logistic,observational,True,6,0.9,0.8936666666666666,1.1270129557072197,0.27813843488385187,0.905,1.4156442615210005,1000 +Linear,Logistic,observational,True,6,0.95,0.9483333333333334,1.3429187662066389,0.27813843488385187,0.952,1.6098259360653087,1000 diff --git a/results/did/did_multi_metadata.csv b/results/did/did_multi_metadata.csv index eddd880..28b7a8c 100644 --- a/results/did/did_multi_metadata.csv +++ b/results/did/did_multi_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.11.dev0,DIDMultiCoverageSimulation,2025-06-06 11:16,337.9197948932648,3.12.3,scripts/did/did_pa_multi_config.yml +0.10.0,DIDMultiCoverageSimulation,2025-06-06 08:13,60.66274849573771,3.12.9,scripts/did/did_pa_multi_config.yml diff --git a/results/did/did_multi_time.csv b/results/did/did_multi_time.csv index c2b076e..166a58b 100644 --- a/results/did/did_multi_time.csv +++ b/results/did/did_multi_time.csv @@ -1,49 +1,49 @@ Learner g,Learner m,Score,In-sample-norm.,DGP,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition -LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.1111111111111111,0.6848879510567563,0.5814212808822293,0.06572769953051644,0.8112788259677559,213 -LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.19092331768388104,0.8160943293200865,0.5814212808822293,0.1596244131455399,0.9363347602930319,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.2190923317683881,0.5442904416410593,0.42804406962185293,0.15023474178403756,0.6610129830707445,213 -LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.3114241001564945,0.6485620636792077,0.42804406962185293,0.2112676056338028,0.7594820835871338,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.9076682316118936,0.540281441005603,0.13166144694063336,0.8967136150234741,0.6574061190218973,213 -LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9530516431924883,0.6437850447817539,0.13166144694063336,0.9436619718309859,0.755440248127496,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.10641627543035993,0.6849270317496503,0.5816182941099456,0.07042253521126761,0.810739883430405,213 -LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.17527386541471046,0.8161408968378937,0.5816182941099456,0.13145539906103287,0.9369536425675097,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.2190923317683881,0.5441327590803698,0.4271332871560829,0.15492957746478872,0.659742375492321,213 -LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.29733959311424096,0.6483741733193132,0.4271332871560829,0.20657276995305165,0.7581397106264711,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.8982785602503913,0.5400972159018558,0.1320306660672212,0.892018779342723,0.65734442843056,213 -LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9546165884194053,0.6435655270310701,0.1320306660672212,0.9248826291079812,0.7544822430389665,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.863849765258216,2.947691714934299,0.8081717452002422,0.9014084507042254,3.6345410887622425,213 -LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9483568075117371,3.5123913180688033,0.8081717452002422,0.9530516431924883,4.148298978134212,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9107981220657277,3.920449689676996,1.0987394652573248,0.9342723004694836,4.803568258598088,213 -LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.974960876369327,4.671503937532334,1.0987394652573248,1.0,5.490527294034555,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.9452269170579031,1.9822578095421914,0.41440886099461754,0.9765258215962441,2.4747132887427976,213 -LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9843505477308294,2.362005865516796,0.41440886099461754,0.9906103286384976,2.82160571997147,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9139280125195618,1.1523823561555302,0.27480514203439516,0.9295774647887324,1.4209065729930848,213 -LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.9733959311424101,1.37314827135733,0.27480514203439516,0.9765258215962441,1.6207292567271263,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.9295774647887324,1.4768427642563355,0.3193093946805207,0.92018779342723,1.8030473738560802,213 -LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9655712050078247,1.7597666937304888,0.3193093946805207,0.971830985915493,2.066811584773276,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9029733959311425,0.9274870427023983,0.22082805477342204,0.92018779342723,1.1562170046781757,213 -LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9593114241001566,1.1051689767638488,0.22082805477342204,0.971830985915493,1.3147550346737336,213 -Linear,Logistic,experimental,False,1,0.9,0.8231611893583725,0.24405699708437717,0.07177094325289468,0.7746478873239436,0.3128134096194603,213 -Linear,Logistic,experimental,False,1,0.95,0.895148669796557,0.290811848922341,0.07177094325289468,0.8685446009389671,0.3532819977654326,213 -Linear,Logistic,experimental,False,4,0.9,0.03912363067292645,0.9696140468983949,1.089724580728028,0.028169014084507043,1.112835689831825,213 -Linear,Logistic,experimental,False,4,0.95,0.0594679186228482,1.1553663983750042,1.089724580728028,0.056338028169014086,1.2908015410381681,213 -Linear,Logistic,experimental,False,6,0.9,0.8841940532081377,0.9637145816970751,0.24601437794174566,0.8591549295774648,1.1098958953800837,213 -Linear,Logistic,experimental,False,6,0.95,0.9420970266040688,1.1483367520081937,0.24601437794174566,0.92018779342723,1.2882569853203092,213 -Linear,Logistic,experimental,True,1,0.9,0.8294209702660407,0.24407331778379834,0.07187555926470358,0.755868544600939,0.31239740640105534,213 -Linear,Logistic,experimental,True,1,0.95,0.8904538341158059,0.29083129623518633,0.07187555926470358,0.8685446009389671,0.35340605322083257,213 -Linear,Logistic,experimental,True,4,0.9,0.03755868544600939,0.9698402059419257,1.089909708050757,0.028169014084507043,1.1118287534726101,213 -Linear,Logistic,experimental,True,4,0.95,0.06572769953051644,1.1556358834968625,1.089909708050757,0.056338028169014086,1.2947346339601935,213 -Linear,Logistic,experimental,True,6,0.9,0.8857589984350547,0.963597334445951,0.2461020735514813,0.8732394366197183,1.1092556702862066,213 -Linear,Logistic,experimental,True,6,0.95,0.9420970266040688,1.1481970432914277,0.2461020735514813,0.9295774647887324,1.288313460888964,213 -Linear,Logistic,observational,False,1,0.9,0.9092331768388106,0.2738536200757641,0.0653695842721706,0.9107981220657277,0.35111788102252456,213 -Linear,Logistic,observational,False,1,0.95,0.9577464788732394,0.32631671511051,0.0653695842721706,0.9577464788732394,0.39595800514716334,213 -Linear,Logistic,observational,False,4,0.9,0.17214397496087636,1.352533603300557,1.0665740251760334,0.15023474178403756,1.523002565481769,213 -Linear,Logistic,observational,False,4,0.95,0.26134585289514867,1.611643192386921,1.0665740251760334,0.22535211267605634,1.7732161740973758,213 -Linear,Logistic,observational,False,6,0.9,0.8857589984350547,1.0021244212459983,0.25876871605083246,0.8591549295774648,1.1510136525260573,213 -Linear,Logistic,observational,False,6,0.95,0.9389671361502347,1.1941048987504526,0.25876871605083246,0.92018779342723,1.3383058201805063,213 -Linear,Logistic,observational,True,1,0.9,0.9123630672926448,0.2715453775356287,0.0644928980540172,0.9154929577464789,0.3477637744633791,213 -Linear,Logistic,observational,True,1,0.95,0.9608763693270735,0.32356627448034075,0.0644928980540172,0.9624413145539906,0.3936582053408612,213 -Linear,Logistic,observational,True,4,0.9,0.17683881064162754,1.3541054412513556,1.0639801560397304,0.15023474178403756,1.5236529859485555,213 -Linear,Logistic,observational,True,4,0.95,0.24413145539906103,1.6135161528270596,1.0639801560397304,0.2112676056338028,1.7775778058708633,213 -Linear,Logistic,observational,True,6,0.9,0.8763693270735524,0.9971323968875062,0.2594135097814355,0.8732394366197183,1.1473737786354752,213 -Linear,Logistic,observational,True,6,0.95,0.9327073552425664,1.1881565348399659,0.2594135097814355,0.9154929577464789,1.33540506645655,213 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.9,0.055,0.6712569909941205,0.6346334487258358,0.035,0.7933048784144286,1000 +LGBM Regr.,LGBM Clas.,experimental,False,1,0.95,0.10733333333333332,0.7998520386021065,0.6346334487258358,0.061,0.9159058461680386,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.9,0.075,0.6060202682055259,0.6100618498175898,0.052,0.7251643151081226,1000 +LGBM Regr.,LGBM Clas.,experimental,False,4,0.95,0.13,0.7221176888459861,0.6100618498175898,0.097,0.8353173376691023,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.9,0.901,0.5988208500851633,0.1464689152752021,0.897,0.7188116426660323,1000 +LGBM Regr.,LGBM Clas.,experimental,False,6,0.95,0.9483333333333334,0.7135390530364837,0.1464689152752021,0.95,0.827434201860584,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.9,0.054,0.6710988857611525,0.6356662481793647,0.03,0.7928754344366965,1000 +LGBM Regr.,LGBM Clas.,experimental,True,1,0.95,0.09766666666666665,0.7996636445971279,0.6356662481793647,0.064,0.9157288402316642,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.9,0.08,0.6061420688480603,0.611475967305118,0.051,0.7250804471897608,1000 +LGBM Regr.,LGBM Clas.,experimental,True,4,0.95,0.12766666666666665,0.7222628232632679,0.611475967305118,0.097,0.8356164365834543,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.9,0.9003333333333333,0.5988274535253045,0.1470455937405394,0.897,0.7193380504386929,1000 +LGBM Regr.,LGBM Clas.,experimental,True,6,0.95,0.9466666666666667,0.7135469215207298,0.1470455937405394,0.939,0.8266210501795132,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.9,0.9633333333333334,0.8625181197134913,0.1705040132107199,0.972,1.045848740623199,1000 +LGBM Regr.,LGBM Clas.,observational,False,1,0.95,0.9886666666666666,1.0277537301509225,0.1705040132107199,0.989,1.2001073696975533,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.9,0.9353333333333333,1.2945843278468319,0.26162576104250007,0.923,1.5341055868187743,1000 +LGBM Regr.,LGBM Clas.,observational,False,4,0.95,0.968,1.542592371719069,0.26162576104250007,0.965,1.7716810296642407,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.9,0.947,0.7672044714361391,0.16043039243421475,0.948,0.9284201701025314,1000 +LGBM Regr.,LGBM Clas.,observational,False,6,0.95,0.9763333333333334,0.9141805131802676,0.16043039243421475,0.973,1.067043785578332,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.9,0.9506666666666667,0.7684651194719102,0.1600966845306246,0.961,0.9322911708810523,1000 +LGBM Regr.,LGBM Clas.,observational,True,1,0.95,0.982,0.9156826679658406,0.1600966845306246,0.978,1.0708669848118426,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.9,0.887,1.1247146971381123,0.25811288498674895,0.868,1.3337069451804375,1000 +LGBM Regr.,LGBM Clas.,observational,True,4,0.95,0.9386666666666666,1.3401802222117185,0.25811288498674895,0.915,1.539509702377364,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.9,0.9233333333333333,0.7013549950930801,0.15406953919220856,0.931,0.8502851669291526,1000 +LGBM Regr.,LGBM Clas.,observational,True,6,0.95,0.9673333333333334,0.835716022529863,0.15406953919220856,0.967,0.9757420279803587,1000 +Linear,Logistic,experimental,False,1,0.9,0.8113333333333334,0.24421066998125654,0.07335238125159724,0.763,0.31287636754980536,1000 +Linear,Logistic,experimental,False,1,0.95,0.885,0.2909949614731166,0.07335238125159724,0.844,0.35353712763106526,1000 +Linear,Logistic,experimental,False,4,0.9,0.03866666666666666,0.9657093192836849,1.0744747295532333,0.034,1.1073798978939535,1000 +Linear,Logistic,experimental,False,4,0.95,0.069,1.1507136284452841,1.0744747295532333,0.055,1.2858869628367815,1000 +Linear,Logistic,experimental,False,6,0.9,0.9016666666666666,0.9646204823976587,0.23468919497366172,0.901,1.1093098237657892,1000 +Linear,Logistic,experimental,False,6,0.95,0.953,1.1494161992718415,0.23468919497366172,0.957,1.2871233271513411,1000 +Linear,Logistic,experimental,True,1,0.9,0.8133333333333334,0.2442088644035013,0.07330749456217257,0.77,0.3129912931800666,1000 +Linear,Logistic,experimental,True,1,0.95,0.8853333333333334,0.2909928099945208,0.07330749456217257,0.845,0.35370879380082904,1000 +Linear,Logistic,experimental,True,4,0.9,0.040333333333333325,0.965669466190919,1.074178736623636,0.034,1.1079673576193978,1000 +Linear,Logistic,experimental,True,4,0.95,0.07066666666666667,1.1506661405562622,1.074178736623636,0.056,1.2874892975542063,1000 +Linear,Logistic,experimental,True,6,0.9,0.905,0.9646743300945927,0.23461429065167058,0.903,1.1086934199469425,1000 +Linear,Logistic,experimental,True,6,0.95,0.9513333333333334,1.1494803627602588,0.23461429065167058,0.954,1.288778040856994,1000 +Linear,Logistic,observational,False,1,0.9,0.8963333333333334,0.27432665056823285,0.06662446847374676,0.892,0.3514300281214646,1000 +Linear,Logistic,observational,False,1,0.95,0.947,0.3268803657075217,0.06662446847374676,0.94,0.3973633331155568,1000 +Linear,Logistic,observational,False,4,0.9,0.16966666666666666,1.3719614152655681,1.057702662307478,0.138,1.5420603037012326,1000 +Linear,Logistic,observational,False,4,0.95,0.2533333333333333,1.6347928581844855,1.057702662307478,0.227,1.7997355197003797,1000 +Linear,Logistic,observational,False,6,0.9,0.893,1.0148202381932496,0.25145709760552504,0.896,1.1671820785014646,1000 +Linear,Logistic,observational,False,6,0.95,0.952,1.2092328977183877,0.25145709760552504,0.948,1.3556655852353603,1000 +Linear,Logistic,observational,True,1,0.9,0.8956666666666666,0.2722754365826716,0.06632302110410243,0.882,0.34896138602531507,1000 +Linear,Logistic,observational,True,1,0.95,0.945,0.3244361934903646,0.06632302110410243,0.935,0.3940464937357552,1000 +Linear,Logistic,observational,True,4,0.9,0.16866666666666666,1.3721900166602485,1.0557853739176288,0.144,1.5416213892347916,1000 +Linear,Logistic,observational,True,4,0.95,0.258,1.6350652535472388,1.0557853739176288,0.226,1.8003166158280433,1000 +Linear,Logistic,observational,True,6,0.9,0.893,1.0066884879769613,0.25076936355127627,0.887,1.1578970654476302,1000 +Linear,Logistic,observational,True,6,0.95,0.949,1.19954331969512,0.25076936355127627,0.942,1.3460255801756882,1000 diff --git a/results/did/did_pa_multi_config.yml b/results/did/did_pa_multi_config.yml index ed4e23a..7ef23dd 100644 --- a/results/did/did_pa_multi_config.yml +++ b/results/did/did_pa_multi_config.yml @@ -18,13 +18,29 @@ learner_definitions: lgbmr: &id003 name: LGBM Regr. params: - n_estimators: 500 - learning_rate: 0.02 + n_estimators: 300 + learning_rate: 0.03 + num_leaves: 7 + max_depth: 3 + min_child_samples: 20 + subsample: 0.8 + colsample_bytree: 0.8 + reg_alpha: 0.1 + reg_lambda: 1.0 + random_state: 42 lgbmc: &id004 name: LGBM Clas. params: - n_estimators: 500 - learning_rate: 0.02 + n_estimators: 300 + learning_rate: 0.03 + num_leaves: 7 + max_depth: 3 + min_child_samples: 20 + subsample: 0.8 + colsample_bytree: 0.8 + reg_alpha: 0.1 + reg_lambda: 1.0 + random_state: 42 dml_parameters: learners: - ml_g: *id001 diff --git a/scripts/did/did_pa_multi_config.yml b/scripts/did/did_pa_multi_config.yml index c89ef8d..ad33a5d 100644 --- a/scripts/did/did_pa_multi_config.yml +++ b/scripts/did/did_pa_multi_config.yml @@ -21,14 +21,30 @@ learner_definitions: lgbmr: &lgbmr name: "LGBM Regr." params: - n_estimators: 500 - learning_rate: 0.02 + n_estimators: 300 # More trees to learn slowly and steadily + learning_rate: 0.03 # Lower learning rate to improve generalization + num_leaves: 7 # Fewer leaves — simpler trees + max_depth: 3 # Shallow trees reduce overfitting + min_child_samples: 20 # Require more samples per leaf + subsample: 0.8 # More row sampling to add randomness + colsample_bytree: 0.8 # More feature sampling + reg_alpha: 0.1 # Add L1 regularization + reg_lambda: 1.0 # Increase L2 regularization + random_state: 42 # Reproducible lgbmc: &lgbmc name: "LGBM Clas." params: - n_estimators: 500 - learning_rate: 0.02 + n_estimators: 300 # More trees to learn slowly and steadily + learning_rate: 0.03 # Lower learning rate to improve generalization + num_leaves: 7 # Fewer leaves — simpler trees + max_depth: 3 # Shallow trees reduce overfitting + min_child_samples: 20 # Require more samples per leaf + subsample: 0.8 # More row sampling to add randomness + colsample_bytree: 0.8 # More feature sampling + reg_alpha: 0.1 # Add L1 regularization + reg_lambda: 1.0 # Increase L2 regularization + random_state: 42 # Reproducible dml_parameters: # ML methods for ml_g and ml_m From b85b2ed44a0146ac86cc21c3345ea9795df86947 Mon Sep 17 00:00:00 2001 From: SvenKlaassen Date: Fri, 6 Jun 2025 13:56:56 +0200 Subject: [PATCH 35/35] fix metadata paths --- doc/irm/irm.qmd | 2 +- doc/irm/irm_cate.qmd | 2 +- doc/irm/irm_gate.qmd | 2 +- doc/plm/pliv.qmd | 2 +- doc/plm/plr.qmd | 2 +- doc/plm/plr_cate.qmd | 4 ++-- doc/plm/plr_gate.qmd | 2 +- doc/rdd/rdd.qmd | 4 ++-- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/irm/irm.qmd b/doc/irm/irm.qmd index f30fe75..a25087c 100644 --- a/doc/irm/irm.qmd +++ b/doc/irm/irm.qmd @@ -30,7 +30,7 @@ The simulations are based on the the [make_irm_data](https://docs.doubleml.org/ ```{python} #| echo: false -metadata_file = '../../results/irm/irm_ate_coverage_metadata.csv' +metadata_file = '../../results/irm/irm_ate_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ``` diff --git a/doc/irm/irm_cate.qmd b/doc/irm/irm_cate.qmd index 52b5abb..df2d3c6 100644 --- a/doc/irm/irm_cate.qmd +++ b/doc/irm/irm_cate.qmd @@ -32,7 +32,7 @@ The non-uniform results (coverage, ci length and bias) refer to averaged values ```{python} #| echo: false -metadata_file = '../../results/irm/irm_cate_coverage_metadata.csv' +metadata_file = '../../results/irm/irm_cate_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ``` diff --git a/doc/irm/irm_gate.qmd b/doc/irm/irm_gate.qmd index c552771..9224fae 100644 --- a/doc/irm/irm_gate.qmd +++ b/doc/irm/irm_gate.qmd @@ -32,7 +32,7 @@ The non-uniform results (coverage, ci length and bias) refer to averaged values ```{python} #| echo: false -metadata_file = '../../results/irm/irm_gate_coverage_metadata.csv' +metadata_file = '../../results/irm/irm_gate_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ``` diff --git a/doc/plm/pliv.qmd b/doc/plm/pliv.qmd index 04f2dcd..eb3b455 100644 --- a/doc/plm/pliv.qmd +++ b/doc/plm/pliv.qmd @@ -30,7 +30,7 @@ The simulations are based on the the [make_pliv_CHS2015](https://docs.doubleml. ```{python} #| echo: false -metadata_file = '../../results/plm/pliv_late_coverage_metadata.csv' +metadata_file = '../../results/plm/pliv_late_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ``` diff --git a/doc/plm/plr.qmd b/doc/plm/plr.qmd index 99f1df5..f9e9304 100644 --- a/doc/plm/plr.qmd +++ b/doc/plm/plr.qmd @@ -30,7 +30,7 @@ The simulations are based on the the [make_plr_CCDDHNR2018](https://docs.double ```{python} #| echo: false -metadata_file = '../../results/plm/plr_ate_coverage_metadata.csv' +metadata_file = '../../results/plm/plr_ate_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ``` diff --git a/doc/plm/plr_cate.qmd b/doc/plm/plr_cate.qmd index ece025e..1581025 100644 --- a/doc/plm/plr_cate.qmd +++ b/doc/plm/plr_cate.qmd @@ -32,7 +32,7 @@ The non-uniform results (coverage, ci length and bias) refer to averaged values ```{python} #| echo: false -metadata_file = '../../results/plm/plr_cate_coverage_metadata.csv' +metadata_file = '../../results/plm/plr_cate_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ``` @@ -116,4 +116,4 @@ generate_and_show_styled_table( rename_map={"Learner g": "Learner l"}, coverage_highlight_cols=["Coverage", "Uniform Coverage"] ) -``` \ No newline at end of file +``` diff --git a/doc/plm/plr_gate.qmd b/doc/plm/plr_gate.qmd index e381fa9..d32bd4e 100644 --- a/doc/plm/plr_gate.qmd +++ b/doc/plm/plr_gate.qmd @@ -32,7 +32,7 @@ The non-uniform results (coverage, ci length and bias) refer to averaged values ```{python} #| echo: false -metadata_file = '../../results/plm/plr_gate_coverage_metadata.csv' +metadata_file = '../../results/plm/plr_gate_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ``` diff --git a/doc/rdd/rdd.qmd b/doc/rdd/rdd.qmd index dce518d..31cdddb 100644 --- a/doc/rdd/rdd.qmd +++ b/doc/rdd/rdd.qmd @@ -31,7 +31,7 @@ The simulations are based on the [make_simple_rdd_data](https://docs.doubleml.or ```{python} #| echo: false -metadata_file = '../../results/rdd/rdd_sharp_coverage_metadata.csv' +metadata_file = '../../results/rdd/rdd_sharp_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ``` @@ -83,7 +83,7 @@ The simulations are based on the [make_simple_rdd_data](https://docs.doubleml.or ```{python} #| echo: false -metadata_file = '../../results/rdd/rdd_fuzzy_coverage_metadata.csv' +metadata_file = '../../results/rdd/rdd_fuzzy_metadata.csv' metadata_df = pd.read_csv(metadata_file) print(metadata_df.T.to_string(header=False)) ```