Sensitivity APO simulations, updated metadata

.github/workflows/apo_sim.yml

@@ -18,6 +18,7 @@ jobs:
       matrix:
         script: [
           'scripts/irm/irm_apo_coverage.py',
+          'scripts/irm/irm_apo_ate_sensitivity.py',
         ]
 
     steps:

doc/irm/apo.qmd

@@ -246,4 +246,113 @@ make_pretty(df_ate_95, level, n_rep)
 level = 0.9
 df_ate_9 = df[df['level'] == level][display_columns]
 make_pretty(df_ate_9, level, n_rep)
-```
+```
+
+
+## Causal Contrast Sensitivity
+
+The simulations are based on the  the ADD-DGP with $10,000$ observations. As the DGP is nonlinear, we will only use corresponding learners. Since the DGP includes an unobserved confounder, we would expect a bias in the ATE estimates, leading to low coverage of the true parameter.
+
+The confounding is set such that both sensitivity parameters are approximately $cf_y=cf_d=0.1$, such that the robustness value $RV$ should be approximately $10\%$.
+Further, the corresponding confidence intervals are one-sided (since the direction of the bias is unkown), such that only one side should approximate the corresponding coverage level (here only the lower coverage is relevant since the bias is positive). Remark that for the coverage level the value of $\rho$ has to be correctly specified, such that the coverage level will be generally (significantly) larger than the nominal level under the conservative choice of $|\rho|=1$.
+
+```{python}
+#| echo: false
+
+import numpy as np
+import pandas as pd
+from itables import init_notebook_mode, show, options
+
+init_notebook_mode(all_interactive=True)
+
+def highlight_range(s, level=0.95, dist=0.05, props=''):
+    color_grid = np.where((s >= level-dist) &
+                          (s <= level+dist), props, '')
+    return color_grid
+
+
+def color_coverage(df, level):
+    # color coverage column order is important
+    styled_df = df.apply(
+        highlight_range,
+        level=level,
+        dist=1.0,
+        props='color:black;background-color:red',
+        subset=["Coverage", "Coverage (Lower)"])
+    styled_df = styled_df.apply(
+        highlight_range,
+        level=level,
+        dist=0.1,
+        props='color:black;background-color:yellow',
+        subset=["Coverage", "Coverage (Lower)"])
+    styled_df = styled_df.apply(
+        highlight_range,
+        level=level,
+        dist=0.05,
+        props='color:white;background-color:darkgreen',
+        subset=["Coverage", "Coverage (Lower)"])
+
+    # set all coverage values to bold
+    styled_df = styled_df.set_properties(
+        **{'font-weight': 'bold'},
+        subset=["Coverage", "Coverage (Lower)"])
+    return styled_df
+
+
+def make_pretty(df, level, n_rep):
+    styled_df = df.style.hide(axis="index")
+    # Format only float columns
+    float_cols = df.select_dtypes(include=['float']).columns
+    styled_df = styled_df.format({col: "{:.3f}" for col in float_cols})
+
+    # color coverage column order is important
+    styled_df = color_coverage(styled_df, level)
+    caption = f"Coverage for {level*100}%-Confidence Interval over {n_rep} Repetitions"
+
+    return show(styled_df, caption=caption)
+```
+
+### ATE
+
+::: {.callout-note title="Metadata"  collapse="true"}
+
+```{python}
+#| echo: false
+metadata_file = '../../results/irm/irm_apo_sensitivity_metadata.csv'
+metadata_df = pd.read_csv(metadata_file)
+print(metadata_df.T.to_string(header=False))
+```
+
+:::
+
+```{python}
+#| echo: false
+
+# set up data and rename columns
+df = pd.read_csv("../../results/irm/irm_apo_sensitivity.csv", index_col=None)
+
+assert df["repetition"].nunique() == 1
+n_rep = df["repetition"].unique()[0]
+
+display_columns = [
+    "Learner g", "Learner m", "Bias", "Bias (Lower)", "Bias (Upper)", "Coverage", "Coverage (Lower)", "Coverage (Upper)", "RV", "RVa"]
+```
+
+```{python}
+#| echo: false
+level = 0.95
+
+df_ate_95 = df[(df['level'] == level)][display_columns]
+df_ate_95.rename(columns={"Learner g": "Learner l"}, inplace=True)
+make_pretty(df_ate_95, level, n_rep)
+```
+
+```{python}
+#| echo: false
+level = 0.9
+
+df_ate_9 = df[(df['level'] == level)][display_columns]
+df_ate_9.rename(columns={"Learner g": "Learner l"}, inplace=True)
+make_pretty(df_ate_9, level, n_rep)
+```
+

doc/irm/irm.qmd

@@ -242,7 +242,6 @@ display_columns = [
 
 ```{python}
 #| echo: false
-score = "partialling out"
 level = 0.95
 
 df_ate_95 = df[(df['level'] == level)][display_columns]
@@ -252,7 +251,6 @@ make_pretty(df_ate_95, level, n_rep)
 
 ```{python}
 #| echo: false
-score = "partialling out"
 level = 0.9
 
 df_ate_9 = df[(df['level'] == level)][display_columns]

requirements.txt

@@ -2,7 +2,7 @@ doubleml[rdd]
 joblib
 numpy
 pandas
-scikit-learn
+scikit-learn==1.5.2
 lightgbm
 itables
 ipykernel

results/irm/irm_apo_coverage_apo.csv

@@ -1,25 +1,25 @@
 Learner g,Learner m,Treatment Level,level,Coverage,CI Length,Bias,repetition
-LGBM,LGBM,0.0,0.9,0.911,8.657690136121921,2.0760918678352267,1000
-LGBM,LGBM,0.0,0.95,0.963,10.316274091547035,2.0760918678352267,1000
-LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.19332007236871,1000
-LGBM,LGBM,1.0,0.95,0.967,45.55789754237906,9.19332007236871,1000
-LGBM,LGBM,2.0,0.9,0.891,37.49194764946096,9.594147407062762,1000
-LGBM,LGBM,2.0,0.95,0.952,44.67441108385784,9.594147407062762,1000
-LGBM,Logistic,0.0,0.9,0.901,5.625897101886533,1.3366474407456235,1000
-LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.3366474407456235,1000
-LGBM,Logistic,1.0,0.9,0.92,7.423300143143785,1.6954254817667072,1000
-LGBM,Logistic,1.0,0.95,0.968,8.84540769378873,1.6954254817667072,1000
-LGBM,Logistic,2.0,0.9,0.918,7.321275660150268,1.660607504960853,1000
-LGBM,Logistic,2.0,0.95,0.971,8.723838024042964,1.660607504960853,1000
-Linear,LGBM,0.0,0.9,0.901,5.498257423071024,1.307400569952483,1000
-Linear,LGBM,0.0,0.95,0.949,6.551577812380716,1.307400569952483,1000
-Linear,LGBM,1.0,0.9,0.951,10.700720020780512,2.1270559074131152,1000
-Linear,LGBM,1.0,0.95,0.982,12.75069435099058,2.1270559074131152,1000
-Linear,LGBM,2.0,0.9,0.932,7.513644049429104,1.634643965398859,1000
-Linear,LGBM,2.0,0.95,0.967,8.953059097926168,1.634643965398859,1000
-Linear,Logistic,0.0,0.9,0.9,5.335670092717667,1.2859337252007816,1000
-Linear,Logistic,0.0,0.95,0.951,6.357843058957276,1.2859337252007816,1000
-Linear,Logistic,1.0,0.9,0.904,5.417512107920403,1.2802845007629777,1000
-Linear,Logistic,1.0,0.95,0.954,6.455363835025866,1.2802845007629777,1000
-Linear,Logistic,2.0,0.9,0.905,5.366403391397197,1.2808554104576877,1000
-Linear,Logistic,2.0,0.95,0.959,6.3944640430685675,1.2808554104576877,1000
+LGBM,LGBM,0.0,0.9,0.91,8.657690136121921,2.077166476271961,1000
+LGBM,LGBM,0.0,0.95,0.966,10.316274091547035,2.077166476271961,1000
+LGBM,LGBM,1.0,0.9,0.914,38.23339285821166,9.216990722545221,1000
+LGBM,LGBM,1.0,0.95,0.968,45.55789754237906,9.216990722545221,1000
+LGBM,LGBM,2.0,0.9,0.894,37.49194764946096,9.572829064615373,1000
+LGBM,LGBM,2.0,0.95,0.953,44.67441108385784,9.572829064615373,1000
+LGBM,Logistic,0.0,0.9,0.904,5.625897101886533,1.341832646808728,1000
+LGBM,Logistic,0.0,0.95,0.955,6.7036698705295725,1.341832646808728,1000
+LGBM,Logistic,1.0,0.9,0.924,7.423300143143785,1.6934000724558478,1000
+LGBM,Logistic,1.0,0.95,0.969,8.84540769378873,1.6934000724558478,1000
+LGBM,Logistic,2.0,0.9,0.92,7.321275660150268,1.6623515097372739,1000
+LGBM,Logistic,2.0,0.95,0.965,8.723838024042964,1.6623515097372739,1000
+Linear,LGBM,0.0,0.9,0.9,5.498257423071024,1.3124516734608647,1000
+Linear,LGBM,0.0,0.95,0.95,6.551577812380716,1.3124516734608647,1000
+Linear,LGBM,1.0,0.9,0.948,10.700720020780512,2.1292560240454224,1000
+Linear,LGBM,1.0,0.95,0.983,12.75069435099058,2.1292560240454224,1000
+Linear,LGBM,2.0,0.9,0.934,7.513644049429104,1.6375362733403969,1000
+Linear,LGBM,2.0,0.95,0.966,8.953059097926168,1.6375362733403969,1000
+Linear,Logistic,0.0,0.9,0.905,5.335670092717667,1.2915169940302822,1000
+Linear,Logistic,0.0,0.95,0.955,6.357843058957276,1.2915169940302822,1000
+Linear,Logistic,1.0,0.9,0.908,5.417512107920403,1.2804808010829234,1000
+Linear,Logistic,1.0,0.95,0.957,6.455363835025866,1.2804808010829234,1000
+Linear,Logistic,2.0,0.9,0.906,5.366403391397197,1.2873523314776736,1000
+Linear,Logistic,2.0,0.95,0.957,6.3944640430685675,1.2873523314776736,1000

results/irm/irm_apo_coverage_apos.csv

@@ -1,9 +1,9 @@
 Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition
-LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.1182934844676025,0.925,36.1943517986729,1000
-LGBM,LGBM,0.95,0.9583333333333334,33.61454856442563,7.1182934844676025,0.973,40.868020474452145,1000
-LGBM,Logistic,0.9,0.9153333333333333,6.789316986971467,1.5714825799469632,0.924,8.394070597867524,1000
-LGBM,Logistic,0.95,0.963,8.089970168806186,1.5714825799469632,0.959,9.592284619044477,1000
-Linear,LGBM,0.9,0.9266666666666666,7.903418354805325,1.726298806364679,0.939,9.850445774370156,1000
-Linear,LGBM,0.95,0.967,9.41750382912841,1.726298806364679,0.974,11.234392064837424,1000
-Linear,Logistic,0.9,0.904,5.372532806955153,1.2803595601237736,0.906,5.7964894175017925,1000
-Linear,Logistic,0.95,0.957,6.4017676921854445,1.2803595601237736,0.953,6.8182385960659,1000
+LGBM,LGBM,0.9,0.9063333333333333,28.21021843292038,7.117854990409516,0.926,36.1943517986729,1000
+LGBM,LGBM,0.95,0.958,33.61454856442563,7.117854990409516,0.973,40.868020474452145,1000
+LGBM,Logistic,0.9,0.9126666666666666,6.789316986971467,1.5740024109154114,0.922,8.394070597867524,1000
+LGBM,Logistic,0.95,0.962,8.089970168806186,1.5740024109154114,0.959,9.592284619044477,1000
+Linear,LGBM,0.9,0.927,7.903418354805325,1.730081847384891,0.936,9.850445774370156,1000
+Linear,LGBM,0.95,0.968,9.41750382912841,1.730081847384891,0.974,11.234392064837424,1000
+Linear,Logistic,0.9,0.9053333333333333,5.372532806955153,1.2840405983352503,0.901,5.7964894175017925,1000
+Linear,Logistic,0.95,0.9556666666666667,6.4017676921854445,1.2840405983352503,0.952,6.8182385960659,1000

results/irm/irm_apo_coverage_apos_contrast.csv

@@ -1,9 +1,9 @@
 Learner g,Learner m,level,Coverage,CI Length,Bias,Uniform Coverage,Uniform CI Length,repetition
-LGBM,LGBM,0.9,0.8885,37.87536523730769,9.786356896553793,0.898,44.828925781533165,1000
-LGBM,LGBM,0.95,0.9485,45.13128131893863,9.786356896553793,0.965,51.42387403222799,1000
-LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2679354841429349,0.926,6.774304315459575,1000
-LGBM,Logistic,0.95,0.962,6.821911652197591,1.2679354841429349,0.961,7.7654094416927215,1000
-Linear,LGBM,0.9,0.958,7.430953406859927,1.5066894086042675,0.975,8.798628447594016,1000
-Linear,LGBM,0.95,0.9885,8.85452711998085,1.5066894086042675,0.992,10.090116294778499,1000
-Linear,Logistic,0.9,0.873,1.1425883251377777,0.29743753232068365,0.869,1.3505222299078565,1000
-Linear,Logistic,0.95,0.9335,1.3614779635902856,0.29743753232068365,0.916,1.5496710496635275,1000
+LGBM,LGBM,0.9,0.8885,37.87536523730769,9.786311172069826,0.898,44.828925781533165,1000
+LGBM,LGBM,0.95,0.9485,45.13128131893863,9.786311172069826,0.965,51.42387403222799,1000
+LGBM,Logistic,0.9,0.9275,5.725128733165455,1.2679687960356285,0.926,6.774304315459575,1000
+LGBM,Logistic,0.95,0.962,6.821911652197591,1.2679687960356285,0.961,7.7654094416927215,1000
+Linear,LGBM,0.9,0.958,7.430953406859927,1.506712453445455,0.975,8.798628447594016,1000
+Linear,LGBM,0.95,0.9885,8.85452711998085,1.506712453445455,0.992,10.090116294778499,1000
+Linear,Logistic,0.9,0.873,1.1425883251377777,0.2976271580660105,0.871,1.3505222299078565,1000
+Linear,Logistic,0.95,0.933,1.3614779635902856,0.2976271580660105,0.916,1.5496710496635275,1000

results/irm/irm_apo_coverage_metadata.csv

@@ -1,2 +1,2 @@
-DoubleML Version,Script,Date,Total Runtime (seconds),Python Version
-0.10.dev0,irm_apo_coverage.py,2025-01-08 15:02:49,10054.695460557938,3.12.8
+DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions
+0.10.dev0,irm_apo_coverage.py,2025-02-17 14:15:48,5805.248526096344,3.12.9,500,1000

results/irm/irm_apo_sensitivity.csv

@@ -0,0 +1,9 @@
+Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),CI Bound Length,repetition
+LGBM,LGBM,0.9,0.0,0.1850901948571434,0.1728687505886857,0.94,1.0,0.11948035202564793,0.07154674763559875,0.031810817504062554,0.31670282436556935,0.431995754637111,100
+LGBM,LGBM,0.95,0.0,0.2205485703210263,0.1728687505886857,0.98,1.0,0.11948035202564793,0.05745432815748635,0.031810817504062554,0.31670282436556935,0.4729106257831797,100
+LGBM,Logistic Regr.,0.9,0.0,0.18131193168905674,0.15121724930885022,0.99,1.0,0.10434537281186201,0.05696357062448621,0.021088484865101825,0.29646127759826496,0.43179145449456513,100
+LGBM,Logistic Regr.,0.95,0.03,0.2160464920739249,0.15121724930885022,1.0,1.0,0.10434537281186201,0.04307260925250725,0.021088484865101825,0.29646127759826496,0.47184900436363103,100
+Linear Reg.,LGBM,0.9,0.0,0.18663164735761398,0.17472687461915876,0.92,1.0,0.12056131194284912,0.07240650957100214,0.03257050661484845,0.31875766896448493,0.4335837394031654,100
+Linear Reg.,LGBM,0.95,0.0,0.2223853242639293,0.17472687461915876,0.99,1.0,0.12056131194284912,0.05822924694927436,0.03257050661484845,0.31875766896448493,0.47483724706967173,100
+Linear Reg.,Logistic Regr.,0.9,0.56,0.1828483122747645,0.08933864685804803,1.0,1.0,0.06284434571148402,0.015959822013915787,0.05670070892745642,0.23481634587221883,0.43345409695347575,100
+Linear Reg.,Logistic Regr.,0.95,0.71,0.21787720245762926,0.08933864685804803,1.0,1.0,0.06284434571148402,0.007943564313924893,0.05670070892745642,0.23481634587221883,0.4738504980497505,100

results/irm/irm_apo_sensitivity_metadata.csv

@@ -0,0 +1,2 @@
+DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Sensitivity Errors,Number of observations,Number of repetitions
+0.10.dev0,irm_apo_sensitivity.py,2025-02-17 15:18:22,9570.017815113068,3.12.9,0,10000,100

results/irm/irm_ate_coverage.csv

@@ -1,9 +1,9 @@
 Learner g,Learner m,level,Coverage,CI Length,Bias,repetition
-Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536219,1000
-Lasso,Logistic Regression,0.95,0.935,0.5573839547492132,0.12336884150536219,1000
-Lasso,Random Forest,0.9,0.906,0.6047145364621751,0.1478860899164366,1000
-Lasso,Random Forest,0.95,0.959,0.7205618135094181,0.1478860899164366,1000
-Random Forest,Logistic Regression,0.9,0.803,0.5176227656141883,0.15019297474976653,1000
-Random Forest,Logistic Regression,0.95,0.883,0.6167855677602845,0.15019297474976653,1000
-Random Forest,Random Forest,0.9,0.899,0.6330777189487165,0.15360554047241584,1000
-Random Forest,Random Forest,0.95,0.959,0.7543586299857805,0.15360554047241584,1000
+Lasso,Logistic Regression,0.9,0.875,0.467771360497192,0.12336884150536215,1000
+Lasso,Logistic Regression,0.95,0.935,0.5573839547492131,0.12336884150536215,1000
+Lasso,Random Forest,0.9,0.918,0.6075675690766795,0.14883418949014868,1000
+Lasso,Random Forest,0.95,0.96,0.7239614115523821,0.14883418949014868,1000
+Random Forest,Logistic Regression,0.9,0.794,0.5179181614571521,0.15026458264746098,1000
+Random Forest,Logistic Regression,0.95,0.886,0.6171375536172053,0.15026458264746098,1000
+Random Forest,Random Forest,0.9,0.887,0.6267427414652039,0.1523909615423744,1000
+Random Forest,Random Forest,0.95,0.951,0.7468100387268922,0.1523909615423744,1000

results/irm/irm_ate_coverage_metadata.csv

@@ -1,2 +1,2 @@
-DoubleML Version,Script,Date,Total Runtime (seconds),Python Version
-0.10.dev0,irm_ate_coverage.py,2025-01-08 13:14:33,3543.555382013321,3.12.8
+DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions
+0.10.dev0,irm_ate_coverage.py,2025-02-17 13:38:59,3587.198892593384,3.12.9,500,1000

results/irm/irm_ate_sensitivity.csv

@@ -1,9 +1,9 @@
-Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),repetition
-LGBM,LGBM,0.9,0.112,0.266748233354866,0.17891290135375168,0.962,1.0,0.12379347892727971,0.05409589192160397,0.04254708028278409,0.32210978560617337,500
-LGBM,LGBM,0.95,0.318,0.31785012462427936,0.17891290135375168,0.998,1.0,0.12379347892727971,0.03441021667548556,0.04254708028278409,0.32210978560617337,500
-LGBM,Logistic Regr.,0.9,0.292,0.2577778025822409,0.14922926552528684,1.0,1.0,0.10066571951295798,0.03493291437943745,0.029012990398602386,0.2979424530565633,500
-LGBM,Logistic Regr.,0.95,0.548,0.30716119707955875,0.14922926552528684,1.0,1.0,0.10066571951295798,0.01869752301454861,0.029012990398602386,0.2979424530565633,500
-Linear Reg.,LGBM,0.9,0.122,0.2675665174758639,0.17873104426193565,0.964,1.0,0.12647219547900976,0.05512739569620471,0.04513946154555041,0.31857328180879246,500
-Linear Reg.,LGBM,0.95,0.314,0.31882517029399604,0.17873104426193565,0.998,1.0,0.12647219547900976,0.035017588858111126,0.04513946154555041,0.31857328180879246,500
-Linear Reg.,Logistic Regr.,0.9,0.86,0.2592281409673473,0.08970251629543106,1.0,1.0,0.06300567732617765,0.006719868195974334,0.05720312141493262,0.23496869651774063,500
-Linear Reg.,Logistic Regr.,0.95,0.974,0.30888938185760084,0.08970251629543106,1.0,1.0,0.06300567732617765,0.0014945204694376396,0.05720312141493262,0.23496869651774063,500
+Learner g,Learner m,level,Coverage,CI Length,Bias,Coverage (Lower),Coverage (Upper),RV,RVa,Bias (Lower),Bias (Upper),CI Bound Length,repetition
+LGBM,LGBM,0.9,0.0,0.18456010957050942,0.17337243340910458,0.95,1.0,0.11953784478980024,0.07183819359114625,0.03136325666565055,0.31756830147937315,0.43226987036337094,100
+LGBM,LGBM,0.95,0.0,0.21991693474354154,0.17337243340910458,0.99,1.0,0.11953784478980024,0.057824558257341376,0.03136325666565055,0.31756830147937315,0.47305732219900365,100
+LGBM,Logistic Regr.,0.9,0.0,0.1812877961255176,0.15119732710067918,0.98,1.0,0.10434720873886104,0.05696666617037721,0.021677485760078018,0.29642303876248005,0.4317358050796223,100
+LGBM,Logistic Regr.,0.95,0.04,0.2160177327761319,0.15119732710067918,1.0,1.0,0.10434720873886104,0.04307608641610582,0.021677485760078018,0.29642303876248005,0.47178796413164226,100
+Linear Reg.,LGBM,0.9,0.0,0.18605425587835156,0.17399977827987403,0.94,1.0,0.11990868556653686,0.07190610415701137,0.03212149599746574,0.3182707711756607,0.4335852019350203,100
+Linear Reg.,LGBM,0.95,0.0,0.22169731988117328,0.17399977827987403,1.0,1.0,0.11990868556653686,0.057787828308944675,0.03212149599746574,0.3182707711756607,0.4747029383060918,100
+Linear Reg.,Logistic Regr.,0.9,0.57,0.1828627595282207,0.08930473133509086,1.0,1.0,0.06281551261599926,0.015943919677551065,0.05677070380042624,0.234795381047018,0.4334912988005637,100
+Linear Reg.,Logistic Regr.,0.95,0.72,0.21789441742191876,0.08930473133509086,1.0,1.0,0.06281551261599926,0.007945274454461557,0.05677070380042624,0.234795381047018,0.4738909034178378,100

results/irm/irm_ate_sensitivity_metadata.csv

@@ -1,2 +1,2 @@
-DoubleML Version,Script,Date,Total Runtime (seconds),Python Version
-0.10.dev0,irm_ate_sensitivity.py,2025-01-08 14:51:20,9351.407655954361,3.12.8
+DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions
+0.10.dev0,irm_ate_sensitivity.py,2025-02-17 13:05:22,1572.6342079639435,3.12.9,10000,100

results/irm/irm_atte_coverage.csv

@@ -1,9 +1,9 @@
 Learner g,Learner m,level,Coverage,CI Length,Bias,repetition
-Lasso,Logistic Regression,0.9,0.887,0.5331759172799809,0.13448723008182129,1000
-Lasso,Logistic Regression,0.95,0.942,0.6353182910443254,0.13448723008182129,1000
-Lasso,Random Forest,0.9,0.895,0.7320685495582887,0.18096827990713532,1000
-Lasso,Random Forest,0.95,0.948,0.8723134799586961,0.18096827990713532,1000
-Random Forest,Logistic Regression,0.9,0.871,0.5508997607447915,0.1495507044107987,1000
-Random Forest,Logistic Regression,0.95,0.928,0.6564375531412434,0.1495507044107987,1000
-Random Forest,Random Forest,0.9,0.901,0.7500491227406757,0.18282767194851973,1000
-Random Forest,Random Forest,0.95,0.948,0.8937386543823805,0.18282767194851973,1000
+Lasso,Logistic Regression,0.9,0.889,0.5331759172799809,0.1351855584332284,1000
+Lasso,Logistic Regression,0.95,0.937,0.6353182910443254,0.1351855584332284,1000
+Lasso,Random Forest,0.9,0.9,0.7254806184755598,0.18158167210289639,1000
+Lasso,Random Forest,0.95,0.955,0.864463475895592,0.18158167210289639,1000
+Random Forest,Logistic Regression,0.9,0.869,0.5498973024315036,0.14982335722803167,1000
+Random Forest,Logistic Regression,0.95,0.919,0.6552430503855857,0.14982335722803167,1000
+Random Forest,Random Forest,0.9,0.9,0.754141932378241,0.18774885787201864,1000
+Random Forest,Random Forest,0.95,0.944,0.8986155378653647,0.18774885787201864,1000

results/irm/irm_atte_coverage_metadata.csv

@@ -1,2 +1,2 @@
-DoubleML Version,Script,Date,Total Runtime (seconds),Python Version
-0.10.dev0,irm_atte_coverage.py,2025-01-08 13:14:10,3538.650551557541,3.12.8
+DoubleML Version,Script,Date,Total Runtime (seconds),Python Version,Number of observations,Number of repetitions
+0.10.dev0,irm_atte_coverage.py,2025-02-17 13:39:11,3599.8705773353577,3.12.9,500,1000