{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.153653Z",
     "start_time": "2019-12-07T23:06:26.470198Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\six.py:31: DeprecationWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n",
      "  \"(https://pypi.org/project/six/).\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import adjustText\n",
    "import pathlib\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, mean_squared_error\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from plotnine import *\n",
    "from imblearn.over_sampling import SMOTE\n",
    "import mlflow\n",
    "import mlflow.sklearn\n",
    "GIT_PYTHON_REFRESH=0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.172602Z",
     "start_time": "2019-12-07T23:06:34.166619Z"
    }
   },
   "outputs": [],
   "source": [
    "def make_data_imbens(df):\n",
    "    df_new=df.drop(['nodegree'], axis=1)\n",
    "    df_new['pos74']=(df_new['RE74']>0).astype(int)\n",
    "    df_new['pos75']=(df_new['RE75']>0).astype(int)\n",
    "    return df_new\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.191551Z",
     "start_time": "2019-12-07T23:06:34.175594Z"
    }
   },
   "outputs": [],
   "source": [
    "#made better forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.226462Z",
     "start_time": "2019-12-07T23:06:34.193546Z"
    }
   },
   "outputs": [],
   "source": [
    "def predict_g(X, y, do_SMOTE, logs=False):\n",
    "    np.random.seed(42)\n",
    "    predictions=np.full_like(y, np.nan, dtype=float)      \n",
    "    kf=StratifiedKFold(n_splits=10, shuffle=True, random_state=42)   \n",
    "    for train_index, test_index in kf.split(X,y):\n",
    "        clf=RandomForestClassifier(random_state=42, oob_score=False, n_estimators=N_EST, max_depth=MAX_DEPTH)\n",
    "        if do_SMOTE:\n",
    "            X_balanced,y_balanced=SMOTE().fit_resample(X[train_index], y[train_index])\n",
    "            clf.fit(X_balanced, y_balanced)\n",
    "        else:\n",
    "            clf.fit(X[train_index], y[train_index])\n",
    "        predictions[test_index]=clf.predict_proba(X[test_index])[:,1]\n",
    "    assert np.isnan(predictions).sum()==0\n",
    "    if logs:\n",
    "        mlflow.log_param('n_est', N_EST)\n",
    "        mlflow.log_param('max_depth', MAX_DEPTH)\n",
    "        mlflow.log_param('g_SMOTE', do_SMOTE)\n",
    "        mlflow.log_metric('g_accuracy', accuracy_score(np.rint(predictions), y))\n",
    "        column_names=['age', 'education', 'black', 'hispanic', 'married', 'RE74', 'RE75', 'pos74', 'pos75']\n",
    "        feature_importances=pd.DataFrame({'variables': column_names, 'score': clf.feature_importances_}).sort_values(by='score', ascending=False)\n",
    "        feature_importances.to_csv('temp_mlflow_files/g_feat_imp.csv')\n",
    "        mlflow.log_artifact('temp_mlflow_files/g_feat_imp.csv')\n",
    "    return predictions\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.240421Z",
     "start_time": "2019-12-07T23:06:34.230448Z"
    }
   },
   "outputs": [],
   "source": [
    "#made better forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-08T01:22:02.727803Z",
     "start_time": "2019-12-08T01:22:02.687322Z"
    }
   },
   "outputs": [],
   "source": [
    "def predict_Q(X, y, logs=False):\n",
    "    np.random.seed(42)\n",
    "    predictions=np.full_like(y, np.nan, dtype=float)\n",
    "    kf=KFold(n_splits=10, shuffle=True, random_state=42)\n",
    "    for train_index, test_index in kf.split(X):\n",
    "        reg=RandomForestRegressor(random_state=42, n_estimators=N_EST, max_depth=MAX_DEPTH)\n",
    "        reg.fit(X[train_index], y[train_index])\n",
    "        predictions[test_index]=reg.predict(X[test_index])\n",
    "    assert np.isnan(predictions).sum()==0\n",
    "    if logs:\n",
    "        mlflow.log_metric('q_rmse', np.sqrt(mean_squared_error(predictions, y)))\n",
    "        column_names=['treatment', 'age', 'education', 'black', 'hispanic', 'married', 'RE74', 'RE75', 'pos74', 'pos75']\n",
    "        feature_importances=pd.DataFrame({'variables': column_names, 'score': reg.feature_importances_}).sort_values(by='score', ascending=False)\n",
    "        feature_importances.to_csv('temp_mlflow_files/q_feat_imp.csv')\n",
    "        mlflow.log_artifact('temp_mlflow_files/q_feat_imp.csv')\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.301258Z",
     "start_time": "2019-12-07T23:06:34.266357Z"
    }
   },
   "outputs": [],
   "source": [
    "def calc_delta(alpha, g, bias):\n",
    "    alpha_term=(1/alpha)-1\n",
    "    bias_term=sp.special.digamma(g*alpha_term+1)\\\n",
    "        -sp.special.digamma((1-g)*alpha_term)\\\n",
    "        -sp.special.digamma(g*alpha_term)\\\n",
    "        +sp.special.digamma((1-g)*alpha_term+1)\n",
    "    delta=bias/np.mean(bias_term)\n",
    "    return delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.314228Z",
     "start_time": "2019-12-07T23:06:34.305248Z"
    }
   },
   "outputs": [],
   "source": [
    "def calc_beta_shapes(g, alpha):\n",
    "    alpha_shape=g*((1/alpha)-1)\n",
    "    beta_shape=(1-g)*((1/alpha)-1)\n",
    "    return alpha_shape, beta_shape\n",
    "    \n",
    "\n",
    "def calc_Rsq(alpha, delta, calculated_params, treatment, response):\n",
    "    alpha_shape, beta_shape=calc_beta_shapes(calculated_params['g'], alpha)\n",
    "    Rsq_num=delta**2*np.mean(sp.special.polygamma(1, alpha_shape+treatment)\\\n",
    "                             +sp.special.polygamma(1, beta_shape+(1-treatment)))\n",
    "    Rsq_den=mean_squared_error(response, calculated_params['Q'])\n",
    "    return Rsq_num/Rsq_den\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.325194Z",
     "start_time": "2019-12-07T23:06:34.317219Z"
    }
   },
   "outputs": [],
   "source": [
    "def calc_Rsq_hat(covariate_col, response_col, df, Q):\n",
    "    y=df[response_col].values\n",
    "    Qhat=predict_Q(df.drop([response_col]+covariate_col, axis=1).values, y)\n",
    "    Rsqhat=(mean_squared_error(y, Qhat)-mean_squared_error(y, Q))/(mean_squared_error(y, Qhat))\n",
    "    if Rsqhat<0:\n",
    "        Rsqhat=0\n",
    "    return Rsqhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.359103Z",
     "start_time": "2019-12-07T23:06:34.328185Z"
    }
   },
   "outputs": [],
   "source": [
    "def calc_alpha_hat(covariate_col, response_col, treatment_col, df, g, do_SMOTE):\n",
    "    #call y t\n",
    "    t=df[treatment_col].values\n",
    "    ghat=predict_g(df.drop([response_col, treatment_col]+covariate_col, axis=1).values, t, do_SMOTE)\n",
    "#     ahat_numerator=np.var(g)-np.var(ghat)\n",
    "#     ahat_denominator=np.mean(ghat*(1-ghat))\n",
    "    ahat=1-(np.mean(g*(1-g))/np.mean(ghat*(1-ghat)))\n",
    "    #ahat=ahat_numerator/ahat_denominator\n",
    "    if ahat<0:\n",
    "        ahat=0\n",
    "    return ahat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.405978Z",
     "start_time": "2019-12-07T23:06:34.363094Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_sensitivity_graph(df, treatment_col, response_col, covariate_cols, covariate_groups, bias, do_SMOTE):\n",
    "    #Calculate g and Q\n",
    "    g=predict_g(df.drop([response_col, treatment_col], axis=1).values, df[treatment_col].values, do_SMOTE, logs=True)\n",
    "    Q=predict_Q(df.drop([response_col], axis=1).values, df[response_col].values, logs=True)\n",
    "    calculated_params=pd.DataFrame({'g': g, 'Q': Q})\n",
    "    \n",
    "    #Calculate alpha, delta, Rsq\n",
    "    alpha_series=pd.Series(np.arange(0.0001,1,0.0001))\n",
    "    #alpha_series=pd.Series(np.arange(0.001,1,0.001))\n",
    "    delta_series=alpha_series.apply(calc_delta, g=calculated_params['g'], bias=bias)\n",
    "    sensitivity_df=pd.DataFrame({'alphas':alpha_series, 'deltas':delta_series})\n",
    "    Rsq=[]\n",
    "    for n in range(len(alpha_series)):\n",
    "        Rsq.append(calc_Rsq(alpha_series[n], delta_series[n], calculated_params, df[treatment_col], df[response_col]))\n",
    "    sensitivity_df['Rsq']=Rsq\n",
    "    \n",
    "    #Plot observed co-variates\n",
    "    alpha_hat=[]\n",
    "    Rsq_hat=[]\n",
    "    \n",
    "    for covar in covariate_cols:\n",
    "        alpha_hat.append(calc_alpha_hat(covar, response_col, treatment_col, df, calculated_params['g'], do_SMOTE))\n",
    "        Rsq_hat.append(calc_Rsq_hat(covar, response_col, df, calculated_params['Q']))\n",
    "    \n",
    "    variable_importances=pd.DataFrame({'covariates':covariate_cols, 'alpha_hat': alpha_hat, 'Rsq_hat':Rsq_hat, 'groups':covariate_groups})\n",
    "    \n",
    "    variable_importances.to_csv('temp_mlflow_files/variable_importances.csv')\n",
    "    mlflow.log_artifact('temp_mlflow_files/variable_importances.csv')\n",
    "    treatment_Rsq=calc_Rsq_hat(['treatment'], response_col, df, calculated_params['Q'])\n",
    "    mlflow.log_metric('treatment_Rsq', treatment_Rsq)\n",
    "    \n",
    "    #print(variable_importances)\n",
    "    \n",
    "    #print(sensitivity_df)\n",
    "    \n",
    "    g=(ggplot(data=sensitivity_df, mapping=aes(x='alphas', y='Rsq'))\n",
    "            +geom_line(color='#585858', size=1)\n",
    "            +theme_light()\n",
    "            +geom_point(data=variable_importances, mapping=aes(x='alpha_hat', y='Rsq_hat', fill='groups'), color=\"black\", alpha=0.6, size=2.5)\n",
    "            +theme(figure_size=(3.5,3.5), legend_key = element_blank(), axis_title=element_text(size=10), axis_text=element_text(color='black'), plot_title=element_text(size=10))\n",
    "            +scale_x_continuous(expand=[0, 0, 0, 0], limits=(-0.01,0.43))\n",
    "            #+scale_y_continuous(expand=[0, 0, 0, 0], limits=(-0.02, min(0.75, max(Rsq+Rsq_hat))))\n",
    "            +scale_y_continuous(expand=[0, 0, 0, 0], limits=(-0.02, 0.6))\n",
    "            +labs(x=r'$\\mathregular{\\alpha}$ (treatment)', fill='', y='partial $R^2$ (outcome)')\n",
    "            +scale_fill_brewer(type='qual', palette='Set1', labels=['Individual covariates', 'Pre-program earnings', 'Recent earnings'])\n",
    "            #+geom_label(data=variable_importances, mapping=aes(x='alpha_hat', y='Rsq_hat', label='labels'), size=8, nudge_y=0.03, adjust_text={'expand_text': (2, 2), 'expand_points': (2,2), 'force_text': (0.1, 0.25), 'force_points':(0.2, 0.5)})\n",
    "            +annotate(\"text\", x=0.35, y=0.065,label='Bias = $'+str(bias), size=8.5, color='#303030')\n",
    "           )\n",
    "    g.save('temp_mlflow_files/plot.png')\n",
    "    mlflow.log_artifact('temp_mlflow_files/plot.png')\n",
    "    return None\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.413957Z",
     "start_time": "2019-12-07T23:06:34.408971Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "col_names=['treatment', 'age', 'education', 'black', 'hispanic', 'married', 'nodegree', 'RE74', 'RE75', 'RE78']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.590487Z",
     "start_time": "2019-12-07T23:06:34.416949Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "control=pd.read_csv(pathlib.Path.cwd().parents[1].joinpath('data', 'nswre74_control.txt'), header=None, sep=r\"\\s\\s\", names=col_names, engine='python')\n",
    "treatment=pd.read_csv(pathlib.Path.cwd().parents[1].joinpath('data', 'nswre74_treated.txt'), header=None, sep=r\"\\s\\s\", names=col_names, engine='python')\n",
    "psid=pd.read_csv(pathlib.Path.cwd().parents[1].joinpath('data', 'psid_controls.txt'), header=None, sep=r\"\\s\\s\", names=col_names, engine='python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.601456Z",
     "start_time": "2019-12-07T23:06:34.594474Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(260, 10) (185, 10) (2490, 10)\n"
     ]
    }
   ],
   "source": [
    "print(control.shape, treatment.shape, psid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.658304Z",
     "start_time": "2019-12-07T23:06:34.607439Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>treatment</th>\n",
       "      <th>age</th>\n",
       "      <th>education</th>\n",
       "      <th>black</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>married</th>\n",
       "      <th>nodegree</th>\n",
       "      <th>RE74</th>\n",
       "      <th>RE75</th>\n",
       "      <th>RE78</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12383.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10740.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11796.47</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   treatment   age  education  black  hispanic  married  nodegree  RE74  RE75  \\\n",
       "0        0.0  23.0       10.0    1.0       0.0      0.0       1.0   0.0   0.0   \n",
       "1        0.0  26.0       12.0    0.0       0.0      0.0       0.0   0.0   0.0   \n",
       "2        0.0  22.0        9.0    1.0       0.0      0.0       1.0   0.0   0.0   \n",
       "3        0.0  18.0        9.0    1.0       0.0      0.0       1.0   0.0   0.0   \n",
       "4        0.0  45.0       11.0    1.0       0.0      0.0       1.0   0.0   0.0   \n",
       "\n",
       "       RE78  \n",
       "0      0.00  \n",
       "1  12383.68  \n",
       "2      0.00  \n",
       "3  10740.08  \n",
       "4  11796.47  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imbens1=pd.concat([control,treatment]).reset_index(drop=True)\n",
    "imbens1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.692213Z",
     "start_time": "2019-12-07T23:06:34.661302Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>treatment</th>\n",
       "      <th>age</th>\n",
       "      <th>education</th>\n",
       "      <th>black</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>married</th>\n",
       "      <th>RE74</th>\n",
       "      <th>RE75</th>\n",
       "      <th>RE78</th>\n",
       "      <th>pos74</th>\n",
       "      <th>pos75</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12383.68</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10740.08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11796.47</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   treatment   age  education  black  hispanic  married  RE74  RE75      RE78  \\\n",
       "0        0.0  23.0       10.0    1.0       0.0      0.0   0.0   0.0      0.00   \n",
       "1        0.0  26.0       12.0    0.0       0.0      0.0   0.0   0.0  12383.68   \n",
       "2        0.0  22.0        9.0    1.0       0.0      0.0   0.0   0.0      0.00   \n",
       "3        0.0  18.0        9.0    1.0       0.0      0.0   0.0   0.0  10740.08   \n",
       "4        0.0  45.0       11.0    1.0       0.0      0.0   0.0   0.0  11796.47   \n",
       "\n",
       "   pos74  pos75  \n",
       "0      0      0  \n",
       "1      0      0  \n",
       "2      0      0  \n",
       "3      0      0  \n",
       "4      0      0  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imbens1=make_data_imbens(imbens1)\n",
    "imbens1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.718143Z",
     "start_time": "2019-12-07T23:06:34.695206Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "imbens2=pd.concat([treatment, psid]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.752053Z",
     "start_time": "2019-12-07T23:06:34.721136Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>treatment</th>\n",
       "      <th>age</th>\n",
       "      <th>education</th>\n",
       "      <th>black</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>married</th>\n",
       "      <th>RE74</th>\n",
       "      <th>RE75</th>\n",
       "      <th>RE78</th>\n",
       "      <th>pos74</th>\n",
       "      <th>pos75</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9930.0460</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3595.8940</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24909.4500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7506.1460</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>289.7899</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   treatment   age  education  black  hispanic  married  RE74  RE75  \\\n",
       "0        1.0  37.0       11.0    1.0       0.0      1.0   0.0   0.0   \n",
       "1        1.0  22.0        9.0    0.0       1.0      0.0   0.0   0.0   \n",
       "2        1.0  30.0       12.0    1.0       0.0      0.0   0.0   0.0   \n",
       "3        1.0  27.0       11.0    1.0       0.0      0.0   0.0   0.0   \n",
       "4        1.0  33.0        8.0    1.0       0.0      0.0   0.0   0.0   \n",
       "\n",
       "         RE78  pos74  pos75  \n",
       "0   9930.0460      0      0  \n",
       "1   3595.8940      0      0  \n",
       "2  24909.4500      0      0  \n",
       "3   7506.1460      0      0  \n",
       "4    289.7899      0      0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imbens2=make_data_imbens(imbens2)\n",
    "imbens2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.835829Z",
     "start_time": "2019-12-07T23:06:34.755045Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>treatment</th>\n",
       "      <th>age</th>\n",
       "      <th>education</th>\n",
       "      <th>black</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>married</th>\n",
       "      <th>RE74</th>\n",
       "      <th>RE75</th>\n",
       "      <th>change_earnings</th>\n",
       "      <th>pos74</th>\n",
       "      <th>pos75</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9930.0460</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3595.8940</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24909.4500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7506.1460</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>289.7899</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   treatment   age  education  black  hispanic  married  RE74  RE75  \\\n",
       "0        1.0  37.0       11.0    1.0       0.0      1.0   0.0   0.0   \n",
       "1        1.0  22.0        9.0    0.0       1.0      0.0   0.0   0.0   \n",
       "2        1.0  30.0       12.0    1.0       0.0      0.0   0.0   0.0   \n",
       "3        1.0  27.0       11.0    1.0       0.0      0.0   0.0   0.0   \n",
       "4        1.0  33.0        8.0    1.0       0.0      0.0   0.0   0.0   \n",
       "\n",
       "   change_earnings  pos74  pos75  \n",
       "0        9930.0460      0      0  \n",
       "1        3595.8940      0      0  \n",
       "2       24909.4500      0      0  \n",
       "3        7506.1460      0      0  \n",
       "4         289.7899      0      0  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imbens3=pd.concat([treatment, psid]).reset_index(drop=True)\n",
    "imbens3['change_earnings']=imbens3['RE78']-imbens3['RE74']\n",
    "imbens3=make_data_imbens(imbens3)\n",
    "imbens3=imbens3.drop('RE78', axis=1)\n",
    "imbens3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.870737Z",
     "start_time": "2019-12-07T23:06:34.837824Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>education</th>\n",
       "      <th>black</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>married</th>\n",
       "      <th>nodegree</th>\n",
       "      <th>RE74</th>\n",
       "      <th>RE75</th>\n",
       "      <th>RE78</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>treatment</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0.0</td>\n",
       "      <td>242</td>\n",
       "      <td>242</td>\n",
       "      <td>242</td>\n",
       "      <td>242</td>\n",
       "      <td>242</td>\n",
       "      <td>242</td>\n",
       "      <td>242</td>\n",
       "      <td>242</td>\n",
       "      <td>242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1.0</td>\n",
       "      <td>148</td>\n",
       "      <td>148</td>\n",
       "      <td>148</td>\n",
       "      <td>148</td>\n",
       "      <td>148</td>\n",
       "      <td>148</td>\n",
       "      <td>148</td>\n",
       "      <td>148</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           age  education  black  hispanic  married  nodegree  RE74  RE75  \\\n",
       "treatment                                                                   \n",
       "0.0        242        242    242       242      242       242   242   242   \n",
       "1.0        148        148    148       148      148       148   148   148   \n",
       "\n",
       "           RE78  \n",
       "treatment        \n",
       "0.0         242  \n",
       "1.0         148  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imbens4=pd.concat([treatment, psid]).reset_index(drop=True)\n",
    "imbens4=imbens4[~((imbens4['RE74']>=5000) | (imbens4['RE75']>=5000))]\n",
    "imbens4=imbens4.reset_index(drop=True)\n",
    "#Does this match imbens' number of datapoints?\n",
    "imbens4.groupby('treatment').count()\n",
    "#yes!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.921600Z",
     "start_time": "2019-12-07T23:06:34.889685Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>treatment</th>\n",
       "      <th>age</th>\n",
       "      <th>education</th>\n",
       "      <th>black</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>married</th>\n",
       "      <th>RE74</th>\n",
       "      <th>RE75</th>\n",
       "      <th>RE78</th>\n",
       "      <th>pos74</th>\n",
       "      <th>pos75</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9930.0460</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3595.8940</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24909.4500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7506.1460</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>289.7899</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   treatment   age  education  black  hispanic  married  RE74  RE75  \\\n",
       "0        1.0  37.0       11.0    1.0       0.0      1.0   0.0   0.0   \n",
       "1        1.0  22.0        9.0    0.0       1.0      0.0   0.0   0.0   \n",
       "2        1.0  30.0       12.0    1.0       0.0      0.0   0.0   0.0   \n",
       "3        1.0  27.0       11.0    1.0       0.0      0.0   0.0   0.0   \n",
       "4        1.0  33.0        8.0    1.0       0.0      0.0   0.0   0.0   \n",
       "\n",
       "         RE78  pos74  pos75  \n",
       "0   9930.0460      0      0  \n",
       "1   3595.8940      0      0  \n",
       "2  24909.4500      0      0  \n",
       "3   7506.1460      0      0  \n",
       "4    289.7899      0      0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imbens4=make_data_imbens(imbens4)\n",
    "imbens4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:06:34.969472Z",
     "start_time": "2019-12-07T23:06:34.947530Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "covariate_cols=['age', 'education', 'black', 'hispanic', 'married', 'RE74', 'RE75', 'pos74', 'pos75']\n",
    "recent_earnings=['RE75', 'pos75']\n",
    "#what about \"pre-program earnings\"? Is that RE74?\n",
    "pre_program_earnings=['RE74', 'RE75', 'pos74', 'pos75']\n",
    "\n",
    "covariates_for_graphs=[[x] for x in covariate_cols]\n",
    "covariates_for_graphs.append(recent_earnings)\n",
    "covariates_for_graphs.append(pre_program_earnings)\n",
    "\n",
    "covariate_groups=['individual_covariates']*len(covariate_cols)+['recent_earnings', 'pre_program_earnings']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-07T23:12:24.450015Z",
     "start_time": "2019-12-07T23:10:55.910247Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:729: PlotnineWarning: Saving 3.5 x 3.5 in image.\n",
      "  from_inches(height, units), units), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:730: PlotnineWarning: Filename: temp_mlflow_files/plot.png\n",
      "  warn('Filename: {}'.format(filename), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\geoms\\geom_path.py:75: PlotnineWarning: geom_path: Removed 5699 rows containing missing values.\n",
      "  warn(msg.format(n1-n2), PlotnineWarning)\n"
     ]
    }
   ],
   "source": [
    "N_EST=100\n",
    "MAX_DEPTH=5\n",
    "BIAS=1000\n",
    "\n",
    "GIT_PYTHON_REFRESH=0\n",
    "mlflow.start_run()\n",
    "mlflow.log_param('data', 'imbens2')\n",
    "mlflow.log_param('bias', BIAS)\n",
    "mlflow.log_param('model', 'rf')\n",
    "plot_sensitivity_graph(imbens2, \n",
    "                       treatment_col='treatment', \n",
    "                       response_col='RE78', \n",
    "                       covariate_cols=covariates_for_graphs, \n",
    "                       covariate_groups=covariate_groups, \n",
    "                       bias=BIAS, \n",
    "                       do_SMOTE=False)\n",
    "\n",
    "mlflow.end_run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-08T01:36:15.483911Z",
     "start_time": "2019-12-08T01:22:34.448169Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:729: PlotnineWarning: Saving 3.5 x 3.5 in image.\n",
      "  from_inches(height, units), units), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:730: PlotnineWarning: Filename: temp_mlflow_files/plot.png\n",
      "  warn('Filename: {}'.format(filename), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\geoms\\geom_path.py:75: PlotnineWarning: geom_path: Removed 5784 rows containing missing values.\n",
      "  warn(msg.format(n1-n2), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:729: PlotnineWarning: Saving 3.5 x 3.5 in image.\n",
      "  from_inches(height, units), units), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:730: PlotnineWarning: Filename: temp_mlflow_files/plot.png\n",
      "  warn('Filename: {}'.format(filename), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\geoms\\geom_path.py:75: PlotnineWarning: geom_path: Removed 5787 rows containing missing values.\n",
      "  warn(msg.format(n1-n2), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:729: PlotnineWarning: Saving 3.5 x 3.5 in image.\n",
      "  from_inches(height, units), units), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:730: PlotnineWarning: Filename: temp_mlflow_files/plot.png\n",
      "  warn('Filename: {}'.format(filename), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\geoms\\geom_path.py:75: PlotnineWarning: geom_path: Removed 5782 rows containing missing values.\n",
      "  warn(msg.format(n1-n2), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:729: PlotnineWarning: Saving 3.5 x 3.5 in image.\n",
      "  from_inches(height, units), units), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:730: PlotnineWarning: Filename: temp_mlflow_files/plot.png\n",
      "  warn('Filename: {}'.format(filename), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\geoms\\geom_path.py:75: PlotnineWarning: geom_path: Removed 5785 rows containing missing values.\n",
      "  warn(msg.format(n1-n2), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:729: PlotnineWarning: Saving 3.5 x 3.5 in image.\n",
      "  from_inches(height, units), units), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:730: PlotnineWarning: Filename: temp_mlflow_files/plot.png\n",
      "  warn('Filename: {}'.format(filename), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\geoms\\geom_path.py:75: PlotnineWarning: geom_path: Removed 5699 rows containing missing values.\n",
      "  warn(msg.format(n1-n2), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:729: PlotnineWarning: Saving 3.5 x 3.5 in image.\n",
      "  from_inches(height, units), units), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:730: PlotnineWarning: Filename: temp_mlflow_files/plot.png\n",
      "  warn('Filename: {}'.format(filename), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\geoms\\geom_path.py:75: PlotnineWarning: geom_path: Removed 5701 rows containing missing values.\n",
      "  warn(msg.format(n1-n2), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:729: PlotnineWarning: Saving 3.5 x 3.5 in image.\n",
      "  from_inches(height, units), units), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:730: PlotnineWarning: Filename: temp_mlflow_files/plot.png\n",
      "  warn('Filename: {}'.format(filename), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\geoms\\geom_path.py:75: PlotnineWarning: geom_path: Removed 5699 rows containing missing values.\n",
      "  warn(msg.format(n1-n2), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:729: PlotnineWarning: Saving 3.5 x 3.5 in image.\n",
      "  from_inches(height, units), units), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\ggplot.py:730: PlotnineWarning: Filename: temp_mlflow_files/plot.png\n",
      "  warn('Filename: {}'.format(filename), PlotnineWarning)\n",
      "C:\\Users\\light\\Anaconda3\\lib\\site-packages\\plotnine\\geoms\\geom_path.py:75: PlotnineWarning: geom_path: Removed 5699 rows containing missing values.\n",
      "  warn(msg.format(n1-n2), PlotnineWarning)\n"
     ]
    }
   ],
   "source": [
    "for data,name in [(imbens1, 'imbens1'), (imbens2, 'imbens2')]:\n",
    "    for smote in [True, False]:\n",
    "        for N_EST, MAX_DEPTH in [(100,5), (10,3)]:\n",
    "            N_EST=N_EST\n",
    "            MAX_DEPTH=MAX_DEPTH\n",
    "            BIAS=1000\n",
    "            GIT_PYTHON_REFRESH=0\n",
    "            mlflow.start_run()\n",
    "            mlflow.log_param('data', name)\n",
    "            mlflow.log_param('bias', BIAS)\n",
    "            mlflow.log_param('model', 'rf')\n",
    "            plot_sensitivity_graph(data, \n",
    "                                   treatment_col='treatment', \n",
    "                                   response_col='RE78', \n",
    "                                   covariate_cols=covariates_for_graphs, \n",
    "                                   covariate_groups=covariate_groups, \n",
    "                                   bias=BIAS, \n",
    "                                   do_SMOTE=smote)\n",
    "\n",
    "            mlflow.end_run()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
