{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "version": "3.7.3"
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pathlib\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>iqsb.36</th>\n      <th>dose400</th>\n      <th>treat</th>\n      <th>bw</th>\n      <th>momage</th>\n      <th>nnhealth</th>\n      <th>birth.o</th>\n      <th>parity</th>\n      <th>moreprem</th>\n      <th>cigs</th>\n      <th>...</th>\n      <th>b.marryF</th>\n      <th>livwhoF</th>\n      <th>languageF</th>\n      <th>whenprenF</th>\n      <th>drugs.1</th>\n      <th>othstudy.1</th>\n      <th>momed4F</th>\n      <th>siteF</th>\n      <th>momraceF</th>\n      <th>workdur.imp.1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>120.0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1559</td>\n      <td>33</td>\n      <td>94</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>2</td>\n      <td>4</td>\n      <td>1</td>\n      <td>w</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>90.0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1420</td>\n      <td>15</td>\n      <td>85</td>\n      <td>2</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>0</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>b</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>76.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1000</td>\n      <td>33</td>\n      <td>89</td>\n      <td>4</td>\n      <td>5</td>\n      <td>0</td>\n      <td>20</td>\n      <td>...</td>\n      <td>3</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>b</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>43.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1430</td>\n      <td>22</td>\n      <td>112</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>b</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>73.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1984</td>\n      <td>20</td>\n      <td>99</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>10</td>\n      <td>...</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>2</td>\n      <td>2</td>\n      <td>2</td>\n      <td>3</td>\n      <td>1</td>\n      <td>b</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 50 columns</p>\n</div>",
      "text/plain": "   iqsb.36  dose400  treat    bw  momage  nnhealth  birth.o  parity  moreprem  \\\n0    120.0        1      1  1559      33        94        2       2         1   \n1     90.0        0      1  1420      15        85        2       2         0   \n2     76.0        0      0  1000      33        89        4       5         0   \n3     43.0        0      0  1430      22       112        1       1         0   \n4     73.0        0      0  1984      20        99        1       1         0   \n\n   cigs  ...  b.marryF  livwhoF  languageF  whenprenF  drugs.1  othstudy.1  \\\n0     0  ...         1        1          1          1        2           2   \n1     0  ...         2        2          1          0        2           2   \n2    20  ...         3        2          1          1        2           2   \n3     0  ...         2        2          1          1        2           2   \n4    10  ...         2        2          1          2        2           2   \n\n   momed4F  siteF  momraceF  workdur.imp.1  \n0        4      1         w              1  \n1        1      1         b              1  \n2        1      1         b              1  \n3        1      1         b              0  \n4        3      1         b              0  \n\n[5 rows x 50 columns]"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('../../data/ihdp-raw/ihdp-hill-2011/data/example_data.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=data.dropna(subset=['iqsb.36'], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>iqsb.36</th>\n      <th>dose400</th>\n      <th>treat</th>\n      <th>bw</th>\n      <th>momage</th>\n      <th>nnhealth</th>\n      <th>birth.o</th>\n      <th>parity</th>\n      <th>moreprem</th>\n      <th>cigs</th>\n      <th>...</th>\n      <th>b.marryF</th>\n      <th>livwhoF</th>\n      <th>languageF</th>\n      <th>whenprenF</th>\n      <th>drugs.1</th>\n      <th>othstudy.1</th>\n      <th>momed4F</th>\n      <th>siteF</th>\n      <th>momraceF</th>\n      <th>workdur.imp.1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>120.0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1559</td>\n      <td>33</td>\n      <td>94</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>2</td>\n      <td>4</td>\n      <td>1</td>\n      <td>w</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>90.0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1420</td>\n      <td>15</td>\n      <td>85</td>\n      <td>2</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>0</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>b</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>76.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1000</td>\n      <td>33</td>\n      <td>89</td>\n      <td>4</td>\n      <td>5</td>\n      <td>0</td>\n      <td>20</td>\n      <td>...</td>\n      <td>3</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>b</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>43.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1430</td>\n      <td>22</td>\n      <td>112</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>b</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>73.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1984</td>\n      <td>20</td>\n      <td>99</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>10</td>\n      <td>...</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>2</td>\n      <td>2</td>\n      <td>2</td>\n      <td>3</td>\n      <td>1</td>\n      <td>b</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 50 columns</p>\n</div>",
      "text/plain": "   iqsb.36  dose400  treat    bw  momage  nnhealth  birth.o  parity  moreprem  \\\n0    120.0        1      1  1559      33        94        2       2         1   \n1     90.0        0      1  1420      15        85        2       2         0   \n2     76.0        0      0  1000      33        89        4       5         0   \n3     43.0        0      0  1430      22       112        1       1         0   \n4     73.0        0      0  1984      20        99        1       1         0   \n\n   cigs  ...  b.marryF  livwhoF  languageF  whenprenF  drugs.1  othstudy.1  \\\n0     0  ...         1        1          1          1        2           2   \n1     0  ...         2        2          1          0        2           2   \n2    20  ...         3        2          1          1        2           2   \n3     0  ...         2        2          1          1        2           2   \n4    10  ...         2        2          1          2        2           2   \n\n   momed4F  siteF  momraceF  workdur.imp.1  \n0        4      1         w              1  \n1        1      1         b              1  \n2        1      1         b              1  \n3        1      1         b              0  \n4        3      1         b              0  \n\n[5 rows x 50 columns]"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=data[~((data['dose400']==0) & (data['treat']==1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(628, 50)"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0    561\n1     67\nName: treat, dtype: int64"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.treat.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0    561\n1     67\nName: dose400, dtype: int64"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dose400.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "628"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data.treat==data.dose400).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=data.drop(columns=['dose400'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('../../data/ihdp-cleaned/example_data_cleaned.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}