{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Python packages used in this code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import os\n",
    "import pickle\n",
    "import time\n",
    "import sklearn\n",
    "import platform\n",
    "import sys\n",
    "from sklearn.base import BaseEstimator, RegressorMixin\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.linear_model import Ridge, LinearRegression\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "import joblib\n",
    "import itertools\n",
    "from IPython.display import clear_output\n",
    "\n",
    "## Keras\n",
    "import tensorflow\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Sequential, Model, model_from_json\n",
    "from tensorflow.keras.layers import Dense, Input, Add, Lambda, Dropout, Subtract, Multiply, Concatenate, Dot, BatchNormalization, Activation, LeakyReLU, ReLU\n",
    "from tensorflow.keras.losses import mse\n",
    "import keras.backend as K\n",
    "from tensorflow.keras import regularizers\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL']='2'\n",
    "tensorflow.get_logger().setLevel(\"ERROR\")\n",
    "\n",
    "from scipy import io\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Environments\n",
    "\n",
    "--Platform--\n",
    "OS : Windows-10-10.0.19044-SP0\n",
    "--Version--\n",
    "python :  3.9.12 (main, Apr  4 2022, 05:22:27) [MSC v.1916 64 bit (AMD64)]\n",
    "numpy : 1.23.1\n",
    "pandas : 1.4.3\n",
    "sklearn : 1.1.1\n",
    "tensorflow : 2.9.1\n",
    "keras : 2.9.0\n",
    "\"\"\"\n",
    "\n",
    "print('--Platform--')\n",
    "print('OS :', platform.platform())\n",
    "print('--Version--')\n",
    "print('python : ', sys.version)\n",
    "print('numpy :', np.__version__)\n",
    "print('pandas :', pd.__version__)\n",
    "print('sklearn :', sklearn.__version__)\n",
    "print('tensorflow :', tensorflow.__version__)\n",
    "print('keras :', keras.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fix_seed function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_seed(seed):\n",
    "    # Numpy\n",
    "    np.random.seed(seed)\n",
    "    # Tensorflow\n",
    "    tensorflow.random.set_seed(seed)\n",
    "    # for built-in random\n",
    "    random.seed(seed)\n",
    "    # for hash seed\n",
    "    os.environ[\"PYTHONHASHSEED\"] = str(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PrintDot function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PrintDot(keras.callbacks.Callback):\n",
    "    def on_epoch_end(self, epoch, logs):\n",
    "        if epoch%100==0: print('')\n",
    "        if epoch%100==1: print(epoch)\n",
    "        print('.', end='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isdir('../30_Output/10_Model/100_MakeSourceModel'):\n",
    "    os.makedirs('../30_Output/10_Model/100_MakeSourceModel')\n",
    "if not os.path.isdir('../30_Output/20_Plot/100_MakeSourceModel'):\n",
    "    os.makedirs('../30_Output/20_Plot/100_MakeSourceModel')\n",
    "if not os.path.isdir('../30_Output/30_csv/100_MakeSourceModel'):\n",
    "    os.makedirs('../30_Output/30_csv/100_MakeSourceModel')\n",
    "if not os.path.isdir('../30_Output/40_pkl/100_MakeSourceModel'):\n",
    "    os.makedirs('../30_Output/40_pkl/100_MakeSourceModel')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fix_seed(373)\n",
    "target_name_list = ['Torque1','Torque2','Torque3','Torque4','Torque5','Torque6','Torque7']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "early_stop = keras.callbacks.EarlyStopping(monitor='val_mse', patience=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = '../10_Data/sarcos_inv.mat'\n",
    "test_path = '../10_Data/sarcos_inv_test.mat'\n",
    "axis_names = ['Position1','Position2','Position3','Position4','Position5','Position6','Position7',\n",
    "              'Velocity1','Velocity2','Velocity3','Velocity4','Velocity5','Velocity6','Velocity7',\n",
    "              'Acceleration1','Acceleration2','Acceleration3','Acceleration4','Acceleration5','Acceleration6','Acceleration7',\n",
    "              'Torque1','Torque2','Torque3','Torque4','Torque5','Torque6','Torque7']\n",
    "\n",
    "sar_train_all = io.loadmat(train_path)\n",
    "sar_test = io.loadmat(test_path)\n",
    "sar_train_all = pd.DataFrame(sar_train_all['sarcos_inv'], columns=axis_names)\n",
    "sar_test = pd.DataFrame(sar_test['sarcos_inv_test'], columns=axis_names)\n",
    "sar_train = sar_train_all.iloc[:30000, :]\n",
    "\n",
    "x_train = sar_train.iloc[:,0:21]\n",
    "x_test = sar_test.iloc[:,0:21]\n",
    "y_train_all = sar_train.iloc[:,21:]\n",
    "y_test_all = sar_test.iloc[:,21:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scaling\n",
    "x_mean = x_train.mean()\n",
    "x_std = x_train.std()\n",
    "y_mean_all = y_train_all.mean()\n",
    "y_std_all = y_train_all.std()\n",
    "\n",
    "x_train_scal = (x_train - x_mean)/x_std\n",
    "x_test_scal = (x_test - x_mean)/x_std\n",
    "y_train_all_scal = (y_train_all - y_mean_all)/y_std_all\n",
    "y_test_all_scal = (y_test_all - y_mean_all)/y_std_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving dataframe\n",
    "df_result = pd.DataFrame(np.zeros([0,5]), columns=['Target name','Seed','MSE', 'Corr', 'MAE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_scatter(y_obs1_list, \n",
    "                 y_prd1_list,\n",
    "                 y_obs2_list, \n",
    "                 y_prd2_list,\n",
    "                 title_list, \n",
    "                 plt_row, \n",
    "                 plt_col, \n",
    "                 position_list, \n",
    "                 col_list1,\n",
    "                 alpha_list1,\n",
    "                 col_list2,\n",
    "                 alpha_list2,\n",
    "                 fig_size, \n",
    "                 save_name, \n",
    "                 title, \n",
    "                 show_flg=True):\n",
    "    fig = plt.figure(figsize=fig_size)\n",
    "\n",
    "    for i_plt in range(len(position_list)):\n",
    "        ax = fig.add_subplot(plt_row, plt_col, position_list[i_plt], \n",
    "                             title=title_list[i_plt], \n",
    "                             xlabel='Observation', \n",
    "                             ylabel='Prediction')\n",
    "        ax.scatter(y_obs1_list[i_plt], y_prd1_list[i_plt], color=col_list1[i_plt], alpha=alpha_list1[i_plt])\n",
    "        ax.scatter(y_obs2_list[i_plt], y_prd2_list[i_plt], color=col_list2[i_plt], alpha=alpha_list2[i_plt])\n",
    "        xy_min = min(ax.get_xlim()[0], ax.get_ylim()[0])\n",
    "        xy_max = max(ax.get_xlim()[1], ax.get_ylim()[1])\n",
    "        ax.axis('equal')\n",
    "        ax.axis('square')\n",
    "        ax.set_xlim([xy_min, xy_max])\n",
    "        ax.set_ylim([xy_min, xy_max])\n",
    "        ax.grid(color='gray', linestyle='dotted', linewidth=1, alpha=0.5)\n",
    "        ax.text(0.03, 0.93, 'Corr : '+str(round(np.corrcoef(y_prd2_list[i_plt], y_obs2_list[i_plt])[0,1], 4)), size=15, transform=ax.transAxes)\n",
    "        ax.text(0.03, 0.87, 'MSE : '+str(round(mean_squared_error(y_obs2_list[i_plt], y_prd2_list[i_plt]), 4)), size=15, transform=ax.transAxes)\n",
    "        ax.text(0.03, 0.81, 'MAE : '+str(round(mean_absolute_error(y_obs2_list[i_plt], y_prd2_list[i_plt]), 4)), size=15, transform=ax.transAxes)\n",
    "        _ = ax.plot([-300, 300], [-300, 300], color='gray', linewidth=0.5)\n",
    "\n",
    "    fig.tight_layout(rect=[0,0,1,0.96])\n",
    "    \n",
    "    plt.suptitle(title,fontsize=20)\n",
    "\n",
    "    fig.savefig(save_name)\n",
    "    if show_flg==False:\n",
    "        plt.close(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for target_name in target_name_list:\n",
    "    y_train_scal = y_train_all_scal[[s for s in target_name_list if s!=target_name]].copy()\n",
    "    y_test_scal = y_test_all_scal[[s for s in target_name_list if s!=target_name]].copy()\n",
    "\n",
    "    for i_seed in [373]:\n",
    "        ## Model definision\n",
    "        act = 'relu'\n",
    "        def make_model():\n",
    "            model_input = Input(shape=(21,))\n",
    "            h = model_input\n",
    "            h = Dense(units=256)(h)\n",
    "            h = Activation(act)(h)\n",
    "            \n",
    "            h = Dense(units=64)(h)\n",
    "            h = Activation(act)(h)\n",
    "            \n",
    "            h = Dense(units=32)(h)\n",
    "            h = Activation(act)(h)\n",
    "\n",
    "            h = Dense(units=16)(h)\n",
    "            h = Activation(act)(h)\n",
    "\n",
    "            out = Dense(units=6)(h) \n",
    "\n",
    "            model = Model(model_input, out)\n",
    "            optimizer = keras.optimizers.Adagrad(learning_rate=0.01)\n",
    "            model.compile(loss='mse',\n",
    "                         optimizer=optimizer,\n",
    "                         metrics=['mae', 'mse'])\n",
    "            return(model)\n",
    "        model = make_model()\n",
    "\n",
    "        fix_seed(373)\n",
    "        history = model.fit(\n",
    "            x_train_scal,\n",
    "            y_train_scal,\n",
    "            batch_size=1024,\n",
    "            epochs=1000,\n",
    "            validation_split = 0.2,\n",
    "            verbose=0,\n",
    "            callbacks=[early_stop, PrintDot()])\n",
    "\n",
    "        y_fits_scal = pd.DataFrame(model.predict(x_train_scal), index=np.arange(30000), columns=y_train_scal.columns)\n",
    "        y_fits = y_fits_scal*y_std_all[[s for s in target_name_list if s!=target_name]] + y_mean_all[[s for s in target_name_list if s!=target_name]]\n",
    "        y_pred_scal = pd.DataFrame(model.predict(x_test_scal), index=np.arange(4449), columns=y_test_scal.columns)\n",
    "        y_pred = y_pred_scal*y_std_all[[s for s in target_name_list if s!=target_name]] + y_mean_all[[s for s in target_name_list if s!=target_name]]\n",
    "        \n",
    "        name_layer = model.layers[7].name\n",
    "        model_Ext = Model(inputs=model.input, outputs=model.get_layer(name_layer).output)\n",
    "        ext_fits = pd.DataFrame(model_Ext.predict(x_train_scal), index=np.arange(30000))\n",
    "        ext_pred = pd.DataFrame(model_Ext.predict(x_test_scal),  index=np.arange(4449))\n",
    "\n",
    "        ## Plot\n",
    "        y_train_plt = y_train_all[[s for s in target_name_list if s!=target_name]].copy()\n",
    "        y_test_plt = y_test_all[[s for s in target_name_list if s!=target_name]].copy()\n",
    "        plot_scatter(\n",
    "            y_obs1_list = [y_train_plt.iloc[:,0], y_train_plt.iloc[:,1], y_train_plt.iloc[:,2], y_train_plt.iloc[:,3], y_train_plt.iloc[:,4], y_train_plt.iloc[:,5]],\n",
    "            y_prd1_list = [y_fits.iloc[:,0],      y_fits.iloc[:,1],      y_fits.iloc[:,2],      y_fits.iloc[:,3],      y_fits.iloc[:,4],      y_fits.iloc[:,5]],\n",
    "            y_obs2_list = [y_test_plt.iloc[:,0],  y_test_plt.iloc[:,1],  y_test_plt.iloc[:,2],  y_test_plt.iloc[:,3],  y_test_plt.iloc[:,4],  y_test_plt.iloc[:,5]],\n",
    "            y_prd2_list = [y_pred.iloc[:,0],      y_pred.iloc[:,1],      y_pred.iloc[:,2],      y_pred.iloc[:,3],      y_pred.iloc[:,4],      y_pred.iloc[:,5]],\n",
    "            title_list  = [y_train_plt.columns[0], y_train_plt.columns[1], y_train_plt.columns[2], y_train_plt.columns[3], y_train_plt.columns[4], y_train_plt.columns[5]],\n",
    "            plt_row     = 2,\n",
    "            plt_col     = 3,\n",
    "            position_list = [1, 2, 3, 4, 5, 6],\n",
    "            col_list1   = ['steelblue','steelblue','steelblue','steelblue','steelblue','steelblue'],\n",
    "            alpha_list1 = [1, 1, 1, 1, 1, 1, 1],\n",
    "            col_list2   = ['tomato', 'tomato', 'tomato', 'tomato', 'tomato', 'tomato'],\n",
    "            alpha_list2 = [1, 1, 1, 1, 1, 1, 1],\n",
    "            fig_size    = (15,10),\n",
    "            save_name   = '../30_Output/20_Plot/100_MakeSourceModel/100_Model_'+target_name+'-'+str(i_seed)+'.png',\n",
    "            title       = target_name,\n",
    "            show_flg    = False)\n",
    "\n",
    "        ### Model\n",
    "        json_string = model.to_json()\n",
    "        open('../30_Output/10_Model/100_MakeSourceModel/100_Model_'+target_name+'-'+str(i_seed)+'.json', 'w').write(json_string)\n",
    "        model.save_weights('../30_Output/10_Model/100_MakeSourceModel/100_Model_'+target_name+'-'+str(i_seed)+'.hdf5')\n",
    "        \n",
    "        ### Save\n",
    "        f = open('../30_Output/40_pkl/100_MakeSourceModel/110_Prediction_for_'+target_name+'-'+str(i_seed)+'.pkl','wb')\n",
    "        pickle.dump({\n",
    "            'output_train'  : y_fits_scal,\n",
    "            'output_test'   : y_pred_scal,\n",
    "            'feature_train' : ext_fits,\n",
    "            'feature_test'  : ext_pred\n",
    "        },f)\n",
    "        f.close()\n",
    "\n",
    "        K.clear_session()\n",
    "        del model\n",
    "        clear_output(True)\n",
    "\n",
    "clear_output(True)\n",
    "print('*** Succeeded ***')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
