{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Python packages used in this code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import os\n",
    "import pickle\n",
    "import time\n",
    "import sklearn\n",
    "import platform\n",
    "import sys\n",
    "from sklearn.base import BaseEstimator, RegressorMixin\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.linear_model import Ridge, LinearRegression\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "import joblib\n",
    "import itertools\n",
    "from IPython.display import clear_output\n",
    "\n",
    "## Keras\n",
    "import tensorflow\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Sequential, Model, model_from_json\n",
    "from tensorflow.keras.layers import Dense, Input, Add, Lambda, Dropout, Subtract, Multiply, Concatenate, Dot, BatchNormalization, Activation, LeakyReLU, ReLU\n",
    "from tensorflow.keras.losses import mse\n",
    "import keras.backend as K\n",
    "from tensorflow.keras import regularizers\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL']='2'\n",
    "tensorflow.get_logger().setLevel(\"ERROR\")\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Environments\n",
    "\n",
    "--Platform--\n",
    "OS : Windows-10-10.0.19044-SP0\n",
    "--Version--\n",
    "python :  3.9.12 (main, Apr  4 2022, 05:22:27) [MSC v.1916 64 bit (AMD64)]\n",
    "numpy : 1.23.1\n",
    "pandas : 1.4.3\n",
    "sklearn : 1.1.1\n",
    "tensorflow : 2.9.1\n",
    "keras : 2.9.0\n",
    "\"\"\"\n",
    "\n",
    "print('--Platform--')\n",
    "print('OS :', platform.platform())\n",
    "print('--Version--')\n",
    "print('python : ', sys.version)\n",
    "print('numpy :', np.__version__)\n",
    "print('pandas :', pd.__version__)\n",
    "print('sklearn :', sklearn.__version__)\n",
    "print('tensorflow :', tensorflow.__version__)\n",
    "print('keras :', keras.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Define the model class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Without transfer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class cls_WithoutTL_NN(BaseEstimator, RegressorMixin):\n",
    "    def __init__(self, dr_rate=0, layer1=4, decay_rate1=0.5, decay_rate2=0.5, l2_lambda=0.01, learning_rate=0.01, epochs=10):\n",
    "        \"\"\"\n",
    "        Define the neural network model without transfer.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "            dr_rate       : dropout rate\n",
    "            layer1        : width of the 1st layer\n",
    "            decay_rate1   : rate of the width decay from the 1st layer to the 2nd layer\n",
    "            decay_rate2   : rate of the width decay from the 2nd layer to the last layer\n",
    "            l2_lambda     : l2-reguralization parameter\n",
    "            learning_rate : learning rate for Adam\n",
    "            epochs        : learning epochs            \n",
    "        \"\"\"\n",
    "        self.dr_rate = dr_rate\n",
    "        self.layer1 = layer1\n",
    "        self.decay_rate1 = decay_rate1\n",
    "        self.decay_rate2 = decay_rate2\n",
    "        self.l2_lambda = l2_lambda\n",
    "        self.learning_rate = learning_rate\n",
    "        self.epochs = epochs\n",
    "\n",
    "    def prop_make_model(self):\n",
    "        \"\"\"\n",
    "        Making the neural network model\n",
    "        \"\"\"\n",
    "        input1 = Input(shape=(dim_x,))\n",
    "\n",
    "        h = Dense(units=self.layer1, kernel_regularizer=regularizers.L2(l2=self.l2_lambda))(input1)\n",
    "        h = LeakyReLU(alpha=0.01)(h)\n",
    "        h = Dropout(self.dr_rate)(h)\n",
    "        \n",
    "        h = Dense(units=self.layer2, kernel_regularizer=regularizers.L2(l2=self.l2_lambda))(h)\n",
    "        h = LeakyReLU(alpha=0.01)(h)\n",
    "        h = Dropout(self.dr_rate)(h)\n",
    "        \n",
    "        out = Dense(units=1, kernel_regularizer=regularizers.L2(l2=self.l2_lambda))(h)\n",
    "\n",
    "        self.model = Model(inputs=input1, outputs=out)\n",
    "        optimizer = keras.optimizers.Adam(self.learning_rate)\n",
    "        self.model.compile(loss='mse',\n",
    "                     optimizer=optimizer,\n",
    "                     metrics=['mae', 'mse'])\n",
    "        return self\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        \"\"\"\n",
    "        Model fitting\n",
    "        \n",
    "        Required grobal variables\n",
    "        -----------------------\n",
    "            dim_x  : dimension of the input\n",
    "            b_size : batch size\n",
    "        \n",
    "        Returns\n",
    "        -------\n",
    "            layer1 : width of the 1st layer\n",
    "            layer2 : width of the 2nd layer\n",
    "            \n",
    "            model   : neural network model\n",
    "            history : training history\n",
    "        \"\"\"\n",
    "        self.layer1 = int(np.floor(dim_x*self.decay_rate1))\n",
    "        self.layer2 = int(np.floor(self.layer1*self.decay_rate2))\n",
    "        fix_seed(373)\n",
    "        self.prop_make_model()\n",
    "        self.history = self.model.fit(\n",
    "            X,\n",
    "            y,\n",
    "            batch_size=b_size,\n",
    "            epochs=self.epochs,\n",
    "            validation_split = 0,\n",
    "            verbose=0\n",
    "        )\n",
    "        K.clear_session()\n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        \"\"\"\n",
    "        Prediction\n",
    "        \"\"\"\n",
    "        return self.model.predict(X, verbose=0)         \n",
    "                             \n",
    "    def score(self, X, y=None):\n",
    "        \"\"\"\n",
    "        Score function for cross-validation\n",
    "        \"\"\"\n",
    "        return -np.sum((y.values - self.predict(X).reshape(-1))**2)\n",
    "    \n",
    "    def get_params(self, deep=True):\n",
    "        \"\"\"\n",
    "        Create parameter dictionary for cross-validation\n",
    "        \"\"\"\n",
    "        return {\n",
    "            'dr_rate' : self.dr_rate,\n",
    "            'layer1' : self.layer1,\n",
    "            'decay_rate1' : self.decay_rate1,\n",
    "            'decay_rate2' : self.decay_rate2,\n",
    "            'l2_lambda' : self.l2_lambda,\n",
    "            'learning_rate' : self.learning_rate,\n",
    "            'epochs' : self.epochs\n",
    "        }\n",
    "    \n",
    "    def set_params(self, **parameters):\n",
    "        \"\"\"\n",
    "        For cross-validation\n",
    "        \"\"\"\n",
    "        for parameter, value in parameters.items():\n",
    "            setattr(self, parameter, value)\n",
    "        return self  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature extract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class cls_FeatureExt_NN(BaseEstimator, RegressorMixin):\n",
    "    def __init__(self, dr_rate=0, layer1=4, decay_rate1=0.5, decay_rate2=0.5, l1_lambda=0.01, l2_lambda=0.01, learning_rate=0.01, epochs=10):\n",
    "        \"\"\"\n",
    "        Define the neural network model for the feature extraction.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "            dr_rate       : dropout rate\n",
    "            layer1        : width of the 1st layer\n",
    "            decay_rate1   : rate of the width decay from the 1st layer to the 2nd layer\n",
    "            decay_rate2   : rate of the width decay from the 2nd layer to the last layer\n",
    "            l1_lambda     : l1-reguralization parameter\n",
    "            l2_lambda     : l2-reguralization parameter\n",
    "            learning_rate : learning rate for Adam\n",
    "            epochs        : learning epochs            \n",
    "        \"\"\"\n",
    "        self.dr_rate       = dr_rate\n",
    "        self.layer1        = layer1\n",
    "        self.decay_rate1   = decay_rate1\n",
    "        self.decay_rate2   = decay_rate2\n",
    "        self.l1_lambda     = l1_lambda\n",
    "        self.l2_lambda     = l2_lambda\n",
    "        self.learning_rate = learning_rate\n",
    "        self.epochs        = epochs\n",
    "\n",
    "    def prop_make_model(self):\n",
    "        \"\"\"\n",
    "        Making the neural network model\n",
    "        \"\"\"\n",
    "        input1 = Input(shape=(num_SourceTasks,))\n",
    "\n",
    "        h = Dense(units=self.layer1, kernel_regularizer=regularizers.L2(l2=self.l2_lambda))(input1)\n",
    "        h = LeakyReLU(alpha=0.01)(h)\n",
    "        h = Dropout(self.dr_rate)(h)\n",
    "        \n",
    "        h = Dense(units=self.layer2, kernel_regularizer=regularizers.L2(l2=self.l2_lambda))(h)\n",
    "        h = LeakyReLU(alpha=0.01)(h)\n",
    "        h = Dropout(self.dr_rate)(h)\n",
    "        \n",
    "        out = Dense(units=1, kernel_regularizer=regularizers.L2(l2=self.l2_lambda))(h)\n",
    "\n",
    "        self.model = Model(inputs=input1, outputs=out)\n",
    "        optimizer = keras.optimizers.Adam(self.learning_rate)\n",
    "        self.model.compile(loss='mse',\n",
    "                     optimizer=optimizer,\n",
    "                     metrics=['mae', 'mse'])\n",
    "        return self\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        \"\"\"\n",
    "        Model fitting\n",
    "        \n",
    "        Required grobal variables\n",
    "        -----------------------\n",
    "            dim_x : dimension of the input\n",
    "            b_size : batch size\n",
    "        \n",
    "        Returns\n",
    "        -------\n",
    "            layer1 : width of the 1st layer\n",
    "            layer2 : width of the 2nd layer\n",
    "            \n",
    "            model : neural network model\n",
    "            history : training history\n",
    "        \"\"\"\n",
    "        self.layer1 = int(np.floor(num_SourceTasks*self.decay_rate1))\n",
    "        self.layer2 = int(np.floor(self.layer1*self.decay_rate2))\n",
    "        fix_seed(373)\n",
    "        self.prop_make_model()\n",
    "        self.history = self.model.fit(\n",
    "            X,\n",
    "            y,\n",
    "            batch_size = b_size,\n",
    "            epochs = self.epochs,\n",
    "            validation_split = 0,\n",
    "            verbose = 0\n",
    "        )\n",
    "        K.clear_session()\n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        \"\"\"\n",
    "        Prediction\n",
    "        \"\"\"\n",
    "        return self.model.predict(X, verbose=0)         \n",
    "                             \n",
    "    def score(self, X, y=None):\n",
    "        \"\"\"\n",
    "        Score function for cross-validation\n",
    "        \"\"\"\n",
    "        return -np.sum((y.values - self.predict(X).reshape(-1))**2)\n",
    "    \n",
    "    def get_params(self, deep=True):\n",
    "        \"\"\"\n",
    "        Create parameter dictionary for cross-validation\n",
    "        \"\"\"\n",
    "        return {\n",
    "            'dr_rate'       : self.dr_rate,\n",
    "            'layer1'        : self.layer1,\n",
    "            'decay_rate1'   : self.decay_rate1,\n",
    "            'decay_rate2'   : self.decay_rate2,\n",
    "            'l2_lambda'     : self.l2_lambda,\n",
    "            'learning_rate' : self.learning_rate,\n",
    "            'epochs'        : self.epochs\n",
    "        }\n",
    "    \n",
    "    def set_params(self, **parameters):\n",
    "        \"\"\"\n",
    "        For cross-validation\n",
    "        \"\"\"\n",
    "        for parameter, value in parameters.items():\n",
    "            setattr(self, parameter, value)\n",
    "        return self     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Proposed method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class cls_InvTrans_NN(BaseEstimator, RegressorMixin):\n",
    "    def __init__(self, dr_rate=0, layer1=4, layer2=4, layer3=4, \n",
    "                 decay_rate1=0.5, decay_rate2=0.5, decay_rate3=0.5,\n",
    "                 l2_lambda1=0.01, l2_lambda2=0.01, learning_rate=0.01, epochs=10):\n",
    "        \"\"\"\n",
    "        Define the neural network model for affine transfer model.\n",
    "            y = g_1(f_s(x)) + g_2(f_s(x)) g_3(x)\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "            dr_rate       : dropout rate\n",
    "            layer1        : width of the 1st layer of g1\n",
    "            layer2        : width of the 1st layer of g2\n",
    "            layer3        : width of the 1st layer of g3\n",
    "            decay_rate1   : rate of the width decay for the g1 network\n",
    "            decay_rate2   : rate of the width decay for the g2 network\n",
    "            decay_rate3   : rate of the width decay for the g3 network\n",
    "            l2_lambda1    : l2-reguralization parameter for g1 and g2 network\n",
    "            l2_lambda2    : l2-reguralization parameter for g3 network\n",
    "            learning_rate : learning rate for Adam\n",
    "            epochs        : learning epochs            \n",
    "        \"\"\"\n",
    "        self.dr_rate       = dr_rate\n",
    "        self.layer1        = layer1\n",
    "        self.layer2        = layer2\n",
    "        self.layer3        = layer3\n",
    "        self.decay_rate1   = decay_rate1\n",
    "        self.decay_rate2   = decay_rate2\n",
    "        self.decay_rate3   = decay_rate3\n",
    "        self.l2_lambda1    = l2_lambda1\n",
    "        self.l2_lambda2    = l2_lambda2\n",
    "        self.learning_rate = learning_rate\n",
    "        self.epochs        = epochs\n",
    "\n",
    "    def prop_make_model(self):\n",
    "        \"\"\"\n",
    "        Making the neural network model\n",
    "        \"\"\"\n",
    "        input1 = Input(shape=(num_SourceTasks,))\n",
    "        input2 = Input(shape=(dim_x,))\n",
    "\n",
    "        h1 = Dense(units=self.layer1, kernel_regularizer=regularizers.L2(l2=self.l2_lambda1))(input1)\n",
    "        h1 = LeakyReLU(alpha=0.01)(h1)\n",
    "        h1 = Dropout(self.dr_rate)(h1)\n",
    "        h1 = Dense(units=1, kernel_regularizer=regularizers.L2(l2=self.l2_lambda1))(h1)\n",
    "        \n",
    "        h2 = Dense(units=self.layer2, kernel_regularizer=regularizers.L2(l2=self.l2_lambda1))(input1)\n",
    "        h2 = LeakyReLU(alpha=0.01)(h2)\n",
    "        h2 = Dropout(self.dr_rate)(h2)\n",
    "        h2 = Dense(units=1, kernel_regularizer=regularizers.L2(l2=self.l2_lambda1))(h2)\n",
    "        \n",
    "        h = Dense(units=self.layer3, kernel_regularizer=regularizers.L2(l2=self.l2_lambda2))(input2)\n",
    "        h = LeakyReLU(alpha=0.01)(h)\n",
    "        h = Dropout(self.dr_rate)(h)\n",
    "        h = Dense(units=1, kernel_regularizer=regularizers.L2(l2=self.l2_lambda2))(h)\n",
    "\n",
    "        h = Multiply()([h2, h])\n",
    "        out = Add()([h1, h])\n",
    "\n",
    "        self.model = Model(inputs=[input1, input2], outputs=out)\n",
    "\n",
    "        optimizer = keras.optimizers.Adam(self.learning_rate)\n",
    "\n",
    "        self.model.compile(\n",
    "            loss='mse',\n",
    "            optimizer=optimizer,\n",
    "            metrics=['mae', 'mse'])\n",
    "        return self\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        self.X_train = X.iloc[:,:dim_x]\n",
    "        self.X_source = X.iloc[:,dim_x:]\n",
    "        \n",
    "        self.layer1_2 = int(np.floor(self.layer1*self.decay_rate1))\n",
    "        self.layer2_2 = int(np.floor(self.layer2*self.decay_rate2))\n",
    "        self.layer3_2 = int(np.floor(self.layer3*self.decay_rate3))\n",
    "        self.layer1   = int(np.floor(num_SourceTasks*self.decay_rate1))\n",
    "        self.layer2   = int(np.floor(num_SourceTasks*self.decay_rate1))\n",
    "        self.layer3   = int(np.floor(dim_x*self.decay_rate3))\n",
    "        \n",
    "        fix_seed(373)\n",
    "        self.prop_make_model()\n",
    "        self.history = self.model.fit(\n",
    "            [self.X_source, self.X_train],\n",
    "            y,\n",
    "            batch_size = b_size,\n",
    "            epochs = self.epochs,\n",
    "            validation_split = 0,\n",
    "            verbose = 0\n",
    "        )\n",
    "        K.clear_session()\n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        X_source_pred = X.iloc[:,dim_x:]\n",
    "        X_train_pred = X.iloc[:,:dim_x]\n",
    "        return self.model.predict([X_source_pred, X_train_pred], verbose=0)\n",
    "\n",
    "    def score(self, X, y=None):\n",
    "        return -np.sum((y.values - self.predict(X).reshape(-1))**2)\n",
    "    \n",
    "    def get_params(self, deep=True):\n",
    "        return {\n",
    "            'dr_rate'       : self.dr_rate,\n",
    "            'layer1'        : self.layer1,\n",
    "            'layer2'        : self.layer2,\n",
    "            'layer3'        : self.layer3,\n",
    "            'decay_rate1'   : self.decay_rate1,\n",
    "            'decay_rate2'   : self.decay_rate2,\n",
    "            'decay_rate3'   : self.decay_rate3,\n",
    "            'l2_lambda1'    : self.l2_lambda1,\n",
    "            'l2_lambda2'    : self.l2_lambda2,\n",
    "            'learning_rate' : self.learning_rate,\n",
    "            'epochs'        : self.epochs\n",
    "        }\n",
    "    \n",
    "    def set_params(self, **parameters):\n",
    "        for parameter, value in parameters.items():\n",
    "            setattr(self, parameter, value)\n",
    "        return self "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class cls_FineTune_NN(BaseEstimator, RegressorMixin):\n",
    "    def __init__(self, learning_rate=0.01, epochs=10, n_frozen=0):\n",
    "        \"\"\"\n",
    "        Define the neural network model for fine-tuning model.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "            learning_rate : learning rate for Adam\n",
    "            epochs        : learning epochs            \n",
    "            n_frozen      : number of frozen layers\n",
    "        \"\"\"\n",
    "        self.learning_rate = learning_rate\n",
    "        self.epochs        = epochs\n",
    "        self.n_frozen      = n_frozen\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        \"\"\"\n",
    "        Model fitting\n",
    "        \n",
    "        Required grobal variables\n",
    "        -----------------------\n",
    "            dim_x : dimension of the input\n",
    "            b_size : batch size\n",
    "        \n",
    "        Returns\n",
    "        -------\n",
    "            layer1 : width of the 1st layer\n",
    "            layer2 : width of the 2nd layer\n",
    "            \n",
    "            model : neural network model\n",
    "            history : training history\n",
    "        \"\"\"\n",
    "        self.model = model_from_json(open(source_model_path_json, 'r').read())\n",
    "        self.model.load_weights(source_model_path_hdf5)\n",
    "        name_layers_dense = [self.model.layers[1].name, self.model.layers[3].name, self.model.layers[5].name]\n",
    "        if self.n_frozen > 0:\n",
    "            for i_frz in np.arange(self.n_frozen):\n",
    "                self.model.get_layer(name_layers_dense).trainable = False\n",
    "\n",
    "        self.model.compile(loss='mse',\n",
    "                     optimizer=keras.optimizers.Adam(self.learning_rate),\n",
    "                     metrics=['mae', 'mse'])\n",
    "        \n",
    "        fix_seed(373)\n",
    "        self.history = self.model.fit(\n",
    "            X,\n",
    "            y,\n",
    "            batch_size = 2,\n",
    "            epochs = self.epochs,\n",
    "            validation_split = 0,\n",
    "            verbose = 0\n",
    "        )\n",
    "        K.clear_session()\n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        \"\"\"\n",
    "        Prediction\n",
    "        \"\"\"\n",
    "        return self.model.predict(X, verbose=0)\n",
    "\n",
    "    def score(self, X, y=None):\n",
    "        \"\"\"\n",
    "        Score function for cross-validation\n",
    "        \"\"\"\n",
    "        return -np.sum((y.values - self.predict(X).reshape(-1))**2)\n",
    "    \n",
    "    def get_params(self, deep=True):\n",
    "        \"\"\"\n",
    "        Create parameter dictionary for cross-validation\n",
    "        \"\"\"\n",
    "        return {\n",
    "            'learning_rate' : self.learning_rate,\n",
    "            'epochs'        : self.epochs\n",
    "        }\n",
    "    \n",
    "    def set_params(self, **parameters):\n",
    "        \"\"\"\n",
    "        For cross-validation\n",
    "        \"\"\"\n",
    "        for parameter, value in parameters.items():\n",
    "            setattr(self, parameter, value)\n",
    "        return self  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fix_seed function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_seed(seed):\n",
    "    # Numpy\n",
    "    np.random.seed(seed)\n",
    "    # Tensorflow\n",
    "    tensorflow.random.set_seed(seed)\n",
    "    # for built-in random\n",
    "    random.seed(seed)\n",
    "    # for hash seed\n",
    "    os.environ[\"PYTHONHASHSEED\"] = str(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make figure function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MakeFigure_Train(figsize,\n",
    "                     dict_result,\n",
    "                     savename,\n",
    "                     plt_show=True):\n",
    "    \n",
    "    plt_row = 2\n",
    "    plt_col = 4\n",
    "    \n",
    "    fig = plt.figure(figsize=figsize)\n",
    "\n",
    "    y_prd1 = dict_result['y_fits_Wotl']\n",
    "    y_obs1 = dict_result['y_train']\n",
    "    y_prd2 = dict_result['y_pred_Wotl']\n",
    "    y_obs2 = dict_result['y_test']\n",
    "    ax = fig.add_subplot(plt_row, plt_col, 1, \n",
    "                         title='Without transfer', \n",
    "                         xlabel='Prediction', \n",
    "                         ylabel='Observation')\n",
    "    ax.scatter(y_prd1, y_obs1, color='steelblue', alpha=0.7)\n",
    "    ax.scatter(y_prd2, y_obs2, color='darkorange', alpha=1)\n",
    "    xy_min = min(ax.get_xlim()[0], ax.get_ylim()[0])\n",
    "    xy_max = max(ax.get_xlim()[1], ax.get_ylim()[1])\n",
    "    ax.axis('equal')\n",
    "    ax.axis('square')\n",
    "    ax.set_xlim([xy_min, xy_max])\n",
    "    ax.set_ylim([xy_min, xy_max])\n",
    "    ax.grid(color='gray', linestyle='dotted', linewidth=1, alpha=0.5)\n",
    "    ax.text(0.53, 0.14, 'Corr : '+str(round(np.corrcoef(y_prd2, y_obs2)[0,1], 4)), size=15, transform=ax.transAxes)\n",
    "    ax.text(0.53, 0.08, 'MSE : '+str(round(np.mean((y_prd2-y_obs2)**2), 4)), size=15, transform=ax.transAxes)\n",
    "    ax.text(0.53, 0.02, 'MAE : '+str(round(np.mean(np.abs(y_prd2-y_obs2)), 4)), size=15, transform=ax.transAxes)\n",
    "    _ = ax.plot([-30000, 30000], [-30000, 30000], color='gray', linewidth=0.5)\n",
    "\n",
    "    for plt_layer in [1,2,3]:\n",
    "        y_prd1 = dict_result['y_fits_Ext'+str(plt_layer)]\n",
    "        y_obs1 = dict_result['y_train']\n",
    "        y_prd2 = dict_result['y_pred_Ext'+str(plt_layer)]\n",
    "        y_obs2 = dict_result['y_test']\n",
    "        ax = fig.add_subplot(plt_row, plt_col, 1+plt_layer, \n",
    "                             title='Feature extractor (neural network) : Layer '+str(plt_layer), \n",
    "                             xlabel='Prediction', \n",
    "                             ylabel='Observation')\n",
    "        ax.scatter(y_prd1, y_obs1, color='steelblue', alpha=0.7)\n",
    "        ax.scatter(y_prd2, y_obs2, color='darkorange', alpha=1)\n",
    "        xy_min = min(ax.get_xlim()[0], ax.get_ylim()[0])\n",
    "        xy_max = max(ax.get_xlim()[1], ax.get_ylim()[1])\n",
    "        ax.axis('equal')\n",
    "        ax.axis('square')\n",
    "        ax.set_xlim([xy_min, xy_max])\n",
    "        ax.set_ylim([xy_min, xy_max])\n",
    "        ax.grid(color='gray', linestyle='dotted', linewidth=1, alpha=0.5)\n",
    "        ax.text(0.53, 0.14, 'Corr : '+str(round(np.corrcoef(y_prd2, y_obs2)[0,1], 4)), size=15, transform=ax.transAxes)\n",
    "        ax.text(0.53, 0.08, 'MSE : '+str(round(np.mean((y_prd2-y_obs2)**2), 4)), size=15, transform=ax.transAxes)\n",
    "        ax.text(0.53, 0.02, 'MAE : '+str(round(np.mean(np.abs(y_prd2-y_obs2)), 4)), size=15, transform=ax.transAxes)\n",
    "        _ = ax.plot([-30000, 30000], [-30000, 30000], color='gray', linewidth=0.5)\n",
    "\n",
    "        y_prd1 = dict_result['y_fits_InvNN'+str(plt_layer)]\n",
    "        y_obs1 = dict_result['y_train']\n",
    "        y_prd2 = dict_result['y_pred_InvNN'+str(plt_layer)]\n",
    "        y_obs2 = dict_result['y_test']\n",
    "        ax = fig.add_subplot(plt_row, plt_col, 5+plt_layer, \n",
    "                             title='Proposed method (neural network) : Layer '+str(plt_layer), \n",
    "                             xlabel='Prediction', \n",
    "                             ylabel='Observation')\n",
    "        ax.scatter(y_prd1, y_obs1, color='steelblue', alpha=0.7)\n",
    "        ax.scatter(y_prd2, y_obs2, color='darkorange', alpha=1)\n",
    "        xy_min = min(ax.get_xlim()[0], ax.get_ylim()[0])\n",
    "        xy_max = max(ax.get_xlim()[1], ax.get_ylim()[1])\n",
    "        ax.axis('equal')\n",
    "        ax.axis('square')\n",
    "        ax.set_xlim([xy_min, xy_max])\n",
    "        ax.set_ylim([xy_min, xy_max])\n",
    "        ax.grid(color='gray', linestyle='dotted', linewidth=1, alpha=0.5)\n",
    "        ax.text(0.53, 0.14, 'Corr : '+str(round(np.corrcoef(y_prd2, y_obs2)[0,1], 4)), size=15, transform=ax.transAxes)\n",
    "        ax.text(0.53, 0.08, 'MSE : '+str(round(np.mean((y_prd2-y_obs2)**2), 4)), size=15, transform=ax.transAxes)\n",
    "        ax.text(0.53, 0.02, 'MAE : '+str(round(np.mean(np.abs(y_prd2-y_obs2)), 4)), size=15, transform=ax.transAxes)\n",
    "        _ = ax.plot([-30000, 30000], [-30000, 30000], color='gray', linewidth=0.5)\n",
    "\n",
    "    y_prd1 = dict_result['y_fits_Fine']\n",
    "    y_obs1 = dict_result['y_train']\n",
    "    y_prd2 = dict_result['y_pred_Fine']\n",
    "    y_obs2 = dict_result['y_test']\n",
    "    ax = fig.add_subplot(plt_row, plt_col, 5, \n",
    "                         title='Fine-tuning', \n",
    "                         xlabel='Prediction', \n",
    "                         ylabel='Observation')\n",
    "    ax.scatter(y_prd1, y_obs1, color='steelblue', alpha=0.7)\n",
    "    ax.scatter(y_prd2, y_obs2, color='darkorange', alpha=1)\n",
    "    xy_min = min(ax.get_xlim()[0], ax.get_ylim()[0])\n",
    "    xy_max = max(ax.get_xlim()[1], ax.get_ylim()[1])\n",
    "    ax.axis('equal')\n",
    "    ax.axis('square')\n",
    "    ax.set_xlim([xy_min, xy_max])\n",
    "    ax.set_ylim([xy_min, xy_max])\n",
    "    ax.grid(color='gray', linestyle='dotted', linewidth=1, alpha=0.5)\n",
    "    ax.text(0.53, 0.14, 'Corr : '+str(round(np.corrcoef(y_prd2, y_obs2)[0,1], 4)), size=15, transform=ax.transAxes)\n",
    "    ax.text(0.53, 0.08, 'MSE : '+str(round(np.mean((y_prd2-y_obs2)**2), 4)), size=15, transform=ax.transAxes)\n",
    "    ax.text(0.53, 0.02, 'MAE : '+str(round(np.mean(np.abs(y_prd2-y_obs2)), 4)), size=15, transform=ax.transAxes)\n",
    "    _ = ax.plot([-30000, 30000], [-30000, 30000], color='gray', linewidth=0.5)\n",
    "\n",
    "    fig.tight_layout(rect=[0,0,1,0.96])\n",
    "\n",
    "    # plt.suptitle(title,fontsize=20)\n",
    "\n",
    "    fig.savefig(savename)\n",
    "    if plt_show == False:\n",
    "        plt.close(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Search parameters in cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For without transfer\n",
    "SearchParams_WithoutTL_NN = {\n",
    "    'dr_rate' : [0.25, 0.5],\n",
    "    'layer1' : [1],\n",
    "    'decay_rate1': [0.5, 0.75],\n",
    "    'decay_rate2': [0.5, 0.75],\n",
    "    'l2_lambda' : [1e-2],\n",
    "    'learning_rate' : [1e-3],\n",
    "    'epochs' : [50, 75, 100]\n",
    "}\n",
    "\n",
    "# For feature extractor\n",
    "SearchParams_FeatureExt_NN = {\n",
    "    'dr_rate' : [0.25, 0.5],\n",
    "    'layer1' : [1],\n",
    "    'decay_rate1': [0.5, 0.75],\n",
    "    'decay_rate2': [0.5, 0.75],\n",
    "    'l2_lambda' : [1e-2],\n",
    "    'learning_rate' : [1e-3],\n",
    "    'epochs' : [50, 75, 100]\n",
    "}\n",
    "\n",
    "# For proposed method\n",
    "SearchParams_InvTrans_NN = {\n",
    "    'dr_rate' : [0.5], \n",
    "    'layer1' : [1],\n",
    "    'layer2' : [1],\n",
    "    'layer3' : [1],\n",
    "    'decay_rate1' : [0.25, 0.5, 0.75], \n",
    "    'decay_rate2' : [1], \n",
    "    'decay_rate3' : [0.25, 0.5, 0.75], \n",
    "    'l2_lambda1' : [1e-2],\n",
    "    'l2_lambda2' : [1e-2],\n",
    "    'learning_rate' : [1e-3], \n",
    "    'epochs' : [50, 75, 100]\n",
    "}\n",
    "\n",
    "SearchParams_FineTune_NN = {\n",
    "    'learning_rate' : [1e-3],\n",
    "    'epochs' : [0, 1, 5, 10, 20, 30, 40, 50],\n",
    "    'n_frozen' : [0]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main codes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_list = ['min:gs_mag_moment','min:num_d_unfilled','min:num_f_unfilled','min:num_f_valence','ave:num_f_unfilled','ave:num_f_valence','sum:num_f_unfilled','sum:num_f_valence','var:num_f_unfilled','var:num_f_valence','max:num_f_unfilled','max:num_f_valence']\n",
    "data_all = joblib.load('../10_Data/SPSTC_290.pkl')\n",
    "x_all = data_all['desc']\n",
    "x_all = x_all.drop(drop_list, axis=1)\n",
    "y_all = data_all['data']\n",
    "y_LTC = y_all[-y_all['TC (W/mK)'].isna()]['TC (W/mK)']\n",
    "x_LTC = x_all[-y_all['TC (W/mK)'].isna()]\n",
    "\n",
    "dim_x = 290 - len(drop_list)\n",
    "b_size = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Scaling parameters\n",
    "x_mean = x_all.mean()\n",
    "x_std = x_all.std()\n",
    "y_LogMean = np.log(y_LTC).mean()\n",
    "y_LogStd = np.log(y_LTC).std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data scaling\n",
    "## Input\n",
    "xs_LTC_scal = (x_LTC - x_mean)/x_std\n",
    "x_LTC_scal = (x_LTC - x_mean)/x_std\n",
    "## Output\n",
    "y_LTC_scal = (np.log(y_LTC) - y_LogMean)/y_LogStd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## User parameter setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of iterations\n",
    "num_itr = 10\n",
    "# Number of fold of cross-validation\n",
    "n_fold = 5\n",
    "# seed for fix_seed function\n",
    "SEED = 373\n",
    "# Source model id\n",
    "i_list_source = [\n",
    "    66,\n",
    "    83,\n",
    "    39,\n",
    "    36,\n",
    "    70,\n",
    "    95,\n",
    "    56,\n",
    "    72,\n",
    "    69,\n",
    "    42\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List for storing the results\n",
    "list_result = list()\n",
    "df_result = pd.DataFrame([], columns=[\n",
    "    'No. source model', \n",
    "    'Itr', \n",
    "    'Count',\n",
    "    'MSE (train, Wotl)',\n",
    "    'MSE (train, Fine)',\n",
    "    'MSE (train, Ext-1)', \n",
    "    'MSE (train, Ext-2)', \n",
    "    'MSE (train, Ext-3)', \n",
    "    'MSE (train, Inv-1)', \n",
    "    'MSE (train, Inv-2)', \n",
    "    'MSE (train, Inv-3)',\n",
    "    'MSE (test, Wotl)',\n",
    "    'MSE (test, Fine)',\n",
    "    'MSE (test, Ext-1)', \n",
    "    'MSE (test, Ext-2)', \n",
    "    'MSE (test, Ext-3)', \n",
    "    'MSE (test, Inv-1)', \n",
    "    'MSE (test, Inv-2)', \n",
    "    'MSE (test, Inv-3)'\n",
    "])\n",
    "i_count = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "t1 = time.time()\n",
    "# Repeat for different splittings of samples\n",
    "for i_itr in range(num_itr):\n",
    "    print(i_itr)\n",
    "    x_train_scal, x_test_scal, xs_train_scal, xs_test_scal, y_train_scal, y_test_scal = train_test_split(x_LTC_scal, xs_LTC_scal, y_LTC_scal, train_size=40, random_state=int(i_itr))\n",
    "    y_train = np.exp(y_train_scal*y_LogStd + y_LogMean)\n",
    "    y_test = np.exp(y_test_scal*y_LogStd + y_LogMean)\n",
    "    \n",
    "    # Model training\n",
    "    ## Without transfer\n",
    "    gsr_Wotl = GridSearchCV(\n",
    "        cls_WithoutTL_NN(),\n",
    "        SearchParams_WithoutTL_NN,\n",
    "        scoring = 'neg_mean_squared_error',\n",
    "        cv = n_fold,\n",
    "        n_jobs = -1,\n",
    "        verbose = False\n",
    "    )\n",
    "    t_tmp = time.time()\n",
    "    fix_seed(SEED)\n",
    "    gsr_Wotl.fit(x_train_scal, y_train_scal)\n",
    "\n",
    "    model_Wotl = cls_WithoutTL_NN(\n",
    "        dr_rate       = gsr_Wotl.best_params_['dr_rate'],\n",
    "        layer1        = gsr_Wotl.best_params_['layer1'],\n",
    "        decay_rate1   = gsr_Wotl.best_params_['decay_rate1'],\n",
    "        decay_rate2   = gsr_Wotl.best_params_['decay_rate2'],\n",
    "        l2_lambda     = gsr_Wotl.best_params_['l2_lambda'],\n",
    "        learning_rate = gsr_Wotl.best_params_['learning_rate'],\n",
    "        epochs        = gsr_Wotl.best_params_['epochs']\n",
    "    )\n",
    "    fix_seed(SEED)\n",
    "    model_Wotl.fit(x_train_scal, y_train_scal)\n",
    "\n",
    "    y_fits_Wotl_scal = model_Wotl.predict(x_train_scal)\n",
    "    y_pred_Wotl_scal = model_Wotl.predict(x_test_scal)\n",
    "    y_fits_Wotl      = np.exp(y_fits_Wotl_scal*y_LogStd+y_LogMean)\n",
    "    y_pred_Wotl      = np.exp(y_pred_Wotl_scal*y_LogStd+y_LogMean)\n",
    "    y_fits_Wotl      = pd.Series(y_fits_Wotl.reshape(-1), index=y_train.index)\n",
    "    y_pred_Wotl      = pd.Series(y_pred_Wotl.reshape(-1), index=y_test.index)\n",
    "    train_MSE_Wotl   = np.mean((y_fits_Wotl - y_train)**2)\n",
    "    test_MSE_Wotl    = np.mean((y_pred_Wotl - y_test)**2)\n",
    "    print('  Without transfer has been done.    '+str(time.time()-t_tmp))\n",
    "    \n",
    "    # Repeat for different source models\n",
    "    for i_source in i_list_source:\n",
    "        if not os.path.isdir('../30_Output/20_Plot/200_TransferLearning/Source'+str(i_source)):\n",
    "            os.makedirs('../30_Output/20_Plot/200_TransferLearning/Source'+str(i_source))\n",
    "        if not os.path.isdir('../30_Output/30_csv/200_TransferLearning'):\n",
    "            os.makedirs('../30_Output/30_csv/200_TransferLearning')\n",
    "        if not os.path.isdir('../30_Output/40_pkl/200_TransferLearning/Source'+str(i_source)):\n",
    "            os.makedirs('../30_Output/40_pkl/200_TransferLearning/Source'+str(i_source))\n",
    "\n",
    "        # Load source model\n",
    "        with open('../30_Output/40_pkl/100_MakeSourceModel/100_Model_'+str(i_source)+'.pkl', 'rb') as f:\n",
    "            data_list = pickle.load(f)\n",
    "        source_model_path_json = '../30_Output/10_Model/100_MakeSourceModel/100_Model_'+str(i_source)+'.json'\n",
    "        source_model_path_hdf5 = '../30_Output/10_Model/100_MakeSourceModel/100_Model_'+str(i_source)+'.hdf5'\n",
    "        num_SourceTasks = int(data_list['width_layers'][0])\n",
    "        source_model = model_from_json(open(source_model_path_json, 'r').read())\n",
    "        source_model.load_weights(source_model_path_hdf5)\n",
    "\n",
    "        print(i_source, ' : ', i_itr)\n",
    "        dict_result = {}\n",
    "        # Data splitting\n",
    "        dict_result['x_train_scal']     = x_train_scal\n",
    "        dict_result['x_test_scal']      = x_test_scal\n",
    "        dict_result['xs_train_scal']    = xs_train_scal\n",
    "        dict_result['xs_test_scal']     = xs_test_scal\n",
    "        dict_result['y_train_scal']     = y_train_scal\n",
    "        dict_result['y_test_scal']      = y_test_scal\n",
    "        dict_result['y_train']          = y_train\n",
    "        dict_result['y_test']           = y_test\n",
    "        dict_result['y_fits_Wotl']      = y_fits_Wotl\n",
    "        dict_result['y_pred_Wotl']      = y_pred_Wotl\n",
    "        dict_result['best_params_Wotl'] = gsr_Wotl.best_params_\n",
    "\n",
    "        # Model training\n",
    "        ## Fine-tuning\n",
    "        gsr_Fine = GridSearchCV(\n",
    "            cls_FineTune_NN(),\n",
    "            SearchParams_FineTune_NN,\n",
    "            scoring = 'neg_mean_squared_error',\n",
    "            cv = n_fold,\n",
    "            n_jobs = -1,\n",
    "            verbose = False\n",
    "        )\n",
    "        t_tmp = time.time()\n",
    "        fix_seed(SEED)\n",
    "        gsr_Fine.fit(x_train_scal, y_train_scal)\n",
    "\n",
    "        model_Fine = cls_FineTune_NN(\n",
    "            learning_rate = gsr_Fine.best_params_['learning_rate'],\n",
    "            epochs        = gsr_Fine.best_params_['epochs'],\n",
    "            n_frozen      = gsr_Fine.best_params_['n_frozen']\n",
    "        )\n",
    "        fix_seed(SEED)\n",
    "        model_Fine.fit(x_train_scal, y_train_scal)\n",
    "\n",
    "        y_fits_Fine_scal = model_Fine.predict(x_train_scal)\n",
    "        y_pred_Fine_scal = model_Fine.predict(x_test_scal)\n",
    "        y_fits_Fine      = np.exp(y_fits_Fine_scal*y_LogStd+y_LogMean)\n",
    "        y_pred_Fine      = np.exp(y_pred_Fine_scal*y_LogStd+y_LogMean)\n",
    "        y_fits_Fine      = pd.Series(y_fits_Fine.reshape(-1), index=y_train.index)\n",
    "        y_pred_Fine      = pd.Series(y_pred_Fine.reshape(-1), index=y_test.index)\n",
    "        dict_result['y_fits_Fine']      = y_fits_Fine\n",
    "        dict_result['y_pred_Fine']      = y_pred_Fine\n",
    "        dict_result['best_params_Fine'] = gsr_Fine.best_params_\n",
    "        train_MSE_Fine = np.mean((y_fits_Fine - y_train)**2)\n",
    "        test_MSE_Fine  = np.mean((y_pred_Fine - y_test )**2)\n",
    "        print('  Fine-tuning has been done.    '+str(time.time()-t_tmp))\n",
    "\n",
    "        list_train_MSE_Ext = list()\n",
    "        list_test_MSE_Ext  = list()\n",
    "        list_train_MSE_Inv = list()\n",
    "        list_test_MSE_Inv  = list()\n",
    "        # Repeat for different layers\n",
    "        for i_layer in [1,2,3]:\n",
    "            name_layer = source_model.layers[2*i_layer].name\n",
    "            source_model_Ext = Model(inputs=source_model.input, outputs=source_model.get_layer(name_layer).output)\n",
    "\n",
    "            ## Feature extract\n",
    "            ll_train = pd.DataFrame(source_model_Ext.predict(xs_train_scal), index=xs_train_scal.index)\n",
    "            ll_test  = pd.DataFrame(source_model_Ext.predict(xs_test_scal), index=xs_test_scal.index)\n",
    "\n",
    "            ll_mean = 0\n",
    "            ll_std  = 1\n",
    "            ll_train_scal = (ll_train - ll_mean)/ll_std\n",
    "            ll_test_scal  = (ll_test  - ll_mean)/ll_std\n",
    "\n",
    "            ## Merged data\n",
    "            x_train_adds = pd.merge(x_train_scal, ll_train_scal, left_index=True, right_index=True)\n",
    "            x_test_adds  = pd.merge(x_test_scal, ll_test_scal, left_index=True, right_index=True)\n",
    "            \n",
    "            ## Feature extractor\n",
    "            gsr_Ext = GridSearchCV(\n",
    "                cls_FeatureExt_NN(),\n",
    "                SearchParams_FeatureExt_NN,\n",
    "                scoring = 'neg_mean_squared_error',\n",
    "                cv = n_fold,\n",
    "                n_jobs = -1,\n",
    "                verbose = False\n",
    "            )\n",
    "            t_tmp = time.time()\n",
    "            fix_seed(SEED)\n",
    "            gsr_Ext.fit(ll_train_scal, y_train_scal)\n",
    "\n",
    "            model_Ext = cls_FeatureExt_NN(\n",
    "                dr_rate       = gsr_Ext.best_params_['dr_rate'],\n",
    "                layer1        = gsr_Ext.best_params_['layer1'],\n",
    "                decay_rate1   = gsr_Ext.best_params_['decay_rate1'],\n",
    "                decay_rate2   = gsr_Ext.best_params_['decay_rate2'],\n",
    "                l2_lambda     = gsr_Ext.best_params_['l2_lambda'],\n",
    "                learning_rate = gsr_Ext.best_params_['learning_rate'],\n",
    "                epochs        = gsr_Ext.best_params_['epochs']\n",
    "            )\n",
    "            fix_seed(SEED)\n",
    "            model_Ext.fit(ll_train_scal, y_train_scal)\n",
    "\n",
    "            y_fits_Ext_scal = model_Ext.predict(ll_train_scal)\n",
    "            y_pred_Ext_scal = model_Ext.predict(ll_test_scal)\n",
    "            y_fits_Ext      = np.exp(y_fits_Ext_scal*y_LogStd+y_LogMean)\n",
    "            y_pred_Ext      = np.exp(y_pred_Ext_scal*y_LogStd+y_LogMean)\n",
    "            y_fits_Ext      = pd.Series(y_fits_Ext.reshape(-1), index=y_train.index)\n",
    "            y_pred_Ext      = pd.Series(y_pred_Ext.reshape(-1), index=y_test.index)\n",
    "            dict_result['y_fits_Ext'+str(i_layer)]      = y_fits_Ext\n",
    "            dict_result['y_pred_Ext'+str(i_layer)]      = y_pred_Ext\n",
    "            dict_result['best_params_Ext'+str(i_layer)] = gsr_Ext.best_params_\n",
    "            list_train_MSE_Ext.append(np.mean((y_fits_Ext - y_train)**2))\n",
    "            list_test_MSE_Ext.append( np.mean((y_pred_Ext - y_test )**2))\n",
    "            print('     Feature extractor (neural network, layer-'+str(i_layer)+') has been done.    '+str(time.time()-t_tmp))\n",
    "\n",
    "            ## Proposed method\n",
    "            gsr_InvNN = GridSearchCV(\n",
    "                cls_InvTrans_NN(),\n",
    "                SearchParams_InvTrans_NN,\n",
    "                scoring = 'neg_mean_squared_error',\n",
    "                cv = n_fold,\n",
    "                n_jobs = -1,\n",
    "                verbose = False\n",
    "            )\n",
    "            t_tmp = time.time()\n",
    "            fix_seed(SEED)\n",
    "            gsr_InvNN.fit(x_train_adds, y_train_scal)\n",
    "\n",
    "            model_InvNN = cls_InvTrans_NN(\n",
    "                dr_rate       = gsr_InvNN.best_params_['dr_rate'],\n",
    "                layer1        = gsr_InvNN.best_params_['layer1'],\n",
    "                layer2        = gsr_InvNN.best_params_['layer2'],\n",
    "                layer3        = gsr_InvNN.best_params_['layer3'],\n",
    "                decay_rate1   = gsr_InvNN.best_params_['decay_rate1'],\n",
    "                decay_rate2   = gsr_InvNN.best_params_['decay_rate2'],\n",
    "                decay_rate3   = gsr_InvNN.best_params_['decay_rate3'],\n",
    "                l2_lambda1    = gsr_InvNN.best_params_['l2_lambda1'],\n",
    "                l2_lambda2    = gsr_InvNN.best_params_['l2_lambda2'],\n",
    "                learning_rate = gsr_InvNN.best_params_['learning_rate'],\n",
    "                epochs        = gsr_InvNN.best_params_['epochs']\n",
    "            )\n",
    "            fix_seed(SEED)\n",
    "            model_InvNN.fit(x_train_adds, y_train_scal)\n",
    "\n",
    "            y_fits_InvNN_scal = model_InvNN.predict(x_train_adds)\n",
    "            y_pred_InvNN_scal = model_InvNN.predict(x_test_adds)\n",
    "            y_fits_InvNN      = np.exp(y_fits_InvNN_scal*y_LogStd+y_LogMean)\n",
    "            y_pred_InvNN      = np.exp(y_pred_InvNN_scal*y_LogStd+y_LogMean)\n",
    "            y_fits_InvNN      = pd.Series(y_fits_InvNN.reshape(-1), index=y_train.index)\n",
    "            y_pred_InvNN      = pd.Series(y_pred_InvNN.reshape(-1), index=y_test.index)\n",
    "            dict_result['y_fits_InvNN'+str(i_layer)]      = y_fits_InvNN\n",
    "            dict_result['y_pred_InvNN'+str(i_layer)]      = y_pred_InvNN\n",
    "            dict_result['best_params_InvNN'+str(i_layer)] = gsr_InvNN.best_params_\n",
    "            list_train_MSE_Inv.append(np.mean((y_fits_InvNN - y_train)**2))\n",
    "            list_test_MSE_Inv.append( np.mean((y_pred_InvNN - y_test )**2))\n",
    "            print('     Proposed method (neural network, layer-'+str(i_layer)+') has been done.    '+str(time.time()-t_tmp))\n",
    "            \n",
    "            del source_model_Ext\n",
    "            K.clear_session()\n",
    "            \n",
    "        MakeFigure_Train(figsize=(25,10), dict_result=dict_result, savename='../30_Output/20_Plot/200_TransferLearning/Source'+str(i_source)+'/dict_result_s'+str(i_source)+'-i'+str(i_itr)+'.png', plt_show=False)\n",
    "        \n",
    "        df_result.loc[str(i_count)] = [\n",
    "            i_source,\n",
    "            i_itr,\n",
    "            i_count,\n",
    "            train_MSE_Wotl,\n",
    "            train_MSE_Fine,\n",
    "            list_train_MSE_Ext[0],\n",
    "            list_train_MSE_Ext[1],\n",
    "            list_train_MSE_Ext[2],\n",
    "            list_train_MSE_Inv[0],\n",
    "            list_train_MSE_Inv[1],\n",
    "            list_train_MSE_Inv[2],\n",
    "            test_MSE_Wotl,\n",
    "            test_MSE_Fine,\n",
    "            list_test_MSE_Ext[0],\n",
    "            list_test_MSE_Ext[1],\n",
    "            list_test_MSE_Ext[2],\n",
    "            list_test_MSE_Inv[0],\n",
    "            list_test_MSE_Inv[1],\n",
    "            list_test_MSE_Inv[2]\n",
    "        ]\n",
    "        df_result.to_csv('../30_Output/30_csv/200_TransferLearning/300_TransferLearning_Result.csv')\n",
    "        f = open('../30_Output/40_pkl/200_TransferLearning/Source'+str(i_source)+'/dict_result_s'+str(i_source)+'-i'+str(i_itr)+'.pkl','wb')\n",
    "        pickle.dump(dict_result,f)\n",
    "        f.close\n",
    "        i_count += 1\n",
    "        clear_output(True)\n",
    "        print(df_result.mean())\n",
    "print(time.time()-t1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
