{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "741992ed-9816-4664-8342-e877246d8a89",
   "metadata": {},
   "source": [
    "### Python packages used in this code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c505020-4919-4226-9bc1-47f81ec3b9ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "%run Setups.py\n",
    "%run Setups_tf.py\n",
    "%matplotlib inline\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL']='2'\n",
    "tensorflow.get_logger().setLevel(\"ERROR\")\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "688e8689-5654-467b-98ee-74edc8e980b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Environments\n",
    "\n",
    "--Platform--\n",
    "OS : Linux-5.15.90.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
    "--Version--\n",
    "python :  3.9.16 (main, Mar  8 2023, 14:00:05) \n",
    "[GCC 11.2.0]\n",
    "numpy : 1.23.5\n",
    "pandas : 2.0.0\n",
    "tensorflow : 2.12.0\n",
    "sklearn : 2.12.0\n",
    "\"\"\"\n",
    "\n",
    "print('--Platform--')\n",
    "print('OS :', platform.platform())\n",
    "print('--Version--')\n",
    "print('python : ', sys.version)\n",
    "print('numpy :', np.__version__)\n",
    "print('pandas :', pd.__version__)\n",
    "print('tensorflow :', tf.__version__)\n",
    "print('sklearn :', tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53635dea-651e-420b-9fd3-35f653035cd5",
   "metadata": {},
   "source": [
    "### Arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7520dee-06c8-42ff-8907-22b7ae19e7a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "project_name = '300_CheckResults'\n",
    "out_dir = '../30_Output/'+project_name+'/'\n",
    "seed = 373"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f2d4c86-fe20-442d-90cd-e168ce8ae6de",
   "metadata": {},
   "source": [
    "### Make directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f0566df-2dbf-4c5a-8580-d88fad059f81",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isdir(out_dir):\n",
    "    os.makedirs(out_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e83cd67-7830-4eff-a9d6-df78b08e25df",
   "metadata": {},
   "source": [
    "### Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c47e340-e3e7-455f-988f-355d986bcb0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model_reg(hp):\n",
    "    fix_seed(seed)\n",
    "    num_layers128 = hp.Int('num_layers128', 0, 3)\n",
    "    num_layers64 = hp.Int('num_layers64', 0, 3)\n",
    "    num_layers32 = hp.Int('num_layers32', 0, 3)\n",
    "    \n",
    "    model = keras.Sequential()\n",
    "    if num_layers128 > 0:\n",
    "        for i in range(num_layers128):\n",
    "            model.add(Dense(\n",
    "                units=128,\n",
    "                kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                bias_initializer=keras.initializers.Zeros()))\n",
    "            model.add(Activation('relu'))\n",
    "    if num_layers64 > 0:\n",
    "        for i in range(num_layers64):\n",
    "            model.add(Dense(\n",
    "                units=64,\n",
    "                kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                bias_initializer=keras.initializers.Zeros()))\n",
    "            model.add(Activation('relu'))\n",
    "    if num_layers32 > 0:\n",
    "        for i in range(num_layers32):\n",
    "            model.add(Dense(\n",
    "                units=32,\n",
    "                kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                bias_initializer=keras.initializers.Zeros()))\n",
    "            model.add(Activation('relu'))\n",
    "    model.add(Dense(1, kernel_initializer=keras.initializers.glorot_uniform(seed=seed),bias_initializer=keras.initializers.Zeros()))\n",
    "    model.compile(\n",
    "        optimizer=keras.optimizers.Adagrad(learning_rate=0.01, epsilon=1e-7),#, decay=0.0),\n",
    "        loss='mse',\n",
    "        metrics=['mae', 'mse'])\n",
    "    return model\n",
    "\n",
    "def build_model_cls(hp):\n",
    "    fix_seed(seed)\n",
    "    num_layers128 = hp.Int('num_layers128', 0, 3)\n",
    "    num_layers64 = hp.Int('num_layers64', 0, 3)\n",
    "    num_layers32 = hp.Int('num_layers32', 0, 3)\n",
    "    \n",
    "    model = keras.Sequential()\n",
    "    if num_layers128 > 0:\n",
    "        for i in range(num_layers128):\n",
    "            model.add(Dense(\n",
    "                units=128,\n",
    "                kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                bias_initializer=keras.initializers.Zeros()))\n",
    "            model.add(Activation('relu'))\n",
    "    if num_layers64 > 0:\n",
    "        for i in range(num_layers64):\n",
    "            model.add(Dense(\n",
    "                units=64,\n",
    "                kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                bias_initializer=keras.initializers.Zeros()))\n",
    "            model.add(Activation('relu'))\n",
    "    if num_layers32 > 0:\n",
    "        for i in range(num_layers32):\n",
    "            model.add(Dense(\n",
    "                units=32,\n",
    "                kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                bias_initializer=keras.initializers.Zeros()))\n",
    "            model.add(Activation('relu'))\n",
    "    model.add(Dense(1, activation='sigmoid', kernel_initializer=keras.initializers.glorot_uniform(seed=seed),bias_initializer=keras.initializers.Zeros()))\n",
    "    model.compile(\n",
    "        optimizer=keras.optimizers.Adagrad(learning_rate=0.01, epsilon=1e-7),#, decay=0.0),\n",
    "        loss='binary_crossentropy',\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "    return model\n",
    "\n",
    "def build_model_reg_Affine(hp):\n",
    "    fix_seed(seed)\n",
    "    num_layers128 = hp.Int('num_layers128', 0, 3)\n",
    "    num_layers64 = hp.Int('num_layers64', 0, 3)\n",
    "    num_layers32 = hp.Int('num_layers32', 0, 3)\n",
    "    lmbd = 10**hp.Float('L1 lambda power', min_value=-3, max_value=1)\n",
    "    \n",
    "    # Define input layers\n",
    "    input_x = keras.layers.Input(shape=(dim_x,), name='x')\n",
    "    input_f = keras.layers.Input(shape=(dim_f,), name='f')\n",
    "    \n",
    "    x = input_x\n",
    "    f = input_f\n",
    "\n",
    "    # Network for f\n",
    "    if num_layers128 > 0:\n",
    "        for i in range(num_layers128):\n",
    "            f = keras.layers.Dense(\n",
    "                    units=128,\n",
    "                    kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                    bias_initializer=keras.initializers.Zeros(), activation='relu')(f)\n",
    "    if num_layers64 > 0:\n",
    "        for i in range(num_layers64):\n",
    "            f = keras.layers.Dense(\n",
    "                    units=64,\n",
    "                    kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                    bias_initializer=keras.initializers.Zeros(), activation='relu')(f)\n",
    "    if num_layers32 > 0:\n",
    "        for i in range(num_layers32):\n",
    "            f = keras.layers.Dense(\n",
    "                    units=32,\n",
    "                    kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                    bias_initializer=keras.initializers.Zeros(), activation='relu')(f)\n",
    "    g1_g2 = keras.layers.Dense(units=2, kernel_regularizer=regularizers.l1(lmbd), name='g1_g2')(f)\n",
    "\n",
    "    # Network for x\n",
    "    g3 = keras.layers.Dense(units=1, name='g3')(input_x)\n",
    "\n",
    "    # Merge the networks\n",
    "    output = keras.layers.Add(name='output')([\n",
    "        g1_g2[:,0],\n",
    "        keras.layers.Multiply(name='g2_g3')([g1_g2[:,1], g3])\n",
    "    ])\n",
    "\n",
    "    # Define the model\n",
    "    model = keras.models.Model(inputs=[input_x, input_f], outputs=output)\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=keras.optimizers.Adagrad(learning_rate=0.01, epsilon=1e-7),\n",
    "        loss='mse',\n",
    "        metrics=['mae', 'mse']\n",
    "    )\n",
    "    \n",
    "    return model\n",
    "\n",
    "def build_model_cls_Affine(hp):\n",
    "    fix_seed(seed)\n",
    "    num_layers128 = hp.Int('num_layers128', 0, 3)\n",
    "    num_layers64 = hp.Int('num_layers64', 0, 3)\n",
    "    num_layers32 = hp.Int('num_layers32', 0, 3)\n",
    "    lmbd = 10**hp.Float('L1 lambda power', min_value=-3, max_value=1)\n",
    "    \n",
    "    # Define input layers\n",
    "    input_x = keras.layers.Input(shape=(dim_x,), name='x')\n",
    "    input_f = keras.layers.Input(shape=(dim_f,), name='f')\n",
    "    \n",
    "    x = input_x\n",
    "    f = input_f\n",
    "\n",
    "    # Network for f\n",
    "    if num_layers128 > 0:\n",
    "        for i in range(num_layers128):\n",
    "            f = keras.layers.Dense(\n",
    "                    units=128,\n",
    "                    kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                    bias_initializer=keras.initializers.Zeros(), activation='relu')(f)\n",
    "    if num_layers64 > 0:\n",
    "        for i in range(num_layers64):\n",
    "            f = keras.layers.Dense(\n",
    "                    units=64,\n",
    "                    kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                    bias_initializer=keras.initializers.Zeros(), activation='relu')(f)\n",
    "    if num_layers32 > 0:\n",
    "        for i in range(num_layers32):\n",
    "            f = keras.layers.Dense(\n",
    "                    units=32,\n",
    "                    kernel_initializer=keras.initializers.glorot_uniform(seed=seed),\n",
    "                    bias_initializer=keras.initializers.Zeros(), activation='relu')(f)\n",
    "    g1_g2 = keras.layers.Dense(units=2, kernel_regularizer=regularizers.l1(lmbd), name='g1_g2')(f)\n",
    "\n",
    "    # Network for x\n",
    "    g3 = keras.layers.Dense(units=1, name='g3')(input_x)\n",
    "\n",
    "    # Merge the networks\n",
    "    output = keras.layers.Add(name='output')([\n",
    "        g1_g2[:,0],\n",
    "        keras.layers.Multiply(name='g2_g3')([g1_g2[:,1], g3])\n",
    "    ])\n",
    "    output = keras.layers.Activation('softmax', name='softmax')(output)\n",
    "\n",
    "    # Define the model\n",
    "    model = keras.models.Model(inputs=[input_x, input_f], outputs=output)\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=keras.optimizers.Adagrad(learning_rate=0.01, epsilon=1e-7),\n",
    "        loss='binary_crossentropy',\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "    \n",
    "    return model\n",
    "\n",
    "def map_to_RA(x):\n",
    "    if x >= 0.5:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def fun_Tuning(f, score, decision, ProjectName='tmp'):\n",
    "    f_tr = f\n",
    "    s_tr = score\n",
    "    d_tr = decision\n",
    "    \n",
    "    ## Regression\n",
    "    tuner_reg = BayesianOptimization(\n",
    "        build_model_reg,\n",
    "        objective='val_loss',\n",
    "        project_name=ProjectName+'_reg',\n",
    "        alpha=0.0001,\n",
    "        beta=2.6,\n",
    "        overwrite=True\n",
    "    )\n",
    "\n",
    "    fix_seed(seed)\n",
    "    tuner_reg.search(\n",
    "        f_tr.values,\n",
    "        s_tr.values,\n",
    "        batch_size=4,\n",
    "        epochs=1000,\n",
    "        validation_split=0.1,\n",
    "        verbose=0,\n",
    "        callbacks=[early_stop]\n",
    "    )\n",
    "\n",
    "    ## Classification\n",
    "    tuner_cls = BayesianOptimization(\n",
    "        build_model_cls,\n",
    "        objective='val_loss',\n",
    "        project_name=ProjectName+'_cls',\n",
    "        alpha=0.0001,\n",
    "        beta=2.6,\n",
    "        overwrite=True\n",
    "    )\n",
    "\n",
    "    fix_seed(seed)\n",
    "    tuner_cls.search(\n",
    "        f_tr.values,\n",
    "        d_tr.values,\n",
    "        batch_size=4,\n",
    "        epochs=1000,\n",
    "        validation_split=0.1,\n",
    "        verbose=0,\n",
    "        callbacks=[early_stop]\n",
    "    )\n",
    "\n",
    "    return tuner_reg, tuner_cls\n",
    "\n",
    "def fun_TuningAffine(x, f, score, decision, ProjectName='tmp'):\n",
    "    x_tr = x\n",
    "    f_tr = f\n",
    "    s_tr = score\n",
    "    d_tr = decision\n",
    "    \n",
    "    ## Regression\n",
    "    tuner_reg = BayesianOptimization(\n",
    "        build_model_reg_Affine,\n",
    "        objective='val_loss',\n",
    "        project_name=ProjectName+'_reg',\n",
    "        alpha=0.0001,\n",
    "        beta=2.6,\n",
    "        overwrite=True\n",
    "    )\n",
    "\n",
    "    fix_seed(seed)\n",
    "    tuner_reg.search(\n",
    "        [x_tr.values, f_tr.values],\n",
    "        s_tr.values,\n",
    "        batch_size=4,\n",
    "        epochs=1000,\n",
    "        validation_split=0.1,\n",
    "        verbose=0,\n",
    "        callbacks=[early_stop]\n",
    "    )\n",
    "\n",
    "    ## Classification\n",
    "    tuner_cls = BayesianOptimization(\n",
    "        build_model_cls_Affine,\n",
    "        objective='val_loss',\n",
    "        project_name=ProjectName+'_cls',\n",
    "        alpha=0.0001,\n",
    "        beta=2.6,\n",
    "        overwrite=True\n",
    "    )\n",
    "\n",
    "    fix_seed(seed)\n",
    "    tuner_cls.search(\n",
    "        [x_tr.values, f_tr.values],\n",
    "        d_tr.values,\n",
    "        batch_size=4,\n",
    "        epochs=1000,\n",
    "        validation_split=0.1,\n",
    "        verbose=0,\n",
    "        callbacks=[early_stop]\n",
    "    )\n",
    "\n",
    "    return tuner_reg, tuner_cls\n",
    "\n",
    "def fun_Training(f, score, decision, tuner_reg, tuner_cls, ScalParams):\n",
    "    f_tr = f[0]\n",
    "    f_te = f[1]\n",
    "    s_tr = score#[0]\n",
    "    # s_te = score[1]\n",
    "    d_tr = decision#[0]\n",
    "    # d_te = decision[1]\n",
    "    \n",
    "    ## Regression\n",
    "    best_hps = tuner_reg.get_best_hyperparameters(num_trials=1)[0]\n",
    "    model_reg = tuner_reg.hypermodel.build(best_hps)\n",
    "\n",
    "    fix_seed(seed)\n",
    "    history_reg = model_reg.fit(\n",
    "        f_tr,\n",
    "        s_tr,\n",
    "        batch_size=4,\n",
    "        epochs=1000,\n",
    "        validation_split=0.1,\n",
    "        verbose=0,\n",
    "        callbacks=[early_stop, PrintDot()])\n",
    "\n",
    "    fits = model_reg.predict(f_tr).reshape(-1)\n",
    "    pred = model_reg.predict(f_te).reshape(-1)\n",
    "    s_fits = pd.Series(fun_invNormScale(fits, ScalParams), f_tr.index)\n",
    "    s_pred = pd.Series(fun_invNormScale(pred, ScalParams), f_te.index)\n",
    "    \n",
    "    ## Classification\n",
    "    best_hps = tuner_cls.get_best_hyperparameters(num_trials=1)[0]\n",
    "    model_cls = tuner_cls.hypermodel.build(best_hps)\n",
    "\n",
    "    fix_seed(seed)\n",
    "    history_reg = model_cls.fit(\n",
    "        f_tr,\n",
    "        d_tr,\n",
    "        batch_size=4,\n",
    "        epochs=1000,\n",
    "        validation_split=0.1,\n",
    "        verbose=0,\n",
    "        callbacks=[early_stop, PrintDot()])\n",
    "\n",
    "    d_fits = model_cls.predict(f_tr).reshape(-1)\n",
    "    d_pred = model_cls.predict(f_te).reshape(-1)\n",
    "    ra_fits = np.array([map_to_RA(x) for x in d_fits])\n",
    "    ra_pred = np.array([map_to_RA(x) for x in d_pred])\n",
    "    \n",
    "    d_fits = pd.Series(d_fits, index=f_tr.index)\n",
    "    d_pred = pd.Series(d_pred, index=f_te.index)\n",
    "    ra_fits = pd.Series(ra_fits, index=f_tr.index)\n",
    "    ra_pred = pd.Series(ra_pred, index=f_te.index)\n",
    "\n",
    "    return model_reg, model_cls, {\n",
    "        'Score fits' : s_fits,\n",
    "        'Score pred' : s_pred,\n",
    "        'DecisionP fits' : d_fits,\n",
    "        'DecisionP pred' : d_pred,\n",
    "        'Decision fits' : ra_fits,\n",
    "        'Decision pred' : ra_pred\n",
    "    }\n",
    "\n",
    "def fun_TrainingAffine(x, f, score, decision, tuner_reg, tuner_cls, ScalParams):\n",
    "    x_tr = x[0]\n",
    "    x_te = x[1]\n",
    "    f_tr = f[0]\n",
    "    f_te = f[1]\n",
    "    s_tr = score#[0]\n",
    "    # s_te = score[1]\n",
    "    d_tr = decision#[0]\n",
    "    # d_te = decision[1]\n",
    "    \n",
    "    ## Regression\n",
    "    best_hps = tuner_reg.get_best_hyperparameters(num_trials=1)[0]\n",
    "    model_reg = tuner_reg.hypermodel.build(best_hps)\n",
    "\n",
    "    fix_seed(seed)\n",
    "    history_reg = model_reg.fit(\n",
    "        [x_tr, f_tr],\n",
    "        s_tr,\n",
    "        batch_size=4,\n",
    "        epochs=1000,\n",
    "        validation_split=0.1,\n",
    "        verbose=0,\n",
    "        callbacks=[early_stop, PrintDot()])\n",
    "\n",
    "    fits = model_reg.predict([x_tr, f_tr]).reshape(-1)\n",
    "    pred = model_reg.predict([x_te, f_te]).reshape(-1)\n",
    "    s_fits = pd.Series(fun_invNormScale(fits, ScalParams), f_tr.index)\n",
    "    s_pred = pd.Series(fun_invNormScale(pred, ScalParams), f_te.index)\n",
    "    \n",
    "    ## Classification\n",
    "    best_hps = tuner_cls.get_best_hyperparameters(num_trials=1)[0]\n",
    "    model_cls = tuner_cls.hypermodel.build(best_hps)\n",
    "\n",
    "    fix_seed(seed)\n",
    "    history_reg = model_cls.fit(\n",
    "        [x_tr, f_tr],\n",
    "        d_tr,\n",
    "        batch_size=4,\n",
    "        epochs=1000,\n",
    "        validation_split=0.1,\n",
    "        verbose=0,\n",
    "        callbacks=[early_stop, PrintDot()])\n",
    "\n",
    "    d_fits = model_cls.predict([x_tr, f_tr]).reshape(-1)\n",
    "    d_pred = model_cls.predict([x_te, f_te]).reshape(-1)\n",
    "    ra_fits = np.array([map_to_RA(x) for x in d_fits])\n",
    "    ra_pred = np.array([map_to_RA(x) for x in d_pred])\n",
    "    \n",
    "    d_fits = pd.Series(d_fits, index=f_tr.index)\n",
    "    d_pred = pd.Series(d_pred, index=f_te.index)\n",
    "    ra_fits = pd.Series(ra_fits, index=f_tr.index)\n",
    "    ra_pred = pd.Series(ra_pred, index=f_te.index)\n",
    "\n",
    "    return model_reg, model_cls, {\n",
    "        'Score fits' : s_fits,\n",
    "        'Score pred' : s_pred,\n",
    "        'DecisionP fits' : d_fits,\n",
    "        'DecisionP pred' : d_pred,\n",
    "        'Decision fits' : ra_fits,\n",
    "        'Decision pred' : ra_pred\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "894c93a0-8b4d-4047-9609-d3f768df1255",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fun_MainFE(df_f, df_y, df_l, n_try, ProjectName):\n",
    "    df_f_tr = df_f[0]\n",
    "    df_f_te = df_f[1]\n",
    "    df_y_tr = df_y[0]\n",
    "    df_y_te = df_y[1]\n",
    "    df_l_tr = df_l[0]\n",
    "    df_l_te = df_l[1]\n",
    "    \n",
    "    fix_seed(seed)\n",
    "    split_seed = np.random.randint(low=1, high=n_try**2, size=n_try)\n",
    "    text_id = df_y_tr.index\n",
    "    \n",
    "    rmse_list = []\n",
    "    acc_list = []\n",
    "    for i_try in range(n_try):\n",
    "        print(i_try)\n",
    "        ## Split\n",
    "        fix_seed(seed)\n",
    "        train_id, test_id = train_test_split(text_id, test_size=0.1, random_state=split_seed[i_try])\n",
    "\n",
    "        score_train = df_y_tr.loc[train_id]\n",
    "        score_test = df_y_te\n",
    "        decision_train = df_l_tr.loc[train_id]\n",
    "        decision_test = df_l_te\n",
    "        f_train = df_f_tr.loc[train_id]\n",
    "        f_test = df_f_te\n",
    "\n",
    "        score_train_scal, ScalParams = fun_NormScale(score_train)\n",
    "        score_test_scal, _ = fun_NormScale(score_test, params=ScalParams)\n",
    "\n",
    "        try:\n",
    "            tuner_reg\n",
    "            tuner_cls\n",
    "        except NameError:\n",
    "            tuner_reg, tuner_cls = fun_Tuning(f=f_train, score=score_train_scal, decision=decision_train, ProjectName=ProjectName)\n",
    "\n",
    "        model_reg, model_cls, logits = fun_Training(f=[f_train, f_test], score=score_train_scal, decision=decision_train, tuner_reg=tuner_reg, tuner_cls=tuner_cls, ScalParams=ScalParams)\n",
    "\n",
    "        rmse_list.append(mean_squared_error(score_test, logits['Score pred'], squared=False))\n",
    "        acc_list.append(accuracy_score(decision_test, logits['Decision pred']))\n",
    "\n",
    "        json_string = model_reg.to_json()\n",
    "        open(out_dir+ProjectName+'/FEModelReg'+str(i_try)+'.json', 'w').write(json_string)\n",
    "        model_reg.save_weights(out_dir+ProjectName+'/FEModelReg'+str(i_try)+'.hdf5')\n",
    "        \n",
    "        json_string = model_cls.to_json()\n",
    "        open(out_dir+ProjectName+'/FEModelCls'+str(i_try)+'.json', 'w').write(json_string)\n",
    "        model_cls.save_weights(out_dir+ProjectName+'/FEModelCls'+str(i_try)+'.hdf5')\n",
    "\n",
    "        clear_output(True)\n",
    "        \n",
    "        del model_reg, model_cls\n",
    "    \n",
    "    del tuner_reg, tuner_cls\n",
    "    return rmse_list, acc_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb57dbf4-2821-4c81-acad-b01cf48ea4ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fun_MainAffine(df_x, df_f, df_y, df_l, n_try, ProjectName):\n",
    "    df_x_tr = df_x[0]\n",
    "    df_x_te = df_x[1]\n",
    "    df_f_tr = df_f[0]\n",
    "    df_f_te = df_f[1]\n",
    "    df_y_tr = df_y[0]\n",
    "    df_y_te = df_y[1]\n",
    "    df_l_tr = df_l[0]\n",
    "    df_l_te = df_l[1]\n",
    "    \n",
    "    fix_seed(seed)\n",
    "    split_seed = np.random.randint(low=1, high=n_try**2, size=n_try)\n",
    "    text_id = df_y_tr.index\n",
    "    \n",
    "    rmse_list = []\n",
    "    acc_list = []\n",
    "    for i_try in range(n_try):\n",
    "        print(i_try)\n",
    "        ## Split\n",
    "        fix_seed(seed)\n",
    "        train_id, test_id = train_test_split(text_id, test_size=0.1, random_state=split_seed[i_try])\n",
    "\n",
    "        score_train = df_y_tr.loc[train_id]\n",
    "        score_test = df_y_te\n",
    "        decision_train = df_l_tr.loc[train_id]\n",
    "        decision_test = df_l_te\n",
    "        f_train = df_f_tr.loc[train_id]\n",
    "        f_test = df_f_te\n",
    "        x_train = df_x_tr.loc[train_id]\n",
    "        x_test = df_x_te\n",
    "\n",
    "        score_train_scal, ScalParams = fun_NormScale(score_train)\n",
    "        score_test_scal, _ = fun_NormScale(score_test, params=ScalParams)\n",
    "\n",
    "        try:\n",
    "            tuner_reg\n",
    "            tuner_cls\n",
    "        except NameError:\n",
    "            tuner_reg, tuner_cls = fun_TuningAffine(x=x_train, f=f_train, score=score_train_scal, decision=decision_train, ProjectName=ProjectName)\n",
    "\n",
    "        model_reg, model_cls, logits = fun_TrainingAffine(x=[x_train, x_test], f=[f_train, f_test], score=score_train_scal, decision=decision_train, tuner_reg=tuner_reg, tuner_cls=tuner_cls, ScalParams=ScalParams)\n",
    "\n",
    "        rmse_list.append(mean_squared_error(score_test, logits['Score pred'], squared=False))\n",
    "        acc_list.append(accuracy_score(decision_test, logits['Decision pred']))\n",
    "        \n",
    "        json_string = model_reg.to_json()\n",
    "        open(out_dir+ProjectName+'/AffineModelReg'+str(i_try)+'.json', 'w').write(json_string)\n",
    "        model_reg.save_weights(out_dir+ProjectName+'/AffineModelReg'+str(i_try)+'.hdf5')\n",
    "        \n",
    "        json_string = model_cls.to_json()\n",
    "        open(out_dir+ProjectName+'/AffineModelCls'+str(i_try)+'.json', 'w').write(json_string)\n",
    "        model_cls.save_weights(out_dir+ProjectName+'/AffineModelCls'+str(i_try)+'.hdf5')\n",
    "\n",
    "        clear_output(True)\n",
    "        \n",
    "        del model_reg, model_cls\n",
    "    \n",
    "    del tuner_reg, tuner_cls\n",
    "    return rmse_list, acc_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7aff6839-d980-4edc-becb-7156e3befd87",
   "metadata": {},
   "source": [
    "## Results summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f2bfa5d5-d157-4785-8a9a-a03ea3dcd462",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Load\n",
    "df = pd.read_csv('../10_Data/df_removeURL.csv', index_col=1)\n",
    "csv_train = pd.read_csv('../10_Data/df_train.csv', index_col=1)\n",
    "csv_test = pd.read_csv('../10_Data/df_test.csv', index_col=1)\n",
    "train_id = csv_train.index\n",
    "test_id = csv_test.index\n",
    "\n",
    "## Encoding\n",
    "text = df['abstract (wo URL)']\n",
    "vectorizer = CountVectorizer(max_df=0.9, min_df=0.01, stop_words='english', ngram_range=(2,2))\n",
    "vector = vectorizer.fit_transform(text)\n",
    "df_bow = pd.DataFrame(vector.toarray(), index=df.index, columns=vectorizer.get_feature_names_out())\n",
    "df_bow = df_bow.iloc[:,9:]\n",
    "\n",
    "## Pre-processing\n",
    "label_map = {'R': 0, 'A': 1}\n",
    "binary_labels = pd.Series(np.array([label_map[label] for label in df['decision']]), index=df.index)\n",
    "\n",
    "dim_x = df_bow.shape[1]\n",
    "dim_f = df_enc.shape[1]\n",
    "\n",
    "## Extract common indices\n",
    "common_indices_train = train_id.intersection(df_bow.index).intersection(df_enc.index).intersection(df.index)\n",
    "common_indices_test = test_id.intersection(df_bow.index).intersection(df_enc.index).intersection(df.index)\n",
    "\n",
    "df_bow_train = df_bow.loc[common_indices_train]\n",
    "df_bow_test = df_bow.loc[common_indices_test]\n",
    "df_enc_train = df_enc.loc[common_indices_train]\n",
    "df_enc_test = df_enc.loc[common_indices_test]\n",
    "df_train = df.loc[common_indices_train]\n",
    "df_test = df.loc[common_indices_test]\n",
    "binary_labels_train = binary_labels.loc[common_indices_train]\n",
    "binary_labels_test = binary_labels.loc[common_indices_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd8b6f5e-0f41-4a3b-831f-fc53160b58fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_all = pd.DataFrame()\n",
    "# for ProjectName in ['BERT', 'SciBERT', 'GPT-3', 'T5']:\n",
    "for ProjectName in ['BERT', 'T5', 'SciBERT', 'GPT-3']:\n",
    "    results = pd.read_csv('../30_Output/200_FeatureExtraction/Results-'+ProjectName+'.csv', index_col=0)\n",
    "    results_summary = results.describe()\n",
    "    summary_all = pd.concat([summary_all, pd.DataFrame(results_summary)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "69557f64-8e18-451f-8e1e-c9382481741d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RMSE (BERT-FE)</th>\n",
       "      <th>Accuracy (BERT-FE)</th>\n",
       "      <th>RMSE (BERT-Affine)</th>\n",
       "      <th>Accuracy (BERT-Affine)</th>\n",
       "      <th>RMSE (T5-FE)</th>\n",
       "      <th>Accuracy (T5-FE)</th>\n",
       "      <th>RMSE (T5-Affine)</th>\n",
       "      <th>Accuracy (T5-Affine)</th>\n",
       "      <th>RMSE (SciBERT-FE)</th>\n",
       "      <th>Accuracy (SciBERT-FE)</th>\n",
       "      <th>RMSE (SciBERT-Affine)</th>\n",
       "      <th>Accuracy (SciBERT-Affine)</th>\n",
       "      <th>RMSE (GPT-3-FE)</th>\n",
       "      <th>Accuracy (GPT-3-FE)</th>\n",
       "      <th>RMSE (GPT-3-Affine)</th>\n",
       "      <th>Accuracy (GPT-3-Affine)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.308614</td>\n",
       "      <td>0.624993</td>\n",
       "      <td>1.306919</td>\n",
       "      <td>0.625233</td>\n",
       "      <td>1.348597</td>\n",
       "      <td>0.634356</td>\n",
       "      <td>1.344193</td>\n",
       "      <td>0.636574</td>\n",
       "      <td>1.285595</td>\n",
       "      <td>0.651956</td>\n",
       "      <td>1.279709</td>\n",
       "      <td>0.650730</td>\n",
       "      <td>1.328394</td>\n",
       "      <td>0.627920</td>\n",
       "      <td>1.323358</td>\n",
       "      <td>0.638605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.003505</td>\n",
       "      <td>0.021688</td>\n",
       "      <td>0.004171</td>\n",
       "      <td>0.016344</td>\n",
       "      <td>0.017509</td>\n",
       "      <td>0.007852</td>\n",
       "      <td>0.002996</td>\n",
       "      <td>0.006507</td>\n",
       "      <td>0.014357</td>\n",
       "      <td>0.010630</td>\n",
       "      <td>0.012155</td>\n",
       "      <td>0.012376</td>\n",
       "      <td>0.013785</td>\n",
       "      <td>0.018100</td>\n",
       "      <td>0.013968</td>\n",
       "      <td>0.009487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.302142</td>\n",
       "      <td>0.546837</td>\n",
       "      <td>1.299471</td>\n",
       "      <td>0.581167</td>\n",
       "      <td>1.327888</td>\n",
       "      <td>0.604014</td>\n",
       "      <td>1.336277</td>\n",
       "      <td>0.609398</td>\n",
       "      <td>1.264615</td>\n",
       "      <td>0.601471</td>\n",
       "      <td>1.261240</td>\n",
       "      <td>0.609314</td>\n",
       "      <td>1.307167</td>\n",
       "      <td>0.534998</td>\n",
       "      <td>1.302356</td>\n",
       "      <td>0.599119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.306040</td>\n",
       "      <td>0.614640</td>\n",
       "      <td>1.303982</td>\n",
       "      <td>0.615988</td>\n",
       "      <td>1.337702</td>\n",
       "      <td>0.629956</td>\n",
       "      <td>1.343241</td>\n",
       "      <td>0.635095</td>\n",
       "      <td>1.274492</td>\n",
       "      <td>0.648897</td>\n",
       "      <td>1.271212</td>\n",
       "      <td>0.647059</td>\n",
       "      <td>1.318566</td>\n",
       "      <td>0.620411</td>\n",
       "      <td>1.313443</td>\n",
       "      <td>0.635585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.308016</td>\n",
       "      <td>0.629720</td>\n",
       "      <td>1.305869</td>\n",
       "      <td>0.630211</td>\n",
       "      <td>1.343814</td>\n",
       "      <td>0.639745</td>\n",
       "      <td>1.344552</td>\n",
       "      <td>0.639256</td>\n",
       "      <td>1.282773</td>\n",
       "      <td>0.653676</td>\n",
       "      <td>1.277261</td>\n",
       "      <td>0.653922</td>\n",
       "      <td>1.325436</td>\n",
       "      <td>0.638522</td>\n",
       "      <td>1.319284</td>\n",
       "      <td>0.640235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.310307</td>\n",
       "      <td>0.641614</td>\n",
       "      <td>1.309623</td>\n",
       "      <td>0.636219</td>\n",
       "      <td>1.353620</td>\n",
       "      <td>0.639745</td>\n",
       "      <td>1.346140</td>\n",
       "      <td>0.640235</td>\n",
       "      <td>1.293363</td>\n",
       "      <td>0.658333</td>\n",
       "      <td>1.284686</td>\n",
       "      <td>0.658824</td>\n",
       "      <td>1.336011</td>\n",
       "      <td>0.639745</td>\n",
       "      <td>1.330935</td>\n",
       "      <td>0.644640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.319683</td>\n",
       "      <td>0.650319</td>\n",
       "      <td>1.321404</td>\n",
       "      <td>0.654242</td>\n",
       "      <td>1.449642</td>\n",
       "      <td>0.640724</td>\n",
       "      <td>1.349479</td>\n",
       "      <td>0.643172</td>\n",
       "      <td>1.326018</td>\n",
       "      <td>0.670588</td>\n",
       "      <td>1.316396</td>\n",
       "      <td>0.671569</td>\n",
       "      <td>1.377478</td>\n",
       "      <td>0.645130</td>\n",
       "      <td>1.363179</td>\n",
       "      <td>0.651982</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       RMSE (BERT-FE)  Accuracy (BERT-FE)  RMSE (BERT-Affine)   \n",
       "count      100.000000          100.000000          100.000000  \\\n",
       "mean         1.308614            0.624993            1.306919   \n",
       "std          0.003505            0.021688            0.004171   \n",
       "min          1.302142            0.546837            1.299471   \n",
       "25%          1.306040            0.614640            1.303982   \n",
       "50%          1.308016            0.629720            1.305869   \n",
       "75%          1.310307            0.641614            1.309623   \n",
       "max          1.319683            0.650319            1.321404   \n",
       "\n",
       "       Accuracy (BERT-Affine)  RMSE (T5-FE)  Accuracy (T5-FE)   \n",
       "count              100.000000    100.000000        100.000000  \\\n",
       "mean                 0.625233      1.348597          0.634356   \n",
       "std                  0.016344      0.017509          0.007852   \n",
       "min                  0.581167      1.327888          0.604014   \n",
       "25%                  0.615988      1.337702          0.629956   \n",
       "50%                  0.630211      1.343814          0.639745   \n",
       "75%                  0.636219      1.353620          0.639745   \n",
       "max                  0.654242      1.449642          0.640724   \n",
       "\n",
       "       RMSE (T5-Affine)  Accuracy (T5-Affine)  RMSE (SciBERT-FE)   \n",
       "count        100.000000            100.000000         100.000000  \\\n",
       "mean           1.344193              0.636574           1.285595   \n",
       "std            0.002996              0.006507           0.014357   \n",
       "min            1.336277              0.609398           1.264615   \n",
       "25%            1.343241              0.635095           1.274492   \n",
       "50%            1.344552              0.639256           1.282773   \n",
       "75%            1.346140              0.640235           1.293363   \n",
       "max            1.349479              0.643172           1.326018   \n",
       "\n",
       "       Accuracy (SciBERT-FE)  RMSE (SciBERT-Affine)   \n",
       "count             100.000000             100.000000  \\\n",
       "mean                0.651956               1.279709   \n",
       "std                 0.010630               0.012155   \n",
       "min                 0.601471               1.261240   \n",
       "25%                 0.648897               1.271212   \n",
       "50%                 0.653676               1.277261   \n",
       "75%                 0.658333               1.284686   \n",
       "max                 0.670588               1.316396   \n",
       "\n",
       "       Accuracy (SciBERT-Affine)  RMSE (GPT-3-FE)  Accuracy (GPT-3-FE)   \n",
       "count                 100.000000       100.000000           100.000000  \\\n",
       "mean                    0.650730         1.328394             0.627920   \n",
       "std                     0.012376         0.013785             0.018100   \n",
       "min                     0.609314         1.307167             0.534998   \n",
       "25%                     0.647059         1.318566             0.620411   \n",
       "50%                     0.653922         1.325436             0.638522   \n",
       "75%                     0.658824         1.336011             0.639745   \n",
       "max                     0.671569         1.377478             0.645130   \n",
       "\n",
       "       RMSE (GPT-3-Affine)  Accuracy (GPT-3-Affine)  \n",
       "count           100.000000               100.000000  \n",
       "mean              1.323358                 0.638605  \n",
       "std               0.013968                 0.009487  \n",
       "min               1.302356                 0.599119  \n",
       "25%               1.313443                 0.635585  \n",
       "50%               1.319284                 0.640235  \n",
       "75%               1.330935                 0.644640  \n",
       "max               1.363179                 0.651982  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d8de6f2-7e54-4c57-a68e-c643748d2c4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_all.to_csv('../30_Output/300_CheckResults/Summary_all.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "846e5d54-a119-4671-90a7-ed4d89454849",
   "metadata": {},
   "source": [
    "## Model parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e620638-e425-4a95-8498-5dc52ca018ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "for ProjectName in ['BERT','T5', 'SciBERT', 'GPT-3']:\n",
    "    df_params = pd.DataFrame()\n",
    "    for i in range(100):\n",
    "        source_model = model_from_json(open('../30_Output/200_FeatureExtraction/'+ProjectName+'/AffineModelReg'+str(i)+'.json', 'r').read())\n",
    "        source_model.load_weights('../30_Output/200_FeatureExtraction/'+ProjectName+'/AffineModelReg'+str(i)+'.hdf5')\n",
    "\n",
    "        df_params['Model-'+str(i)] = source_model.get_layer('g3').get_weights()[0].reshape(-1)\n",
    "\n",
    "    df_params = df_params.T\n",
    "    df_params.columns = df_bow.columns\n",
    "    df_params_sorted = df_params.mean().sort_values()\n",
    "    df_params_sorted.to_csv('../30_Output/300_CheckResults/Sort_Params_'+ProjectName+'.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
