#!/bin/bash

echo "mode: $1"    # feedback, correction, comp_feedback, meta_feedback
echo "format: $2"    # sub, obj
echo "set: $3"    # test, dev
echo "save results into: $4"    # any name for saving the evaluation results
if [ $1 == 'feedback' ];
then
    if [ $2 == "obj" ]; 
    then  
        echo "Inference Objective Evaluation for Feedback Critique Task"
        python run_feedback.py --root_dir "../data/CriticBench" --prediction_dir "../inference/20240501_320_output_feedback" --batch_size 16 --split $3 --obj True
        python run_feedback.py --root_dir "../data/CriticBench" --prediction_dir "../inference/20240501_320_output_feedback_mathcode_train_no_reference_mathcode" --batch_size 16 --split $3 --obj True
    else
        echo "Inference Subjective Evaluation for Feedback Critique Task"
        python run_feedback.py --root_dir "../data/CriticBench" --prediction_dir "../inference/20240501_320_output_feedback_without_task_with_my_criteria" --evaluation_dir "20240501_output_meta_feedback_without_task_with_my_criteria_2" --batch_size 32 --split test --obj False
    fi
elif [ $1 == 'correction' ];
then
    if [ $2 == "obj" ]; 
    then  
        echo "Inference Objective Evaluation for Correction Critique Task"
        python run_correction.py --root_dir "../data/CriticBench" --prediction_dir "../inference/20240501_320_output_correction" --batch_size 16 --split dev --obj True
    else
        echo "Inference Subjective Evaluation for Correction Critique Task"
        python run_correction.py --root_dir "../data/CriticBench" --prediction_dir "../inference/20240501_output_correction_internlm2_20b_chat" --evaluation_dir "20240501_output_correction_evaluation" --batch_size 32 --split $3 --obj False
    fi
elif [ $1 == 'comp_feedback' ];
then
    if [ $2 == "obj" ]; 
    then  
        echo "Inference Objective Evaluation for Comparison-based Feedback Critique Task"
        python run_comp_feedback.py --root_dir "../data/criticbench_v1.3" --prediction_dir "../example_data/prediction_v1.3" --batch_size 16 --split $3 --obj True
    else
        echo "Inference Subjective Evaluation for Comparison-based Feedback Critique Task"
        python run_comp_feedback.py --root_dir "../data/criticbench_v1.3" --prediction_dir "../example_data/prediction_v1.3" --evaluation_dir "../example_data/evaluation_v1.3/" --batch_size 1 --split $3 --obj False
    fi
elif [ $1 == 'meta_feedback' ];
then
    echo "Inference Objective Evaluation for Meta-Feedback Critique Task"
    python run_meta_feedback.py --root_dir "../data/CriticBench" --prediction_dir "../inference/" --batch_size 32 --split $3 --obj True
fi
