master_port=14515

# export CUDA_VISIBLE_DEVICES=1,0
model_name=llama-2
model_path="mistralai/Mistral-7B-Instruct-v0.2"
model=llama-2-7b


# export CUDA_VISIBLE_DEVICES=3


# OUTDIRNAME="eval_hf-baseline-harry-0515/noqa"
OUTDIRNAME="eval_hf-baseline-harry-0521/withqa"
TTTS=(
	""
)

export CUDA_VISIBLE_DEVICES=3,0jjG
num_devices=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)

for TTT in "${TTTS[@]}"; do
	# TARGETDIRS=($(find $TTT -mindepth 1 -maxdepth 1 -type d))

	TARGETDIRS=(
        # "trained_models2/hf-baseline-copyright/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_4|gradient_accumulation_steps_4|model_train.weight_decay_0.01|data_harry|data_mode_forget|model_train.model_name_llama-2|model_train_grad_ascent"

        # "trained_models2/hf-baseline-copyright/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_4|gradient_accumulation_steps_4|model_train.weight_decay_0.01|data_harry|model_train.model_name_llama-2|model_train_grad_diff|data_mode_forget_retain"

		# "trained_models2/hf-baseline-copyright/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_2|gradient_accumulation_steps_8|model_train_npo|model_train.weight_decay_0.01|data_harry|data_mode_forget|model_train.model_name_llama-2"

		# "trained_models/hf-copyright-baseline/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_2|gradient_accumulation_steps_4|model_train.weight_decay_0.01|data_harry|model_train.model_name_llama-2|model_train_dpo_gddiff|data_mode_forget_dpo_retain"

		# "trained_models/hf-copyright-baseline/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_2|gradient_accumulation_steps_8|num_epochs_5|model_train.weight_decay_0.01|data_harry|model_train_dpo_kl|model_train.model_name_llama-2|data_mode_forget_dpo_retain/2024-05-20_05-54-39/logs/harry-mybaseline-hf/split_None|loss_dpo_kldata_forget_dpo_retain|lr_1e-05|lora_0|assist_False_2024-05-20T05-54-39"

		# "trained_models2/hf-baseline-copyright/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_2|gradient_accumulation_steps_8|model_train_npo|model_train.weight_decay_0.01|data_harry|data_mode_forget|model_train.model_name_llama-2"

		# "trained_models/hf-copyright-baseline/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_2|gradient_accumulation_steps_8|num_epochs_5|model_train.weight_decay_0.01|data_harry|model_train.model_name_llama-2|model_train_npo_gddiff|data_mode_forget_retain"

		# "trained_models/hf-copyright-baseline/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_2|gradient_accumulation_steps_8|num_epochs_5|model_train.weight_decay_0.01|data_harry|model_train_npo_kl|model_train.model_name_llama-2|data_mode_forget_retain"

		"trained_models2/hf-baseline-copyright/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_2|gradient_accumulation_steps_4|model_train.weight_decay_0.01|data_harry|model_train.model_name_llama-2|data_mode_forget_retain|model_train_offset_npo_kl"

		"trained_models2/hf-baseline-copyright/hf_forget_train-None/Lorar_0-lr_1e-05-layer_0/data.batch_size_2|gradient_accumulation_steps_4|model_train.weight_decay_0.01|data_harry|model_train.model_name_llama-2|model_train_offset_kl|data_mode_forget_retain"

	)

	for TARGETDIR in "${TARGETDIRS[@]}"; do
		rawcheckpoints=($(find $TARGETDIR -type d -name "checkpoint-[0-9]*"))
		# checkpoints=($(printf "%s\n" "${rawcheckpoints[@]}" | awk -F'-' '{print $NF, $0}' | sort -rn | cut -d' ' -f2-))
		checkpoints=($(printf "%s\n" "${rawcheckpoints[@]}" | awk -F'-' '{print $NF, $0}' | sort -n | cut -d' ' -f2-))

        # for checkpoint in "${checkpoints[@]}"; do
        #     checkpoint_number=$(echo "$checkpoint" | awk -F'-' '{print $NF}')
        #     if (( checkpoint_number >= 3 & )); then
        #         filtered_checkpoints+=("$checkpoint")
        #     fi
        # done

    	for ((i=0; i<${#checkpoints[@]}; i+=1)); do
    		checkpoint="${checkpoints[$i]}"
    		gpuid=$(((gpuid+1) % $num_devices))
    		checkpoint="${checkpoint//=/\\=}"

    		python scripts/eval_copyright.py \
    			data=harry \
    			model=${model} \
    			model.model_path=${checkpoint} \
    			OUTDIRNAME=$OUTDIRNAME \
    			gpu=gpu$((gpuid)) 
				# &

			if (((i+1) % $num_devices == 0)); then
                wait
                break
			fi
        done
        echo "waiting"
        wait
    done

done
