#!/bin/bash

CKPT="$1" 
# the ckpt starts with ming{SIZE}b, extract the SIZE (maybe 1.8 or 7) and save it to SIZE variable
SIZE=$(echo $CKPT | grep -oP 'qwen\K[0-9.]+(?=b)')

if [[ $CKPT == *"llama2"* ]]; then
    MODEL_BASE=/mnt/petrelfs/usr/models/llama2_7b_chat
    conv_mode="llama2"
elif [[ $CKPT == *"llama3"* ]]; then
    MODEL_BASE=/mnt/petrelfs/usr/models/Meta-Llama-3-8B-Instruct
    conv_mode="llama3"
else
    MODEL_BASE=/mnt/petrelfs/usr/models/models--Qwen--Qwen1.5-${SIZE}B-Chat
    conv_mode="qwen"
fi
domain=alpaca_eval
LOGS_BASE_PATH="./logs/diverse"

MODEL_PATH=/mnt/petrelfs/usr/checkpoints/${CKPT}

version=""


echo "Processing $domain"
mkdir -p ${LOGS_BASE_PATH}/${domain}


srun -p partition --gres=gpu:1  --quotatype=auto --output=${LOGS_BASE_PATH}/${domain}/${CKPT}${version}.infer.log python -m ming.eval.model_alpaca_gen \
    --model-path ${MODEL_BASE} \
    --answers-file ${LOGS_BASE_PATH}/${domain}/${CKPT}${version}.json \
    --s3-answers-file s3://bucket/logs/diverse/${domain}/${CKPT}${version}.json \
    --temperature 0 \
    --max-tokens 1024 \
    --keep-local \
    --conv-mode ${conv_mode} \
    --infer-answer \
    --use-logit-bias \
    --resume &

wait 

echo "Evaluating $domain"


export OPENAI_API_KEY="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"
srun -p partition alpaca_eval --model_outputs ${LOGS_BASE_PATH}/${domain}/${CKPT}${version}.json \
    --annotators_config 'alpaca_eval_gpt4_turbo_fn'