script_folder=$HOME/scripts
salt=`head /dev/urandom | tr -dc a-z0-9 | head -c4`
# first dump the scontrol results
script_file=$script_folder/`date +'%Y-%m-%d_%H-%M-%S'`_tt.txt
echo $script_file
scontrol show job | grep "Command" > $script_file
###################################################
IMAGENET_DIR=/datasets01/imagenet_full_size/061417

###################################################
TAG=coco_large_no_aug
MODEL=mae_vit_large_patch16
PRETRAIN_CHKPT=checkpoints_mae/converted/large_coco.pth

for lr in 1e-3; do
    for wd in .5; do
        for dp in .1; do
            for ep in 50; do
                JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_dp${dp}_ep${ep}
                running_this_job=`grep "${JOB_DIR}" $script_file`
                TARGET_FILE=$JOB_DIR/checkpoint-final.pth

                if [ ! -f $TARGET_FILE ] && [ -z "${running_this_job}" ]; then
                    echo "submitting: $JOB_DIR"
                    python submitit_prob.py \
                    --partition dev,learnfair \
                    --exclude learnfair055,learnfair063,learnfair071,learnfair118 \
                    --job_dir ${JOB_DIR} \
                    --nodes 8 \
                    --batch_size 16 \
                    --model $MODEL \
                    --finetune ${PRETRAIN_CHKPT} \
                    --lr ${lr} \
                    --weight_decay ${wd} \
                    --drop_path ${dp} \
                    --epochs ${ep} \
                    --warmup_epochs 20 \
                    --head_type vit_head \
                    --dist_eval \
                    --data_path ${IMAGENET_DIR} 
                fi
            done
        done
    done
done

# for lr in 1e-3; do
#     for wd in .5; do
#         for dp in .1; do
#             for ep in 50; do
#                 JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_dp${dp}_ep${ep}
#                 running_this_job=`grep "${JOB_DIR}" $script_file`
#                 TARGET_FILE=$JOB_DIR/checkpoint-final.pth

#                 if [ ! -f $TARGET_FILE ] && [ -z "${running_this_job}" ]; then
#                     echo "submitting: $JOB_DIR"
#                     python submitit_prob.py \
#                     --partition mae,learnlab,learnfair \
#                     --aws \
#                     --job_dir ${JOB_DIR} \
#                     --nodes 4 \
#                     --batch_size 32 \
#                     --model $MODEL \
#                     --finetune ${PRETRAIN_CHKPT} \
#                     --lr ${lr} \
#                     --weight_decay ${wd} \
#                     --drop_path ${dp} \
#                     --epochs ${ep} \
#                     --warmup_epochs 20 \
#                     --head_type vit_head \
#                     --dist_eval \
#                     --data_path ${IMAGENET_DIR} 
#                 fi
#             done
#         done
#     done
# done
###################################################
# TAG=aug_for_real_prob
# MODEL=mae_vit_large_patch16
# PRETRAIN_CHKPT=checkpoints_mae/released/mae_pretrain_vit_large_full.pth

# for lr in 1e-3; do
#     for wd in .5; do
#         for dp in .1; do
#             for ep in 100; do
#                 JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_dp${dp}_ep${ep}
#                 running_this_job=`grep "${JOB_DIR}" $script_file`
#                 TARGET_FILE=$JOB_DIR/checkpoint-final.pth

#                 echo "submitting: $JOB_DIR"
#                 python submitit_prob.py \
#                 --partition learnfair \
#                 --exclude learnfair055,learnfair071,learnfair118 \
#                 --job_dir ${JOB_DIR} \
#                 --nodes 4 \
#                 --batch_size 32 \
#                 --model $MODEL \
#                 --finetune ${PRETRAIN_CHKPT} \
#                 --blr ${lr} \
#                 --weight_decay ${wd} \
#                 --drop_path ${dp} \
#                 --epochs ${ep} \
#                 --head_type vit_head \
#                 --dist_eval \
#                 --data_path ${IMAGENET_DIR} \
#                 --use_augmentations
#             done
#         done
#     done
# done
# ###################################################
# TAG=aug_for_real_vis
# MODEL=mae_vit_large_patch16
# PRETRAIN_CHKPT=checkpoints_mae/released/mae_visualize_vit_large.pth

# for lr in 1e-3; do
#     for wd in .5; do
#         for dp in .1; do
#             for ep in 100; do
#                 JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_dp${dp}_ep${ep}
#                 running_this_job=`grep "${JOB_DIR}" $script_file`
#                 TARGET_FILE=$JOB_DIR/checkpoint-final.pth

#                 if [ ! -f $TARGET_FILE ] && [ -z "${running_this_job}" ]; then
#                     echo "submitting: $JOB_DIR"
#                     python submitit_prob.py \
#                     --partition learnfair \
#                     --exclude learnfair055,learnfair071,learnfair118 \
#                     --job_dir ${JOB_DIR} \
#                     --nodes 4 \
#                     --batch_size 32 \
#                     --model $MODEL \
#                     --finetune ${PRETRAIN_CHKPT} \
#                     --blr ${lr} \
#                     --weight_decay ${wd} \
#                     --drop_path ${dp} \
#                     --epochs ${ep} \
#                     --head_type vit_head \
#                     --dist_eval \
#                     --data_path ${IMAGENET_DIR} \
#                     --use_augmentations
#                 fi
#             done
#         done
#     done
# done
# ###################################################
# TAG=joint_aug
# MODEL=mae_vit_large_patch16
# PRETRAIN_CHKPT=checkpoints_mae/released/mae_pretrain_vit_large_full.pth

# for lr in 1e-4; do
#     for wd in .5; do
#         for ep in 50; do
#             for mlc in .1 1.; do
#                 JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_ep${ep}_mlc${mlc}
#                 running_this_job=`grep "${JOB_DIR}" $script_file`
#                 running_this_job=
#                 TARGET_FILE=$JOB_DIR/checkpoint-final.pth

#                 if [ ! -f $TARGET_FILE ] && [ -z "${running_this_job}" ]; then
#                     echo "submitting: $JOB_DIR"
#                     python submitit_prob.py \
#                     --partition learnfair \
#                     --exclude learnfair055,learnfair071,learnfair118 \
#                     --job_dir ${JOB_DIR} \
#                     --nodes 4 \
#                     --batch_size 8 \
#                     --accum_iter 4 \
#                     --model $MODEL \
#                     --finetune ${PRETRAIN_CHKPT} \
#                     --blr ${lr} \
#                     --weight_decay ${wd} \
#                     --drop_path .1 \
#                     --epochs ${ep} \
#                     --head_type vit_head \
#                     --dist_eval \
#                     --shared_probing \
#                     --mae_loss_coeff $mlc \
#                     --data_path ${IMAGENET_DIR} \
#                     --use_augmentations \
#                     --norm_pix_loss
#                 fi
#             done
#         done
#     done
# done
# ###################################################
# TAG=joint_no_aug
# MODEL=mae_vit_large_patch16
# PRETRAIN_CHKPT=checkpoints_mae/released/mae_pretrain_vit_large_full.pth

# for lr in 1e-4; do
#     for wd in .5; do
#         for ep in 50; do
#             for mlc in .1 1.; do
#                 JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_ep${ep}_mlc${mlc}
#                 running_this_job=`grep "${JOB_DIR}" $script_file`
#                 running_this_job=
#                 TARGET_FILE=$JOB_DIR/checkpoint-final.pth

#                 if [ ! -f $TARGET_FILE ] && [ -z "${running_this_job}" ]; then
#                     echo "submitting: $JOB_DIR"
#                     python submitit_prob.py \
#                     --partition learnfair \
#                     --exclude learnfair055,learnfair071,learnfair118 \
#                     --job_dir ${JOB_DIR} \
#                     --nodes 4 \
#                     --batch_size 8 \
#                     --accum_iter 4 \
#                     --model $MODEL \
#                     --finetune ${PRETRAIN_CHKPT} \
#                     --blr ${lr} \
#                     --weight_decay ${wd} \
#                     --drop_path .1 \
#                     --epochs ${ep} \
#                     --head_type vit_head \
#                     --dist_eval \
#                     --shared_probing \
#                     --mae_loss_coeff $mlc \
#                     --data_path ${IMAGENET_DIR} \
#                     --norm_pix_loss
#                 fi
#             done
#         done
#     done
# done
###################################################
# TAG=large_aug_post_ddl
# MODEL=mae_vit_large_patch16
# PRETRAIN_CHKPT=checkpoints_mae/released/mae_pretrain_vit_large_full.pth

# for lr in 1e-3; do
#     for wd in .3; do
#         for dp in .1; do
#             for ep in 100; do
#                 JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_dp${dp}_ep${ep}
#                 running_this_job=`grep "${JOB_DIR}" $script_file`
#                 TARGET_FILE=$JOB_DIR/checkpoint-final.pth

#                 if [ ! -f $TARGET_FILE ] && [ -z "${running_this_job}" ]; then
#                     echo "submitting: $JOB_DIR"
#                     python submitit_prob.py \
#                     --partition learnlab,learnfair \
#                     --aws \
#                     --job_dir ${JOB_DIR} \
#                     --nodes 4 \
#                     --batch_size 32 \
#                     --model $MODEL \
#                     --finetune ${PRETRAIN_CHKPT} \
#                     --lr ${lr} \
#                     --weight_decay ${wd} \
#                     --drop_path ${dp} \
#                     --epochs ${ep} \
#                     --warmup_epochs 20 \
#                     --head_type vit_head \
#                     --dist_eval \
#                     --data_path ${IMAGENET_DIR} \
#                     --use_augmentations
#                 fi
#             done
#         done
#     done
# done
###################################################
# TAG=large_no_aug_post_ddl
# MODEL=mae_vit_large_patch16
# PRETRAIN_CHKPT=checkpoints_mae/released/mae_pretrain_vit_large_full.pth

# for lr in 1e-3; do
#     for wd in .5; do
#         for dp in .1; do
#             for ep in 50 80 100; do
#                 JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_dp${dp}_ep${ep}
#                 running_this_job=`grep "${JOB_DIR}" $script_file`
#                 TARGET_FILE=$JOB_DIR/checkpoint-final.pth

#                 if [ ! -f $TARGET_FILE ] && [ -z "${running_this_job}" ]; then
#                     echo "submitting: $JOB_DIR"
#                     python submitit_prob.py \
#                     --partition learnlab,learnfair \
#                     --aws \
#                     --job_dir ${JOB_DIR} \
#                     --nodes 4 \
#                     --batch_size 32 \
#                     --model $MODEL \
#                     --finetune ${PRETRAIN_CHKPT} \
#                     --lr ${lr} \
#                     --weight_decay ${wd} \
#                     --drop_path ${dp} \
#                     --epochs ${ep} \
#                     --warmup_epochs 20 \
#                     --head_type vit_head \
#                     --dist_eval \
#                     --data_path ${IMAGENET_DIR} 
#                 fi
#             done
#         done
#     done
# done
###################################################
# TAG=huge_aug_post_ddl
# MODEL=mae_vit_huge_patch14
# PRETRAIN_CHKPT=checkpoints_mae/released/mae_pretrain_vit_huge_full.pth

# for lr in 1e-3; do
#     for wd in .3; do
#         for dp in .1; do
#             for ep in 100; do
#                 JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_dp${dp}_ep${ep}
#                 running_this_job=`grep "${JOB_DIR}" $script_file`
#                 TARGET_FILE=$JOB_DIR/checkpoint-final.pth

#                 if [ ! -f $TARGET_FILE ] && [ -z "${running_this_job}" ]; then
#                     echo "submitting: $JOB_DIR"
#                     python submitit_prob.py \
#                     --partition learnlab,learnfair \
#                     --aws \
#                     --job_dir ${JOB_DIR} \
#                     --nodes 4 \
#                     --batch_size 32 \
#                     --model $MODEL \
#                     --finetune ${PRETRAIN_CHKPT} \
#                     --lr ${lr} \
#                     --weight_decay ${wd} \
#                     --drop_path ${dp} \
#                     --epochs ${ep} \
#                     --warmup_epochs 20 \
#                     --head_type vit_head \
#                     --dist_eval \
#                     --data_path ${IMAGENET_DIR} \
#                     --use_augmentations
#                 fi
#             done
#         done
#     done
# done
###################################################
# TAG=huge_no_aug_post_ddl
# MODEL=mae_vit_huge_patch14
# PRETRAIN_CHKPT=checkpoints_mae/released/mae_pretrain_vit_huge_full.pth

# for lr in 1e-3; do
#     for wd in .5; do
#         for dp in .1; do
#             for ep in 50 80 100; do
#                 JOB_DIR=checkpoints/${TAG}_lr${lr}_wd${wd}_dp${dp}_ep${ep}
#                 running_this_job=`grep "${JOB_DIR}" $script_file`
#                 TARGET_FILE=$JOB_DIR/checkpoint-final.pth

#                 if [ ! -f $TARGET_FILE ] && [ -z "${running_this_job}" ]; then
#                     echo "submitting: $JOB_DIR"
#                     python submitit_prob.py \
#                     --partition learnlab,learnfair \
#                     --aws \
#                     --job_dir ${JOB_DIR} \
#                     --nodes 4 \
#                     --batch_size 32 \
#                     --model $MODEL \
#                     --finetune ${PRETRAIN_CHKPT} \
#                     --lr ${lr} \
#                     --weight_decay ${wd} \
#                     --drop_path ${dp} \
#                     --epochs ${ep} \
#                     --warmup_epochs 20 \
#                     --head_type vit_head \
#                     --dist_eval \
#                     --data_path ${IMAGENET_DIR} 
#                 fi
#             done
#         done
#     done
# done
###################################################
rm -rf $script_file
echo ">>>>> $salt <<<<<"