function runner_crossmice_aug_methodn(n, d1_ratio, d2_ratio, threshold, balance, balance_pretrained,align,repeat)
% this file is the runner of the active learning fon cross-mice experiment
% it includes 6 dataset pairs, 16 annotators for each dataset pairs
%% load data and create folders
home_path = "../../../ActSort/";
addpath(genpath(home_path));

folderName = sprintf("./results-d1ratio%s-d2ratio%s-balance%s-balancept%s-align%s-repeat%s", ...
                    num2str(d1_ratio), num2str(d2_ratio), num2str(balance), num2str(balance_pretrained), num2str(align), num2str(repeat));
% Check if the folder exists
if ~exist(folderName, 'dir')
    % If it doesn't exist, create the folder
    mkdir(folderName);
    fprintf('[INFO] : Directory "%s" has been created.\n', folderName);
else
    fprintf('[INFO] : Directory "%s" already exists.\n', folderName);
end

% rng(19991112)
load('../../data/hemisphere_dataset_summary.mat')
%% define the annotator indices
days_lst = [1,2; 1,3; 2,1; 2,3; 3,1; 3,2];

annotator_lst = [1, 2, 3, 4; % dataset1
                 1, 3, 4, 5; % dataset2
                 1, 2, 3, 6];% dataset3
%% define hyperparameters
config_pretrain.DO_ZSCORING = true;
config_pretrain.n = 1;
config_pretrain.repeat = 1;
config_pretrain.balance = balance; % for balancing dataset when training from scratch
config_pretrain.balance_pretrained = balance_pretrained; % for balancing dataset when pretraining
config_pretrain.align = align;

config_finetune.DO_ZSCORING = true;
config_finetune.n = 1;
config_finetune.repeat = repeat;
config_finetune.balance = balance; % for balancing dataset when training from scratch
config_finetune.balance_pretrained = balance_pretrained; % for balancing dataset when pretraining
config_finetune.align = align;

method_rand = struct("name", "random", "continue", false);
method_cal  = struct("name", "cal", "continue", false);
method_dal  = struct("name", "dal", "continue", false);
methods     = {method_rand, method_cal, method_dal};

weights_lst = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9];
for i=1:length(weights_lst)
    weight = weights_lst(i);
    method_dcal = struct("name", "dcal", "weight", weight, "continue", false);
    methods{end+1} = method_dcal;
end

% % initializing multi-arm bandit exp3 method
%  [reward_func_lst, reward_name_lst] = get_reward_funcs([3, 7, 8, 10]); % return all predefined reward function0
%  gamma_lst = [0.3, 0.5];
%  for i=1:length(reward_func_lst)
%      for g=1:length(gamma_lst)
%          gamma       = gamma_lst(g);
%          reward_func = reward_func_lst{i};
%          reward_name = reward_name_lst(i);
%          method_mab  = struct("name", "mab-exp3", "gamma", gamma, ...
%                        "reward_func", reward_func, "reward_name", reward_name);
%          methods{end+1} = method_mab;
%      end
%  end
% 
% % initializing multi-arm ubc method
% alpha_lst = [sqrt(2)];
% for i=1:length(alpha_lst)
%     alpha      = alpha_lst(i);
%     method_ucb = struct("name", "mab-ucb", "alpha", alpha);
%     methods{end+1} = method_ucb;
% end

clear g i gamma method_cal method_cal method_dcal method_mab reward_func reward_name weight;

methods = methods(n);

num_methods = length(methods);
num_dparis  = size(days_lst,1);

%% ActSort
eval_lst_ft = cell(num_dparis, 4, 4, num_methods);
eval_lst_sc = cell(num_dparis, 4, 4, num_methods);
for dpair = 1:num_dparis
    d1 = days_lst(dpair, 1);
    d2 = days_lst(dpair, 2);
    metrics_d1    = metrics_all{d1};
    metrics_d2    = metrics_all{d2};
    for ann1 = 1:4
        ann1_name = annotator_lst(d1, ann1);
        choices1    = choices_all{ann1_name, d1};
        choices1_gt = choices_gt_all{ann1_name, d1};
        for ann2 = 1:4
            ann2_name = annotator_lst(d2, ann2);
            choices2    = choices_all{ann2_name, d2};
            choices2_gt = choices_gt_all{ann2_name, d2}; 
            
            k = 1;
            method = methods{k};
            dspname = get_legend_name(method);

            [fullPath, ckpt_exist_flag] = check_ckpt_exist(folderName, dspname, dpair, ann1, ann2);
            if ckpt_exist_flag
                fprintf("%s: Data Pair %i Annotators %i-%i Exist! Load eval_metrics ===>", datetime("now"), dpair, ann1, ann2);
                load(fullPath, 'eval_metrics_ft')
                eval_lst_ft{dpair, ann1, ann2, k} = eval_metrics_ft;
                fprintf("update eval_lst. \n")
            else
                pretrain_method = struct("name", "cal", "continue", false);
                pm_dspname = get_legend_name(pretrain_method);
                fprintf("%s: Start training on dataset %i (%.2f) with Annotators %i-%i using method %s ...", datetime("now"), d1, d1_ratio, ann1, ann2, pm_dspname);
                [~, ~, dataset] = play_active_learning_new(metrics_d1, choices1, choices1_gt,...
                                                            d1_ratio, pretrain_method, config_pretrain);

                fprintf("Fine-tuning on dataset %i (%.2f) using method %s ...", d2, d2_ratio, dspname);
                pretrained           = dataset;
                pretrained.threshold = threshold;
%                 labels_exml = combine_exml_labels(pretrained);
%                 pretrained.labels_ml = labels_exml;
%                   pretrained = balance_pretrained_dataset(pretrained);
                method.continue = true;
                method.mdl      = dataset.mdl;
                method.pretrained  = pretrained;
                [~, eval_metrics_ft, dataset_ft] = play_active_learning_new(metrics_d2, choices2,choices2_gt,...
                                                                            d2_ratio, method, config_finetune);

%                     fprintf("Start training on dataset %i (%.2f)", d2, d2_ratio);
%                     method = methods{k};
%                     [~, eval_metrics_sc, dataset_sc] = play_active_learning_new(metrics_d2, choices2, choices2_gt,...
%                                                                                 d2_ratio, method, config);
                eval_lst_ft{dpair, ann1, ann2, k} = eval_metrics_ft;
%                     eval_lst_sc{dpair, ann1, ann2, k} = eval_metrics_sc;
                fprintf('\n');
                ckpt_name = folderName+"/ckpt-"+dspname+"-dp"+num2str(dpair)+"-ann"+num2str(ann1)+"-ann"+num2str(ann2)+".mat";
                save(ckpt_name, 'eval_metrics_ft', '-v7.3');
                fprintf("[INFO] : checkpoint %s saved\n", ckpt_name);
            end
        end
    end 
end
%% save
save_name = folderName+"/runner_crossmice_aug_method-"+dspname+".mat";
save(save_name, '-v7.3')
fprintf("[INFO] : finished running, %s saved", save_name);
end

function [fullPath, ckpt_exist] = check_ckpt_exist(folderName, dspname, dp, ann1, ann2)
    directory = folderName;
    filename = sprintf('ckpt-%s-dp%i-ann%i-ann%i.mat', dspname, dp, ann1, ann2);
    
    % Combine the directory and filename to create the full file path
    fullPath = fullfile(directory, filename);
    
    % Check if the file exists
    % 'exist' returns 2 if the file exists
    if exist(fullPath, 'file') == 2
        ckpt_exist = true;
    else
        ckpt_exist = false;
    end

end