function runner_batchprocess_methodn(n, ratio, balance, repeat)
% this file is the runner of the active learning on batch processing using
% consensus as the groundtruth for each dataset. 
home_path = '../../../ActSort/';
addpath(genpath(home_path));

folderName = sprintf('./results-ratio%s-balance%s-repeat%s', num2str(ratio), num2str(balance), num2str(repeat));

% Check if the folder exists
if ~exist(folderName, 'dir')
    % If it doesn't exist, create the folder
    mkdir(folderName);
    fprintf('[INFO] : Directory "%s" has been created.\n', folderName);
else
    fprintf('[INFO] : Directory "%s" already exists.\n', folderName);
end
% clear;clc;
% rng(19991112)
%% load data
load('../../data/hemisphere_dataset_summary.mat')

annotator_lst = [1, 2, 3, 4; % dataset1
                 1, 3, 4, 5; % dataset2
                 1, 2, 3, 6];% dataset3

metrics_3d   = [];
for d=1:3
    metrics     = metrics_all{d};
    metrics_3d   = [metrics_3d, metrics];
end
clear d;
%% define the hyperparameters
%     ratio = 0.1;

config.DO_ZSCORING = true;
config.n = 1;
config.repeat = repeat;
config.balance = balance;

method_rand = struct("name", "random");
method_cal  = struct("name", "cal");
method_dal  = struct("name", "dal");
methods     = {method_rand, method_cal, method_dal};

weights_lst = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9];
for i=1:length(weights_lst)
    weight = weights_lst(i);
    method_dcal = struct("name", "dcal", "weight", weight);
    methods{end+1} = method_dcal;
end

% % initializing multi-arm bandit exp3 method
%  [reward_func_lst, reward_name_lst] = get_reward_funcs([3, 7, 8, 10]); % return all predefined reward function0
%  gamma_lst = [0.3,0.5];
%  for i=1:length(reward_func_lst)
%      for g=1:length(gamma_lst)
%          gamma       = gamma_lst(g);
%          reward_func = reward_func_lst{i};
%          reward_name = reward_name_lst(i);
%          method_mab  = struct("name", "mab-exp3", "gamma", gamma, ...
%                        "reward_func", reward_func, "reward_name", reward_name);
%          methods{end+1} = method_mab;
%      end
%  end
% 
% % initializing multi-arm ubc method
% alpha_lst = [sqrt(2)];
% for i=1:length(alpha_lst)
%     alpha      = alpha_lst(i);
%     method_ucb = struct("name", "mab-ucb", "alpha", alpha);
%     methods{end+1} = method_ucb;
% end

clear g i gamma method_cal method_cal method_dcal method_mab reward_func reward_name weight;
methods = methods(n);
num_methods = length(methods);
%% ActSort 
eval_lst    = cell(4, 4, 4, num_methods);
for ann1 = 1:4 % annotator for dataset 1
    ann1_name    = annotator_lst(1, ann1);
    choices_ann1 = choices_all{ann1_name, 1};
    choices_gt1  = choices_gt_all{ann1_name, 1};
    for ann2 = 1:4 % annotator for dataset 2
        ann2_name    = annotator_lst(2, ann2);
        choices_ann2 = choices_all{ann2_name, 2};
        choices_gt2  = choices_gt_all{ann2_name, 2};
        for ann3 = 1:4 % annotator for dataset 3
            ann3_name    = annotator_lst(3, ann3);
            choices_ann3 = choices_all{ann3_name, 3};
            choices_gt3  = choices_gt_all{ann3_name, 3};
            % concatenate the choices and choices_gt
            choices_3d   = [choices_ann1, choices_ann2, choices_ann3];
            choices_3dgt = [choices_gt1, choices_gt2, choices_gt3];
            
            % play active learning
            k=1;
            method = methods{k};
            dspname = get_legend_name(method);

            [ckpt_exist_flag, fullPath] = check_ckpt_exist(folderName, dspname, ann1, ann2, ann3);

            if ckpt_exist_flag
                fprintf("%s: Annotators %i-%i-%i using method %s Exist! Load eval metrics==>\n", datetime("now"), ann1, ann2, ann3,dspname);
                load(fullPath, 'eval_metrics');
                eval_lst{ann1, ann2, ann3, k} = eval_metrics;
            else
                fprintf("%s: Working on Annotators %i-%i-%i using method %s...", datetime("now"), ann1, ann2, ann3,dspname);
                [valid, eval_metrics, dataset] = play_active_learning_new(metrics_3d,choices_3d,choices_3dgt,ratio,method,config);
                eval_lst{ann1, ann2, ann3, k} = eval_metrics;
                
                ckpt_name = folderName+"/ckpt-"+dspname+"-ann"+num2str(ann1)+"-ann"+num2str(ann2)+"-ann"+num2str(ann3)+".mat";
                save(ckpt_name, 'eval_metrics', '-v7.3');
                fprintf("\n");
                fprintf("[INFO] : checkpoint %s saved\n", ckpt_name);
            end

        end
    end
end
%% save
save_name = folderName+'/runner_batchprocess_'+dspname+'.mat';
save(save_name, '-v7.3')
fprintf("[INFO] : finished running, %s saved", save_name);
end

function [ckpt_exist, fullPath] = check_ckpt_exist(folderName, dspname, ann1, ann2, ann3)
directory = folderName;
filename = sprintf('ckpt-%s-ann%i-ann%i-ann%i.mat', dspname, ann1, ann2, ann3);

% Combine the directory and filename to create the full file path
fullPath = fullfile(directory, filename);

% Check if the file exists
% 'exist' returns 2 if the file exists
if exist(fullPath, 'file') == 2
    ckpt_exist = true;
else
    ckpt_exist = false;
end

end