function runner_hemisphereal_methodn(n, ratio, balance, repeat)
% this file is the runner of the active learning averaging over all 12
% annotators using consensus as the ground truth
home_path = "../../../ActSort/";
addpath(genpath(home_path));

folderName = "./results";
% Check if the folder exists
if ~exist(folderName, 'dir')
    % If it doesn't exist, create the folder
    mkdir(folderName);
    fprintf('[INFO] : Directory "%s" has been created.\n', folderName);
else
    fprintf('[INFO] : Directory "%s" already exists.\n', folderName);
end

load('../../data/hemisphere_dataset_summary.mat')
%% define the annotator indices
annotator_lst = [1, 2, 3, 4; % dataset 1
                 1, 3, 4, 5; % dataset 2
                 1, 2, 3, 6];% dataset 3
%% define the hyperparameters
config.DO_ZSCORING = true;
config.n = 1;
config.repeat  = repeat;
config.balance = balance;

method_rand = struct("name", "random", "continue", false);
method_cal  = struct("name", "cal", "continue", false);
method_dal  = struct("name", "dal", "continue", false);
methods     = {method_rand, method_cal, method_dal};
% initializing dcal method
weights_lst = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9];
for i=1:length(weights_lst)
    weight = weights_lst(i);
    method_dcal = struct("name", "dcal", "weight", weight, "continue", false);
    methods{end+1} = method_dcal;
end

% % initializing multi-arm bandit exp3 method
%  [reward_func_lst, reward_name_lst] = get_reward_funcs([3, 7, 8, 10]); % return all predefined reward function0
%  gamma_lst = [0.3, 0.5];
%  for i=1:length(reward_func_lst)
%      for g=1:length(gamma_lst)
%          gamma       = gamma_lst(g);
%          reward_func = reward_func_lst{i};
%          reward_name = reward_name_lst(i);
%          method_mab  = struct("name", "mab-exp3", "gamma", gamma, ...
%                        "reward_func", reward_func, "reward_name", reward_name);
%          methods{end+1} = method_mab;
%      end
%  end
% 
% % initializing multi-arm ubc method
% alpha_lst = [sqrt(2)];
% for i=1:length(alpha_lst)
%     alpha      = alpha_lst(i);
%     method_ucb = struct("name", "mab-ucb", "alpha", alpha);
%     methods{end+1} = method_ucb;
% end

clear g i gamma method_cal method_cal method_dcal method_mab method_ucb reward_func reward_name weight;
methods = methods(n);
num_methods = length(methods);
%% ActSort 
eval_lst    = cell(3, 4, num_methods);
for d = 1:3
    for ann = 1:4
        ann_idx = annotator_lst(d, ann);
        choices    = choices_all{ann_idx,d};
        choices_gt = choices_gt_all{ann_idx,d};
        metrics    = metrics_all{d};
        
        k = 1;
        method = methods{k};
        dspname = get_legend_name(method);

        [fullPath, ckpt_exist_flag] = check_ckpt_exist(folderName, dspname, d, ann);

        if ckpt_exist_flag
            fprintf("%s: Data Pair %i Annotators %i Exist! Load eval_metrics ===>", datetime("now"), d, ann);
            load(fullPath, 'eval_metrics')
            eval_lst{d, ann, k} = eval_metrics;
            fprintf("update eval_lst. \n")
        else
            fprintf("%s: Working on dataset %i, annotator %i using method %s\n",datetime("now"), d, ann, dspname);
            [valid, eval_metrics, dataset] = play_active_learning_new(metrics,choices,choices_gt,ratio,method,config);
            eval_lst{d, ann, k} = eval_metrics;
            ckpt_name = folderName+"/ckpt-"+dspname+"-d"+num2str(d)+"-ann"+num2str(ann)+".mat";
            save(ckpt_name, 'eval_metrics', '-v7.3');
            fprintf("[INFO] : checkpoint %s saved\n", ckpt_name);
        end
    end
end
save_name = folderName+"/runner_hemisphereal_method-"+dspname+".mat";
save(save_name, '-v7.3')
fprintf("[INFO] : finished running, %s saved", save_name);
end

function [fullPath, ckpt_exist] = check_ckpt_exist(folderName, dspname, d, ann)
    directory = folderName;
    filename = sprintf('ckpt-%s-d%i-ann%i.mat', dspname, d, ann);
    
    % Combine the directory and filename to create the full file path
    fullPath = fullfile(directory, filename);
    
    % Check if the file exists
    % 'exist' returns 2 if the file exists
    if exist(fullPath, 'file') == 2
        ckpt_exist = true;
    else
        ckpt_exist = false;
    end

end