function runner_oscar_methodn(n, ratio, balance, repeat)
% this file is the runner of the active learning averaging over all 12
% annotators using consensus as the ground truth
home_path = "../../../ActSort/";
addpath(genpath(home_path));

folderName = sprintf("./results-ratio%s-balance%i-repeat%i", num2str(ratio), balance, repeat);
% Check if the folder exists
if ~exist(folderName, 'dir')
    % If it doesn't exist, create the folder
    mkdir(folderName);
    fprintf('[INFO] : Directory "%s" has been created.\n', folderName);
else
    fprintf('[INFO] : Directory "%s" already exists.\n', folderName);
end

load('./twophoton_motion.mat')
%%
annotators = cell(1,4);
annotators{1} = cheetah_choices;
annotators{2} = dragon_choices;
annotators{3} = koala_choices;
annotators{4} = panda_choices;

evaluators = cell(1,4);
evaluators{1} = 2*((dragon_choices+koala_choices+panda_choices)>0)-1;
evaluators{2} = 2*((cheetah_choices+koala_choices+panda_choices)>0)-1;
evaluators{3} = 2*((cheetah_choices+dragon_choices+panda_choices)>0)-1;
evaluators{4} = 2*((cheetah_choices+dragon_choices+koala_choices)>0)-1;
%% define the hyperparameters
config.DO_ZSCORING = true;
config.n = 1;
config.repeat  = repeat;
config.balance = balance;
lam_all = [1];

method_rand = struct("name", "random", "continue", false);
method_cal  = struct("name", "cal", "continue", false);
method_dal  = struct("name", "dal", "continue", false);
methods     = {method_rand, method_cal, method_dal};
% initializing dcal method
weights_lst = [0.3,0.5,0.7];
for i=1:length(weights_lst)
    weight = weights_lst(i);
    method_dcal = struct("name", "dcal", "weight", weight, "continue", false);
    methods{end+1} = method_dcal;
end

% % initializing multi-arm bandit exp3 method
%  [reward_func_lst, reward_name_lst] = get_reward_funcs([3, 7, 8, 10]); % return all predefined reward function0
%  gamma_lst = [0.3, 0.5];
%  for i=1:length(reward_func_lst)
%      for g=1:length(gamma_lst)
%          gamma       = gamma_lst(g);
%          reward_func = reward_func_lst{i};
%          reward_name = reward_name_lst(i);
%          method_mab  = struct("name", "mab-exp3", "gamma", gamma, ...
%                        "reward_func", reward_func, "reward_name", reward_name);
%          methods{end+1} = method_mab;
%      end
%  end
% 
% % initializing multi-arm ubc method
% alpha_lst = [sqrt(2)];
% for i=1:length(alpha_lst)
%     alpha      = alpha_lst(i);
%     method_ucb = struct("name", "mab-ucb", "alpha", alpha);
%     methods{end+1} = method_ucb;
% end

clear g i gamma method_cal method_cal method_dcal method_mab method_ucb reward_func reward_name weight;
methods = methods(n);
num_methods = length(methods);
%% ActSort 
eval_lst    = cell(1,4,num_methods);


config.lam = "auto";
for ll = 1:1
    config.lam = lam_all(ll);
    for ann = 1:4
        choices    = annotators{ann};
        choices_gt = evaluators{ann};

        k = 1;
        method = methods{k};
        dspname = get_legend_name(method);
        [fullPath, ckpt_exist_flag] = check_ckpt_exist(folderName, dspname, ann);

        if ckpt_exist_flag
            fprintf("%s: Annotators %i Exist! Load eval_metrics ===>", datetime("now"), ann);
            load(fullPath, 'eval_metrics')
            eval_lst{ll, ann, k} = eval_metrics;
            fprintf("update eval_lst. \n")
        else
            fprintf("%s: Working on annotator %i using method %s",datetime("now"), ann, dspname);
            [valid, eval_metrics, dataset] = play_active_learning_new(metrics,choices,choices_gt,ratio,method,config);
            eval_lst{ll, ann, k} = eval_metrics;
            ckpt_name = folderName+"/ckpt-"+dspname+"-d"+"-ann"+num2str(ann)+".mat";
            save(ckpt_name, 'eval_metrics', '-v7.3');
            fprintf("\n")
            fprintf("[INFO] : checkpoint %s saved\n", ckpt_name);
        end


        fprintf("%s: Working on lam %d. annotator %i using method %s", datetime("now"), ll, ann, dspname);
        [valid, eval_metrics, dataset] = play_active_learning_new(metrics,choices,choices_gt,ratio,method,config);
        eval_lst{ll,ann, k} = eval_metrics;
    end
end

save_name = folderName+"/runner_oscar_method-"+dspname+".mat";
save(save_name, '-v7.3')
fprintf("[INFO] : finished running, %s saved\n", save_name);
end

function [fullPath, ckpt_exist] = check_ckpt_exist(folderName, dspname, ann)
    directory = folderName;
    filename = sprintf('ckpt-%s-ann%i.mat', dspname, ann);
    
    % Combine the directory and filename to create the full file path
    fullPath = fullfile(directory, filename);
    
    % Check if the file exists
    % 'exist' returns 2 if the file exists
    if exist(fullPath, 'file') == 2
        ckpt_exist = true;
    else
        ckpt_exist = false;
    end

end