
clear
clc
load('hemisphere_dataset_summary.mat')

vals = zeros(6,6,3,2)+NaN;

for i = 1:6
    for j = 1:6
        for d=1:3
            annotator = choices_all{j,d};
            evaluator = choices_all{i,d};
            try
                ind = find(evaluator==1);
                accuracy_accept = nanmean(annotator(ind) == evaluator(ind));
                ind = find(evaluator==-1);
                accuracy_reject = nanmean(annotator(ind) == evaluator(ind));
    
                fprintf('Evaluator %d Annotator %d Movie %d Accept %.3f Reject %.3f \n', ...
                    i,j,d,accuracy_accept,accuracy_reject)
              
            vals(i,j,d,1) =  accuracy_accept;
            vals(i,j,d,2) = accuracy_reject;
            catch
                fprintf('Evaluator %d Annotator %d Movie %d NaN \n',...
                    i,j,d)
            end

        end
    end
end