
%% Parameter definition

clear; close;clc
tic

seed = randi(1000);
seed=814;
rng(seed);

Q = rand(25,25);
Theta = -40:10:40;
NUser = length(Theta);
SNR_dB = 15*ones(NUser,1);
SNR_dB([1,3]) = 5;


sigmas = 10.^(SNR_dB/10);
sigmas2_assume = 100;

th = 0.2;
alpha = .5;
discount = .5;
n_loop = 10;
state_theta = cell(n_loop,1);
epsilon = 0.4;
%% discretized probability
deltaTheta = 0.1;
Theta_ = [-90:deltaTheta:90];

%% Signal generation


pmf = 1/length(Theta_)*ones(1,length(Theta_)); % uniform

% Intial state
p = get_phi(Theta,pmf,SNR_dB);
pmf1 = p/max(p);
pmf11 = p/sum(p);
thh = 0.2*max(pmf11);
fig_gen_rein(pmf11,thh,Theta);
close
states = states_comp(pmf1,th,Theta_);
Q_state = Q(:,states);
[action,highest_theta,theta_loc,highest_theta_actions,action_loc,higher2,pmf21] = generating_action(pmf1,th,Theta_,Q_state,epsilon,seed);
thh = 0.2*max(pmf21);
fig_gen_rein(pmf21,thh,Theta);


for ioutloop = 1:5


    states_old = states;

    action_old = action;

    %% Actions
    %



    %for a_loop = 1:length(action_space)


    % [highest_theta_actions,action_loc] = theta_pmf2(pmf2,action,Theta_);

    action_theta= highest_theta_actions;




    pmf_action = zeros(size(Theta_));
    pmf_action(action_loc) = 1;
    pmf_action = pmf_action/sum(pmf_action);
    diff_loc = setdiff(action_loc,theta_loc);
    pmf_action(diff_loc) = 1e3;

    fig_gen_rein(pmf_action,Theta)

    p_out = get_phi(Theta,pmf_action,SNR_dB);


    states = states_comp(p_out/max(p_out),th,Theta_);

    Q_state = Q(:,states);

    %for a_loop = 1:length(action_space)

    %% Reward
    R1 = sum(p_out(theta_loc));
    R2 = sum(p_out(setdiff(1:length(Theta_),higher2)));
    R3 = action-states;

    R = 10*R1-R2-2*R3;
    [action,highest_theta,theta_loc,highest_theta_actions,action_loc,higher2] = generating_action(p_out/max(p_out),th,Theta_,Q_state,epsilon,seed);

    % states_update = states_comp(p_out,th,Theta_);
    %
    Q(states_old,action_old) = Q(states_old,action_old)+ alpha*(R+discount* Q(states,action)-Q(states_old,action_old));
    Q_state = Q(:,states);
    pmf = p_out/sum(p_out);

    %% try 
    pmf = pmf_action;
    %%


    % figure(2 + ioutloop*10)
    % plot([-90:0.1:90], abs(p))
    % xlim([-90 90])
    % xlabel('Angle (degree)')
    % ylabel('Estimated spectrum')
    % fontsize(16)
    % vline(Theta);




end



toc

function fig_gen_rein(p,th,Theta)
   
    plot([-90:0.1:90], abs(p))
    hold on 
    yline(th,"k--",'LineWidth',1.5)
    xlim([-90 90])
    xlabel('Angle (degree)')
    ylabel('PMF')
    fontsize(16)
    vline(Theta);
end