epsilon_greedy_policy.h

Go to the documentation of this file.

#ifndef EPSILON_GREEDY_POLICY_H
#define EPSILON_GREEDY_POLICY_H
 
#include "cuberl/base/cubeai_config.h"
#include "cuberl/base/cuberl_types.h"
#include "cuberl/rl/policies/max_tabular_policy.h"
#include "cuberl/rl/policies/random_tabular_policy.h"
 
#ifdef USE_PYTORCH
#include "cuberl/utils/torch_adaptor.h"
#endif
 
#include <random>
#include <cmath>
 
namespace cuberl {
namespace rl {
namespace policies {
 
 
enum class EpsilonDecayOption{NONE, EXPONENTIAL, INVERSE_STEP, CONSTANT_RATE};
 
class EpsilonGreedyPolicy
{
public:
 
    
    typedef uint_t output_type;
    
    constexpr static real_t MIN_EPS = 0.01;
    constexpr static real_t MAX_EPS = 1.0;
    constexpr static real_t EPSILON_DECAY_FACTOR = 0.01;
 
    EpsilonGreedyPolicy(real_t eps);
 
    explicit EpsilonGreedyPolicy(real_t eps, uint_t seed);
 
    explicit EpsilonGreedyPolicy(real_t eps, uint_t seed, 
                                 EpsilonDecayOption decay_op,
                                 real_t min_eps = MIN_EPS,
                                 real_t max_eps=MAX_EPS,
                                 real_t epsilon_decay = EPSILON_DECAY_FACTOR);
 
    template<typename MapType>
    output_type operator()(const MapType& q_map, uint_t state)const;
 
 
    template<typename VecType>
    output_type operator()(const VecType& vec)const;
    
    template<typename MatType>
    output_type get_action(const MatType& q_map, uint_t state_idx);
    
    template<typename VecTp>
    output_type get_action(const VecTp& q_map);
 
#ifdef USE_PYTORCH
    output_type operator()(const torch_tensor_t& vec, torch_tensor_value_type<real_t>)const;
    output_type operator()(const torch_tensor_t& vec, torch_tensor_value_type<float_t>)const;
    output_type operator()(const torch_tensor_t& vec, torch_tensor_value_type<int_t>)const;
    output_type operator()(const torch_tensor_t& vec, torch_tensor_value_type<lint_t>)const;
#endif
 
    void on_episode(uint_t episode_idx)noexcept;
 
    void reset()noexcept{eps_ = eps_init_;}
 
    real_t eps_value()const noexcept{return eps_;}
    
    void set_eps_value(real_t eps);
 
    EpsilonDecayOption decay_option()const noexcept{return decay_op_;}
 
 
private:
 
    real_t eps_init_;
    real_t eps_;
    real_t min_eps_;
    real_t max_eps_;
    real_t epsilon_decay_;
    EpsilonDecayOption decay_op_;
 
    mutable std::mt19937 generator_;
 
    // how to select the action
    RandomTabularPolicy random_policy_;
    MaxTabularPolicy max_policy_;
};
 
inline
EpsilonGreedyPolicy::EpsilonGreedyPolicy(real_t eps, uint_t seed, EpsilonDecayOption decay_op,
                                         real_t min_eps, real_t max_eps, real_t epsilon_decay)
:
eps_init_(eps),
eps_(eps),
min_eps_(min_eps),
max_eps_(max_eps),
epsilon_decay_(epsilon_decay),
decay_op_(decay_op),
generator_(seed),
random_policy_(seed),
max_policy_()
{}
 
inline
EpsilonGreedyPolicy::EpsilonGreedyPolicy(real_t eps)
    :
      eps_init_(eps),
      eps_(eps),
      min_eps_(eps),
      max_eps_(eps),
      epsilon_decay_(eps),
      decay_op_(EpsilonDecayOption::NONE),
      random_policy_(),
      max_policy_()
{}
 
inline
EpsilonGreedyPolicy::EpsilonGreedyPolicy(real_t eps, uint_t seed)
    :
    EpsilonGreedyPolicy(eps, seed, EpsilonDecayOption::NONE,
                        eps, eps, eps)
{}
 
 
template<typename VecType>
EpsilonGreedyPolicy::output_type
EpsilonGreedyPolicy::operator()(const VecType& vec)const{
 
    // generate a number in [0, 1]
    std::uniform_real_distribution<> real_dist_(0.0, 1.0);
 
    if(real_dist_(generator_) > eps_){
        // select greedy action with probability 1 - epsilon
        return max_policy_.get_action(vec);
    }
 
    // else select a random action
    return random_policy_(vec);
}
 
 
template<typename VecTp>
EpsilonGreedyPolicy::output_type
EpsilonGreedyPolicy::get_action(const VecTp& vec){
    // generate a number in [0, 1]
    std::uniform_real_distribution<> real_dist_(0.0, 1.0);
 
    if(real_dist_(generator_) > eps_){
        // select greedy action with probability 1 - epsilon
        return max_policy_.get_action(vec);
    }
 
    // else select a random action
    return random_policy_(vec);
}
 
}
}
}
 
#endif // EPSILON_GREEDY_POLICY_H