max_tabular_policy.h

Go to the documentation of this file.

#ifndef MAX_TABULAR_POLICY_H
#define MAX_TABULAR_POLICY_H
 
#include "cuberl/base/cubeai_config.h"
#include "cuberl/base/cuberl_types.h"
#include "cuberl/rl/algorithms/utils.h"
#include "cuberl/maths/vector_math.h"
 
#ifdef USE_PYTORCH
#include <torch/torch.h>
#endif
 
#include <type_traits>
#include <vector>
#include <string>
#include <iostream>
 
namespace cuberl {
namespace rl {
namespace policies {
    
struct MaxTabularPolicyBuilder;
    
class MaxTabularPolicy
{
public:
 
    typedef uint_t output_type;
    typedef uint_t state_type;
    typedef uint_t action_type;
    
    template<typename MatType>
    static output_type get_action(const MatType& q_map, uint_t state_idx);
    
    template<typename VecTp>
    static output_type get_action(const VecTp& q_map);
    
#ifdef USE_PYTORCH
    static output_type get_action(const torch_tensor_t& vec);
#endif
 
    friend struct MaxTabularPolicyBuilder;
 
    MaxTabularPolicy()=default;
 
    void on_episode(uint_t)noexcept{}
 
    void reset()noexcept{state_action_map_.clear();}
    
    action_type on_state(state_type s)const{return state_action_map_[s];}
    
    void save(const std::string& filename)const;
    
private:
 
        std::vector<uint_t> state_action_map_;
};
 
 
#ifdef USE_PYTORCH
inline
uint_t
MaxTabularPolicy::get_action(const torch_tensor_t& vec){
    return torch::argmax(vec).item<uint_t>();
}
#endif
 
 
template<typename VecTp>
MaxTabularPolicy::action_type
MaxTabularPolicy::get_action(const VecTp& vec){
 
    return std::distance(vec.begin(),
                         std::max_element(vec.begin(),
                                          vec.end()));
 
}
 
 
struct MaxTabularPolicyBuilder
{
 
    template<typename EnvType>
    void build_from_state_function(const EnvType& env,
                                   const DynVec<real_t>& v,
                                   real_t gamma, 
                                   MaxTabularPolicy& policy);
                                   
    void build_from_state_action_function(const DynMat<real_t>& q,
                                          MaxTabularPolicy& policy);   
};
 
template<typename EnvType>
void
MaxTabularPolicyBuilder::build_from_state_function(const EnvType& env,
                                                   const DynVec<real_t>& v,
                                                   real_t gamma, 
                                                   MaxTabularPolicy& policy){
                                                                
    static_assert(std::is_integral_v<typename EnvType::state_type>, 
                  "state type must be integral");
    static_assert(std::is_integral_v<typename EnvType::action_type>, 
                  "action type must be integral");
                                
 
    typedef typename EnvType::action_type action_type;
    policy.state_action_map_.clear();
    policy.state_action_map_.resize(env.n_states());
 
    for(uint_t s=0; s<env.n_states(); ++s){
    
        auto state_vals = cuberl::rl::algos::state_actions_from_v(env, v, 
                                                                  gamma, s);
                                                                  
        action_type action = policy.get_action(state_vals);
        policy.state_action_map_[s] = action;
    }
    
}
 
 
  
 
 
 
}
}
}
 
#endif // MAX_TABULAR_POLICY_H