policy_improvement.h

Go to the documentation of this file.

#ifndef POLICY_IMPROVEMENT_H
#define POLICY_IMPROVEMENT_H
 
#include "cuberl/rl/algorithms/dp/dp_algo_base.h"
#include "cuberl/rl/algorithms/utils.h"
#include "cuberl/rl/policies/adaptors/policy_stochastic_adaptor.h"
 
#include <any>
#include <map>
#include <string>
 
namespace cuberl{
namespace rl::algos::dp
{
 
    template<typename EnvType, typename PolicyType>
    class PolicyImprovement: public DPSolverBase<EnvType>
    {
    public:
 
        typedef typename DPSolverBase<EnvType>::env_type env_type;
 
        typedef PolicyType policy_type;
 
        PolicyImprovement(uint_t action_space_size,
                          real_t gamma,
                          const DynVec<real_t>& val_func,
                          policy_type& policy);
 
        virtual void actions_before_training_begins(env_type& /*env*/)override{}
 
        virtual void actions_after_training_ends(env_type& /*env*/)override{}
 
        virtual void actions_before_episode_begins(env_type&, uint_t /*episode_idx*/)override{}
 
        virtual void actions_after_episode_ends(env_type&, uint_t /*episode_idx*/,
                                                const EpisodeInfo& /*einfo*/)override{}
 
        virtual EpisodeInfo on_training_episode(env_type& env, uint_t episode_idx) override;
 
        const policy_type& policy()const{return  policy_;}
 
        policy_type& policy(){return  policy_;}
 
        void set_value_function(const DynVec<real_t>& v){v_ = v;}
 
    protected:
 
        real_t gamma_;
 
        DynVec<real_t> v_;
 
        policy_type& policy_;
 
        cuberl::rl::policies::StochasticAdaptorPolicy<policy_type> policy_adaptor_;
    };
 
    template<typename EnvType, typename PolicyType>
    PolicyImprovement<EnvType, PolicyType>::PolicyImprovement(uint_t action_space_size,
                                                              real_t gamma, const DynVec<real_t>& val_func,
                                                              policy_type& policy)
        :
        DPSolverBase<EnvType>(),
        gamma_(gamma),
        v_(val_func),
        policy_(policy),
        policy_adaptor_(val_func.size(), action_space_size, policy)
    {}
 
    template<typename EnvType, typename PolicyType>
    EpisodeInfo
    PolicyImprovement<EnvType, PolicyType>::on_training_episode(env_type& env, uint_t episode_idx){
 
        auto start = std::chrono::steady_clock::now();
 
        std::map<std::string, std::any> options;
 
        for(uint_t s=0; s<env.n_states(); ++s){
 
            auto state_actions = state_actions_from_v(env, v_, gamma_, s);
 
            options.insert_or_assign("state", s);
            options.insert_or_assign("state_actions", std::any(state_actions));
            policy_ = policy_adaptor_(options);
        }
 
        auto end = std::chrono::steady_clock::now();
        std::chrono::duration<real_t> elapsed_seconds = end-start;
 
        EpisodeInfo info;
        info.episode_index = episode_idx;
        info.episode_iterations = env.n_states();
        info.total_time = elapsed_seconds;
        return info;
    }
 
 
}
}
 
#endif // POLICY_IMPROVEMENT_H