actor_critic_solver_base.h

Go to the documentation of this file.

#ifndef ACTOR_CRITIC_SOLVER_BASE_H
#define ACTOR_CRITIC_SOLVER_BASE_H
 
 
#include "cuberl/base/cubeai_config.h"
 
#ifdef USE_PYTORCH
 
#include "cuberl/base/cuberl_types.h"
#include "cuberl/utils/torch_adaptor.h"
#include "cuberl/rl/algorithms/rl_algorithm_base.h"
#include "cuberl/rl/algorithms/utils.h"
#include "cuberl/rl/episode_info.h"
#include "cuberl/rl/algorithms/pg/a2c_config.h"
#include "cuberl/rl/algorithms/pg/a2c_monitor.h"
#include "cuberl/data_structs/experience_buffer.h"
 
 
#include <torch/torch.h>
 
 
#ifdef CUBERL_DEBUG
#include <cassert>
#include <boost/log/trivial.hpp>
#endif
 
#include <string>
#include <chrono>
#include <map>
#include <any>
#include <memory>
#include <tuple>
#include <string>
#include <exception>
#include <iostream>
 
namespace cuberl{
namespace rl{
namespace algos {
namespace pg {
 
template<typename EnvType, typename PolicyType, 
         typename CriticType, typename MonitorType, 
         typename ConfigType>
class ACSolverBase:  public RLSolverBase<EnvType>
{
public:
 
    typedef EnvType env_type;
 
    typedef PolicyType policy_type;
 
    typedef CriticType critic_type;
    
    typedef typename env_type::state_type state_type;
    typedef typename env_type::action_type action_type;
    
    typedef MonitorType monitor_type;
 
    typedef typename monitor_type::experience_buffer_type experience_buffer_type;
    typedef typename monitor_type::experience_tuple_type experience_tuple_type;
    
    typedef ConfigType config_type;
    
    virtual ~ACSolverBase()=default;
    
    virtual void actions_after_training_ends(env_type&){}
 
    virtual void actions_before_episode_begins(env_type&, 
                                               uint_t /*episode_idx*/){}
 
    virtual void actions_after_episode_ends(env_type&, 
                                            uint_t /*episode_idx*/, 
                                            const EpisodeInfo&){}
                                            
    virtual void actions_before_training_begins(env_type&);
 
    void set_train_mode()noexcept;
 
    void set_evaluation_mode()noexcept;
    
    monitor_type& get_monitor(){return monitor_;}
    
    
protected:
 
    ACSolverBase(const config_type& config,
                 policy_type& policy, critic_type& critic,
                 std::unique_ptr<torch::optim::Optimizer>& policy_optimizer,
                 std::unique_ptr<torch::optim::Optimizer>& critic_optimizer);
 
    config_type config_;
 
    policy_type& policy_;
 
    critic_type& critic_;
    
    monitor_type monitor_;
 
    std::unique_ptr<torch::optim::Optimizer> policy_optimizer_;
 
    std::unique_ptr<torch::optim::Optimizer> critic_optimizer_;
 
 
    uint_t
    create_episode_batch_(env_type& env, uint_t /*episode_idx*/, experience_buffer_type& buffer);
 
};
 
template<typename EnvType, typename PolicyType, 
         typename CriticType, typename MonitorType, 
         typename ConfigType>
ACSolverBase<EnvType, PolicyType, CriticType, 
             MonitorType, ConfigType>::ACSolverBase(const config_type& config,
                                                      policy_type& policy, critic_type& critic,
                                                      std::unique_ptr<torch::optim::Optimizer>& policy_optimizer,
                                                      std::unique_ptr<torch::optim::Optimizer>& critic_optimizer)
    :
    RLSolverBase<EnvType>(),
    config_(config),
    policy_(policy),
    critic_(critic),
    monitor_(),
    policy_optimizer_(std::move(policy_optimizer)),
    critic_optimizer_(std::move(critic_optimizer))
{}
 
template<typename EnvType, typename PolicyType, 
         typename CriticType, typename MonitorType, 
         typename ConfigType>
void
ACSolverBase<EnvType, PolicyType, CriticType, 
             MonitorType, ConfigType>::set_train_mode()noexcept{
    policy_ -> train();
    critic_ -> train();
 
}
 
template<typename EnvType, typename PolicyType, 
         typename CriticType, typename MonitorType, 
         typename ConfigType>
void
ACSolverBase<EnvType, PolicyType, CriticType, 
             MonitorType, ConfigType>::set_evaluation_mode()noexcept{
    policy_ -> eval();
    critic_ -> eval();
 
}
 
template<typename EnvType, typename PolicyType, 
         typename CriticType, typename MonitorType, 
         typename ConfigType>
void
ACSolverBase<EnvType, PolicyType, CriticType, 
             MonitorType, ConfigType>::actions_before_training_begins(env_type& /*env*/){
    
    monitor_.reset();
    monitor_.policy_loss_values.reserve(config_.n_episodes);
    monitor_.critic_loss_values.reserve(config_.n_episodes);
    monitor_.rewards.reserve(config_.n_episodes);
    monitor_.episode_duration.reserve(config_.n_episodes);
    set_train_mode();
}
 
template<typename EnvType, typename PolicyType,
         typename CriticType, typename MonitorType,
         typename ConfigType>
uint_t
ACSolverBase<EnvType, PolicyType, CriticType,
                 MonitorType, ConfigType>::create_episode_batch_(env_type& env, uint_t episode_idx, experience_buffer_type& buffer)
{
 
#ifdef CUBERL_DEBUG
BOOST_LOG_TRIVIAL(info)<<"Collecting batch for episode: "<<episode_idx;
#endif
 
 
    typedef typename MonitorType::experience_tuple_type experience_tuple_type;
 
    // reset the environment
    //  for every episode reset the environment
    auto old_timestep = env.reset();
 
    // loop over the iterations
    uint_t itrs = 0;
    for(; itrs < config_.max_itrs_per_episode; ++itrs){
 
        auto [action, log_prob] = policy_ -> act(old_timestep.observation());
        auto values = critic_ -> evaluate(old_timestep.observation());
        
        // step into the environment
        auto next_time_step = env.step(action);
        auto next_state = next_time_step.observation();
        auto reward = next_time_step.reward();
 
        experience_tuple_type exp = {old_timestep.observation(),
            action,
            reward,
            next_time_step.done(),
            log_prob,
            values};
 
        // put the observation into the buffer
        buffer.append(exp);
 
        // if the step is done then break
        if (next_time_step.done()){
            break;
        }
 
        old_timestep = next_time_step;
 
    }
    
#ifdef CUBERL_DEBUG
BOOST_LOG_TRIVIAL(info)<<"Done... ";
#endif
 
 
    return itrs + 1;
}
 
}
}
}
}
 
#endif // USE_PYTORCH
#endif