rl_serial_agent_trainer.h

Go to the documentation of this file.

#ifndef RL_SERIAL_AGENT_TRAINER_H
#define RL_SERIAL_AGENT_TRAINER_H
 
#include "cuberl/base/cuberl_types.h"
#include "bitrl/bitrl_consts.h"
 
#include "bitrl/utils/iterative_algorithm_result.h"
#include "bitrl/utils/iterative_algorithm_controller.h"
 
#include <boost/noncopyable.hpp>
#include <boost/log/trivial.hpp>
#include <vector>
#include <chrono>
//#include <iostream>
 
namespace cuberl {
namespace rl {
 
// forward declare
struct EpisodeInfo;
 
struct RLSerialTrainerConfig
{
    uint_t output_msg_frequency{bitrl::consts::INVALID_ID};
    uint_t n_episodes{0};
    real_t tolerance{bitrl::consts::TOLERANCE};
};
 
 
template<typename EnvType, typename AgentType>
class RLSerialAgentTrainer: private boost::noncopyable
{
public:
 
    typedef EnvType env_type;
    typedef AgentType agent_type;
 
    RLSerialAgentTrainer(const RLSerialTrainerConfig& config, agent_type& agent);
 
    virtual bitrl::utils::IterativeAlgorithmResult train(env_type& env);
 
    virtual void actions_before_training_begins(env_type&);
 
    virtual void actions_before_episode_begins(env_type&, uint_t);
 
    virtual void actions_after_episode_ends(env_type&, uint_t /*episode_idx*/, 
                                            const EpisodeInfo& einfo);
 
    virtual void actions_after_training_ends(env_type&);
 
    const std::vector<real_t>& episodes_total_rewards()const noexcept
    {return total_reward_per_episode_;}
 
    const std::vector<uint_t>& n_itrs_per_episode()const noexcept
    {return n_itrs_per_episode_;}
 
protected:
 
    uint_t output_msg_frequency_;
 
    bitrl::utils::IterativeAlgorithmController itr_ctrl_;
 
    agent_type& agent_;
 
    std::vector<real_t> total_reward_per_episode_;
 
    std::vector<uint_t> n_itrs_per_episode_;
 
};
 
template<typename EnvType, typename AgentType>
RLSerialAgentTrainer<EnvType, AgentType>::RLSerialAgentTrainer(const RLSerialTrainerConfig& config, 
                                                               agent_type& agent)
    :
    output_msg_frequency_(config.output_msg_frequency),
    itr_ctrl_(config.n_episodes, config.tolerance),
    agent_(agent),
    total_reward_per_episode_(),
    n_itrs_per_episode_()
{}
 
template<typename EnvType, typename AgentType>
void
RLSerialAgentTrainer<EnvType, AgentType>::actions_before_training_begins(env_type& env){
 
    agent_.actions_before_training_begins(env);
    total_reward_per_episode_.clear();
    n_itrs_per_episode_.clear();
 
    total_reward_per_episode_.reserve(itr_ctrl_.get_max_iterations());
    n_itrs_per_episode_.reserve(itr_ctrl_.get_max_iterations());
}
 
template<typename EnvType, typename AgentType>
void
RLSerialAgentTrainer<EnvType, AgentType>::actions_before_episode_begins(env_type& env, 
                                                                        uint_t episode_idx){
   agent_.actions_before_episode_begins(env, episode_idx);
}
 
template<typename EnvType, typename AgentType>
void
RLSerialAgentTrainer<EnvType, AgentType>::actions_after_episode_ends(env_type& env, uint_t episode_idx, 
                                                                     const EpisodeInfo& einfo){
    agent_.actions_after_episode_ends(env, episode_idx, einfo);
}
 
template<typename EnvType, typename AgentType>
void
RLSerialAgentTrainer<EnvType, AgentType>::actions_after_training_ends(env_type& env){
    agent_.actions_after_training_ends(env);
}
 
template<typename EnvType, typename AgentType>
bitrl::utils::IterativeAlgorithmResult
RLSerialAgentTrainer<EnvType, AgentType>::train(env_type& env){
 
    BOOST_LOG_TRIVIAL(info)<<" Start training on environment..."; //<<env.name;
    
    // start timing the training
    auto start = std::chrono::steady_clock::now();
 
    this->actions_before_training_begins(env);
 
    uint_t episode_counter = 0;
    bool stop_training = false;
    while(itr_ctrl_.continue_iterations()){
 
        this->actions_before_episode_begins(env, episode_counter);
        auto episode_info = agent_.on_training_episode(env, episode_counter);
 
        if(output_msg_frequency_ != bitrl::consts::INVALID_ID &&
                episode_counter % output_msg_frequency_  == 0){
 
            BOOST_LOG_TRIVIAL(info)<<episode_info;
        }
 
        total_reward_per_episode_.push_back(episode_info.episode_reward);
        n_itrs_per_episode_.push_back(episode_info.episode_iterations);
        this->actions_after_episode_ends(env, episode_counter, episode_info);
 
        if(episode_info.stop_training){
            BOOST_LOG_TRIVIAL(info)<<" Stopping training at index="<<episode_counter;
            
            // assume that if we were told to stop
            // that we have converge
            stop_training = true;
            break;
        }
        episode_counter += 1;
    }
 
    this->actions_after_training_ends(env);
    auto end = std::chrono::steady_clock::now();
    std::chrono::duration<real_t> elapsed_seconds = end-start;
    
    BOOST_LOG_TRIVIAL(info)<<" Done... ";
    
    auto state = itr_ctrl_.get_state();
    state.total_time = elapsed_seconds;
    state.converged = stop_training;
    return state;
}
 
 
}
}
 
#endif // RL_SERIAL_AGENT_TRAINER_H