q_learning.h

Go to the documentation of this file.

#ifndef Q_LEARNING_H
#define Q_LEARNING_H
 
#include "cuberl/base/cuberl_types.h"
#include "cuberl/rl/episode_info.h"
#include "cuberl/rl/algorithms/td/td_algo_base.h"
#include "cuberl/rl/worlds/envs_concepts.h"
#include "cuberl/rl/policies/max_tabular_policy.h"
#include "cuberl/maths/matrix_utilities.h"
 
#include "bitrl/utils/io/csv_file_writer.h"
#include "bitrl/bitrl_consts.h"
 
#ifdef CUBERL_DEBUG
#include <cassert>
#endif
 
#include <chrono>
 
namespace cuberl {
namespace rl::algos::td
{
 
    struct QLearningConfig
    {
        bool average_episode_reward{true};
        uint_t n_episodes;
        uint_t max_num_iterations_per_episode;
        real_t tolerance;
        real_t gamma;
        real_t eta;
        std::string path{bitrl::consts::INVALID_STR};
 
    };
 
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    class QLearningSolver final: public TDAlgoBase<EnvTp>
    {
 
    public:
 
        typedef typename TDAlgoBase<EnvTp>::env_type env_type;
 
        typedef typename TDAlgoBase<EnvTp>::action_type action_type;
 
        typedef typename TDAlgoBase<EnvTp>::state_type state_type;
 
        typedef PolicyType policy_type;
 
        QLearningSolver(const QLearningConfig config, const PolicyType& policy);
 
        virtual void actions_before_training_begins(env_type&);
 
        virtual void actions_after_training_ends(env_type&);
 
        virtual void actions_before_episode_begins(env_type&, uint_t /*episode_idx*/){}
 
        virtual void actions_after_episode_ends(env_type&, uint_t episode_idx,
                                                const EpisodeInfo& /*einfo*/);
 
        virtual EpisodeInfo on_training_episode(env_type&, uint_t episode_idx);
 
        void save(const std::string& filename)const;
 
        cuberl::rl::policies::MaxTabularPolicy build_policy()const;
 
    private:
 
        QLearningConfig config_;
 
        policy_type policy_;
 
        DynMat<real_t> q_table_;
 
        void update_q_table_(const action_type& action, const state_type& cstate,
                             const state_type& next_state, const  action_type& next_action,
                             real_t reward);
 
    };
 
    template <envs::discrete_world_concept EnvTp, typename PolicyType>
    QLearningSolver<EnvTp, PolicyType>::QLearningSolver(const QLearningConfig config,
                                                        const PolicyType& policy)
        :
        TDAlgoBase<EnvTp>(),
        config_(config),
        policy_(policy),
        q_table_()
    {}
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    void
    QLearningSolver<EnvTp, PolicyType>::actions_before_training_begins(env_type& env){
        q_table_ = DynMat<real_t>(env.n_states(), env.n_actions());
 
        for(uint_t i=0; i < env.n_states(); ++i)
            for(uint_t j=0; j < env.n_actions(); ++j)
                q_table_(i, j) = 0.0;
 
    }
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    void
    QLearningSolver<EnvTp, PolicyType>::actions_after_training_ends(env_type&){
 
        if(config_.path != bitrl::consts::INVALID_STR){
            save(config_.path);
        }
    }
 
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    EpisodeInfo
    QLearningSolver<EnvTp, PolicyType>::on_training_episode(env_type& env, uint_t episode_idx){
 
        auto start = std::chrono::steady_clock::now();
        EpisodeInfo info;
 
        // total score for the episode
        auto episode_score = 0.0;
        auto state = env.reset().observation();
 
        uint_t itr=0;
        for(;  itr < config_.max_num_iterations_per_episode; ++itr){
 
 
            // select an action
            auto action = policy_(q_table_, state);
 
            // Take a on_episode
            auto step_type_result = env.step(action);
 
            auto next_state = step_type_result.observation();
            auto reward = step_type_result.reward();
            auto done = step_type_result.done();
 
            // accumulate score
            episode_score += reward;
 
            if(!done){
                auto next_action = policy_(q_table_, state);
                update_q_table_(action, state, next_state, next_action, reward);
                state = next_state;
                action = next_action;
            }
            else{
 
                update_q_table_(action, state,
                                bitrl::consts::INVALID_ID,
                                bitrl::consts::INVALID_ID,
                                reward);
 
 
                break;
            }
        }
 
        auto end = std::chrono::steady_clock::now();
        std::chrono::duration<real_t> elapsed_seconds = end-start;
 
        info.episode_index = episode_idx;
        info.episode_reward = config_.average_episode_reward ? episode_score / static_cast<real_t>(itr) : episode_score;
        info.episode_iterations = itr;
        info.total_time = elapsed_seconds;
        return info;
    }
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    void
    QLearningSolver<EnvTp, PolicyType>::actions_after_episode_ends(env_type&, uint_t episode_idx,
                                                                   const EpisodeInfo& /*einfo*/){
        policy_.on_episode(episode_idx);
    }
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    void
    QLearningSolver<EnvTp, PolicyType>::save(const std::string& filename)const{
 
        bitrl::utils::io::CSVWriter file_writer(filename, ',');
        file_writer.open();
 
        std::vector<std::string> col_names(1 + q_table_.cols());
        col_names[0] = "state_index";
 
        for(uint_t i = 0; i< static_cast<uint_t>(q_table_.cols()); ++i){
            col_names[i + 1] = "action_" + std::to_string(i);
        }
 
        file_writer.write_column_names(col_names);
 
        for(uint_t s=0; s < static_cast<uint_t>(q_table_.rows()); ++s){
            auto actions = maths::get_row(q_table_, s);
            auto row = std::make_tuple(s, actions);
            file_writer.write_row(row);
        }
    }
 
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    cuberl::rl::policies::MaxTabularPolicy
    QLearningSolver<EnvTp, PolicyType>::build_policy()const{
 
        cuberl::rl::policies::MaxTabularPolicy policy;
        cuberl::rl::policies::MaxTabularPolicyBuilder builder;
        builder.build_from_state_action_function(q_table_,policy);
        return policy;
 
    }
 
    template <envs::discrete_world_concept EnvTp, typename PolicyType>
    void
    QLearningSolver<EnvTp, PolicyType>::update_q_table_(const action_type& action, const state_type& cstate,
                                                        const state_type& next_state,
                                                        const  action_type& /*next_action*/, real_t reward){
 
        auto q_current = q_table_(cstate, action);
        auto q_next = next_state != bitrl::consts::INVALID_ID ? cuberl::maths::get_row_max(q_table_, next_state) : 0.0;
 
 
        auto td_target = reward + config_.gamma * q_next;
        q_table_(cstate, action) = q_current + (config_.eta * (td_target - q_current));
 
    }
 
 
 
}
}
 
#endif // Q_LEARNING_H