sarsa.h

Go to the documentation of this file.

#ifndef SARSA_H
#define SARSA_H
 
#include "cuberl/rl/algorithms/td/td_algo_base.h"
#include "cuberl/rl/worlds/envs_concepts.h"
#include "cuberl/rl/episode_info.h"
#include "cuberl/rl/policies/max_tabular_policy.h"
#include "cuberl/maths/matrix_utilities.h"
 
#include "bitrl/utils/io/csv_file_writer.h"
#include "bitrl/bitrl_consts.h"
 
 
#ifdef CUBERL_DEBUG
#include <cassert>
#endif
 
#include <chrono>
#include <iostream>
#include <string>
 
 
namespace cuberl{
namespace rl::algos::td
{
 
 
    struct SarsaConfig
    {
        uint_t n_episodes;
        real_t tolerance;
        real_t gamma;
        real_t eta;
        uint_t max_num_iterations_per_episode;
        std::string path{bitrl::consts::INVALID_STR};
    };
 
    template<envs::discrete_world_concept EnvType, typename PolicyType>
    class SarsaSolver final: public TDAlgoBase<EnvType>
    {
    public:
 
        typedef typename TDAlgoBase<EnvType>::env_type env_type;
 
        typedef typename TDAlgoBase<EnvType>::action_type action_type;
 
        typedef typename TDAlgoBase<EnvType>::state_type state_type;
 
        typedef PolicyType policy_type;
 
        SarsaSolver(SarsaConfig config, const PolicyType& selector);
 
        virtual void actions_before_training_begins(env_type&);
 
        virtual void actions_after_training_ends(env_type&);
 
        virtual void actions_before_episode_begins(env_type&, uint_t /*episode_idx*/){}
 
        virtual void actions_after_episode_ends(env_type&, uint_t /*episode_idx*/,
                                                const EpisodeInfo& /*einfo*/){}
 
        virtual EpisodeInfo on_training_episode(env_type&, uint_t episode_idx);
 
        void save(const std::string& filename)const;
 
        cuberl::rl::policies::MaxTabularPolicy build_policy()const;
 
    private:
 
        SarsaConfig config_;
 
        policy_type policy_;
 
        DynMat<real_t> q_table_;
 
        void update_q_table_(const action_type& action,
                             const state_type& cstate,
                             const state_type& next_state,
                             const  action_type& next_action, real_t reward);
    };
 
 
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    SarsaSolver<EnvTp, PolicyType>::SarsaSolver(SarsaConfig config,
                                                const PolicyType& selector)
        :
        TDAlgoBase<EnvTp>(),
        config_(config),
        policy_(selector)
    {}
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    void
    SarsaSolver<EnvTp, PolicyType>::actions_before_training_begins(env_type& env){
        q_table_ = DynMat<real_t>(env.n_states(), env.n_actions());
 
        for(uint_t i=0; i < env.n_states(); ++i)
            for(uint_t j=0; j < env.n_actions(); ++j)
                q_table_(i, j) = 0.0;
 
    }
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    void
    SarsaSolver<EnvTp, PolicyType>::actions_after_training_ends(env_type&){
 
        if(config_.path != bitrl::consts::INVALID_STR){
            save(config_.path);
        }
    }
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    EpisodeInfo
    SarsaSolver<EnvTp, PolicyType>::on_training_episode(env_type& env,
                                                        uint_t episode_idx){
 
        auto start = std::chrono::steady_clock::now();
        EpisodeInfo info;
 
        // total score for the episode
        auto episode_score = 0.0;
        auto time_step = env.reset();
        auto state = time_step.observation();
 
        uint_t itr=0;
        for(;  itr < config_.max_num_iterations_per_episode; ++itr){
 
            // select an action
            auto action = policy_(q_table_, state);
 
            // Take a on_episode
            auto step_type_result = env.step(action);
 
            auto next_state = step_type_result.observation();
            auto reward = step_type_result.reward();
            auto done = step_type_result.done();
 
            // accumulate score
            episode_score += reward;
 
            if(!done){
 
                // use the policy to select the next action
                auto next_action = policy_(q_table_, state);
                update_q_table_(action, state, next_state, next_action, reward);
                state = next_state;
                action = next_action;
            }
            else{
 
                update_q_table_(action, state,
                                bitrl::consts::INVALID_ID,
                                bitrl::consts::INVALID_ID,
                                reward);
 
                break;
            }
        }
 
        auto end = std::chrono::steady_clock::now();
        std::chrono::duration<real_t> elapsed_seconds = end-start;
 
        info.episode_index = episode_idx;
        info.episode_reward = episode_score;
        info.episode_iterations = itr;
        info.total_time = elapsed_seconds;
        return info;
    }
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    void
    SarsaSolver<EnvTp, PolicyType>::save(const std::string& filename)const{
 
        bitrl::utils::io::CSVWriter file_writer(filename, ',');
        file_writer.open();
 
        std::vector<std::string> col_names(1 + q_table_.cols());
        col_names[0] = "state_index";
 
        for(uint_t i = 0; i< static_cast<uint_t>(q_table_.cols()); ++i){
            col_names[i + 1] = "action_" + std::to_string(i);
        }
 
        file_writer.write_column_names(col_names);
        for(uint_t s=0; s < static_cast<uint_t>(q_table_.rows()); ++s){
            auto actions = maths::get_row(q_table_, s);
            auto row = std::make_tuple(s, actions);
            file_writer.write_row(row);
        }
 
    }
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    void
    SarsaSolver<EnvTp, PolicyType>::update_q_table_(const action_type& action,
                                                    const state_type& cstate,
                                                    const state_type& next_state,
                                                    const action_type& next_action, real_t reward){
 
        auto q_current = q_table_(cstate, action);
 
        // with the SARSA solver we query the 
        // q-function about its value at next state when taking next action
        // in Q-learning we form a maximum instead
        auto q_next = next_state != bitrl::consts::INVALID_ID ? q_table_(next_state, next_action) : 0.0;
        auto td_target = reward + config_.gamma * q_next;
        q_table_(cstate, action) = q_current + (config_.eta * (td_target - q_current));
 
    }
 
    template<envs::discrete_world_concept EnvTp, typename PolicyType>
    cuberl::rl::policies::MaxTabularPolicy
    SarsaSolver<EnvTp, PolicyType>::build_policy()const{
 
        cuberl::rl::policies::MaxTabularPolicy policy;
        cuberl::rl::policies::MaxTabularPolicyBuilder builder;
        builder.build_from_state_action_function(q_table_,policy);
        return policy;
 
    }
 
 
}
}
 
#endif // SARSA_H