value_iteration.h

Go to the documentation of this file.

#ifndef VALUE_ITERATION_H
#define VALUE_ITERATION_H
 
 
#include "cuberl/base/cuberl_types.h"
#include "cuberl/rl/algorithms/dp/dp_algo_base.h"
#include "cuberl/rl/algorithms/dp/policy_improvement.h"
#include "cuberl/rl/algorithms/utils.h"
#include "cuberl/rl/episode_info.h"
#include "cuberl/rl/policies/max_tabular_policy.h"
#include "bitrl/utils/io/csv_file_writer.h"
#include "bitrl/bitrl_consts.h"
 
#include <memory>
#include <cmath>
#include <string>
 
 
namespace cuberl{
namespace rl::algos::dp
{
 
    struct ValueIterationConfig
    {
        real_t gamma{1.0};
        real_t tolerance {bitrl::consts::TOLERANCE};
        std::string save_path{bitrl::consts::INVALID_STR};
    };
 
    template<typename EnvType>
    class ValueIteration: public DPSolverBase<EnvType>
    {
    public:
 
        typedef typename DPSolverBase<EnvType>::env_type env_type;
 
        ValueIteration(const ValueIterationConfig config);
 
        virtual void actions_before_training_begins(env_type& env)override;
 
        virtual void actions_after_training_ends(env_type& /*env*/)override;
 
        virtual void actions_before_episode_begins(env_type&, uint_t /*episode_idx*/)override{}
 
        virtual void actions_after_episode_ends(env_type&, uint_t /*episode_idx*/,
                                                const EpisodeInfo& /*einfo*/)override{}
 
        virtual EpisodeInfo on_training_episode(env_type& env, uint_t episode_idx) override;
 
        void save(const std::string& filename)const;
 
 
        cuberl::rl::policies::MaxTabularPolicy build_policy(const env_type& env)const;
 
    private:
 
        ValueIterationConfig config_;
 
        DynVec<real_t> v_;
 
    };
 
    template<typename EnvType>
    ValueIteration<EnvType>::ValueIteration(const ValueIterationConfig config)
        :
        DPSolverBase<EnvType>(),
        config_(config)
    {}
 
 
    template<typename EnvType>
    void
    ValueIteration<EnvType>::actions_before_training_begins(env_type& env){
        v_ = DynVec<real_t>::Zero(env.n_states());
    }
 
    template<typename EnvType>
    EpisodeInfo
    ValueIteration<EnvType>::on_training_episode(env_type& env,
                                                 uint_t episode_idx){
 
        // start timing the training
        auto start = std::chrono::steady_clock::now();
 
        EpisodeInfo info;
        auto delta = 0.0;
        for(uint_t s=0; s< env.n_states(); ++s){
 
            auto v = v_[s];
            auto max_val = state_actions_from_v(env, v_, config_.gamma, s).maxCoeff();
 
            v_[s] = max_val;
            delta = std::max(delta, std::fabs(v_[s] - v));
        }
 
        // inform the outer loop that
        // we converged
        if(delta < config_.tolerance){
            info.stop_training = true;
        }
 
        auto end = std::chrono::steady_clock::now();
        std::chrono::duration<real_t> elapsed_seconds = end-start;
 
        info.episode_index = episode_idx;
        info.episode_iterations = env.n_states();
        info.total_time = elapsed_seconds;
 
        // this is artificial but helps
        // to monitor convergence
        info.episode_reward = delta;
 
        return info;
    }
 
    template<typename EnvType>
    void
    ValueIteration<EnvType>::actions_after_training_ends(env_type&){
        if(config_.save_path != bitrl::consts::INVALID_STR){
            save(config_.save_path);
        }
 
    }
 
    template<typename EnvType>
    void
    ValueIteration<EnvType>::save(const std::string& filename)const{
 
        bitrl::utils::io::CSVWriter file_writer(filename, ',');
        file_writer.open();
 
        file_writer.write_column_names({"state_index", "value_function"});
 
        for(uint_t s=0; s < static_cast<uint_t>(v_.size()); ++s){
            auto row = std::make_tuple(s, v_[s]);
            file_writer.write_row(row);
        }
    }
 
    template<typename EnvType>
    cuberl::rl::policies::MaxTabularPolicy
    ValueIteration<EnvType>::build_policy(const env_type& env)const{
 
        cuberl::rl::policies::MaxTabularPolicy policy;
        cuberl::rl::policies::MaxTabularPolicyBuilder builder;
        builder.build_from_state_function(env, v_,
                                          config_.gamma,policy);
        return policy;
 
    }
 
}
}
 
#endif // VALUE_ITERATION_H