Reinforcement Learning Toolbox 2.0 last updated:

General Documentation Manual Tutorial Class Reference Master Thesis Examples Related Papers Downloads Links News mailto:webmaster Main Page     Class Hierarchy   Compound List   File List   Compound Members   File Members ctdlearner.h Go to the documentation of this file. // Copyright (C) 2003 // Gerhard Neumann (gneumann@gmx.net) // Stephan Neumann (sneumann@gmx.net) // // This file is part of RL Toolbox. // http://www.igi.tugraz.at/ril_toolbox // // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // 3. The name of the author may not be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef CABSTRACTRILEARNER_H #define CABSTRACTRILEARNER_H #include "cerrorlistener.h" #include "cagentlistener.h" #include "cbaseobjects.h" class CAgentController; class CDeterministicController; class CAbstractQFunction; class CAbstractQETraces; class CActionDataSet; class CGradientQFunction; class CGradientQETraces; class CResidualFunction; class CResidualGradientFunction; class CFeatureList; class CAbstractBetaCalculator; class CFeatureQFunction; class CTDLearner : public CSemiMDPRewardListener, public CErrorSender { protected: bool externETraces; CAgentController *estimationPolicy; CAction *lastEstimatedAction; CAbstractQFunction *qfunction; CAbstractQETraces *etraces; CActionDataSet *actionDataSet; virtual void learnStep(CStateCollection *oldState, CAction *action, double reward, CStateCollection *nextState); virtual double getTemporalDifference(CStateCollection *oldState, CAction *action, double reward, CStateCollection *nextState); virtual double getResidual(double oldQ, double reward, int duration, double newQ); virtual void addETraces(CStateCollection *oldState, CStateCollection *newState, CAction *action); public: CTDLearner(CRewardFunction *rewardFunction, CAbstractQFunction *qfunction, CAbstractQETraces *etraces, CAgentController *estimationPolicy); CTDLearner(CRewardFunction *rewardFunction, CAbstractQFunction *qfunction, CAgentController *estimationPolicy); virtual ~CTDLearner(); virtual void loadValues(char *filename); virtual void saveValues(char *filename); virtual void loadValues(FILE *stream); virtual void saveValues(FILE *stream); virtual void nextStep(CStateCollection *oldState, CAction *action, double reward, CStateCollection *nextState); virtual void intermediateStep(CStateCollection *oldState, CAction *action, double reward, CStateCollection *nextState); virtual void newEpisode(); // void setGamma(double gamma); void setAlpha(double alpha); void setLambda(double lambda); CAgentController* getEstimationPolicy(); void setEstimationPolicy(CAgentController * estimationPolicy); CAbstractQFunction* getQFunction(); CAbstractQETraces *getETraces(); }; class CQLearner : public CTDLearner { public: CQLearner(CRewardFunction *rewardFunction, CAbstractQFunction *qfunction); ~CQLearner(); }; class CSarsaLearner : public CTDLearner { public: CSarsaLearner(CRewardFunction *rewardFunction, CAbstractQFunction *qfunction, CDeterministicController *agent); ~CSarsaLearner(); }; class CTDGradientLearner : public CTDLearner { protected: CResidualFunction *residual; CResidualGradientFunction *residualGradient; CGradientQFunction *gradientQFunction; CGradientQETraces *gradientQETraces; CFeatureList *oldGradient; CFeatureList *newGradient; CFeatureList *residualGradientFeatures; virtual double getResidual(double oldQ, double reward, int duration, double newQ); virtual void addETraces(CStateCollection *oldState, CStateCollection *newState, CAction *action); public: CTDGradientLearner(CRewardFunction *rewardFunction, CGradientQFunction *qfunction, CAgentController *agent, CResidualFunction *residual, CResidualGradientFunction *residualGradient); ~CTDGradientLearner(); }; class CTDResidualLearner : public CTDGradientLearner { protected: CGradientQETraces *residualGradientTraces; CGradientQETraces *directGradientTraces; CGradientQETraces *residualETraces; CAbstractBetaCalculator *betaCalculator; virtual void learnStep(CStateCollection *oldState, CAction *action, double reward, CStateCollection *nextState); public: CTDResidualLearner(CRewardFunction *rewardFunction, CGradientQFunction *qfunction, CAgentController *agent, CResidualFunction *residual, CResidualGradientFunction *residualGradient, CAbstractBetaCalculator *betaCalc); ~CTDResidualLearner(); void newEpisode(); virtual void addETraces(CStateCollection *oldState, CStateCollection *newState, CAction *action, double td); CGradientQETraces *getResidualETraces() {return residualETraces;}; }; class CQAverageTDErrorLearner : public CErrorListener, public CStateObject { protected: double updateRate; CFeatureQFunction *averageErrorFunction; public: CQAverageTDErrorLearner(CFeatureQFunction *averageErrorFunction, double updateRate); virtual ~CQAverageTDErrorLearner(); virtual void onParametersChanged(); virtual void receiveError(double error, CStateCollection *state, CAction *action, CActionData *data = NULL); }; class CQAverageTDVarianceLearner : public CQAverageTDErrorLearner { public: CQAverageTDVarianceLearner(CFeatureQFunction *averageErrorFunction, double updateRate); virtual ~CQAverageTDVarianceLearner(); virtual void receiveError(double error, CStateCollection *state, CAction *action, CActionData *data = NULL); }; #endif