Reinforcement Learning Toolbox 2.0 last updated:

General Documentation Manual Tutorial Class Reference Master Thesis Examples Related Papers Downloads Links News mailto:webmaster Main Page     Class Hierarchy   Compound List   File List   Compound Members   File Members cqfunction.h Go to the documentation of this file. // Copyright (C) 2003 // Gerhard Neumann (gneumann@gmx.net) // Stephan Neumann (sneumann@gmx.net) // // This file is part of RL Toolbox. // http://www.igi.tugraz.at/ril_toolbox // // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // 3. The name of the author may not be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef CRIABSTRACTQFUNCTION_H #define CRIABSTRACTQFUNCTION_H #include <stdio.h> #include "clearndataobject.h" #include "cbaseobjects.h" #include "cmyexception.h" #include "cgradientfunction.h" class CAbstractFeatureStochasticModel; class CAbstractQETraces; class CGradientQETraces; class CAbstractVFunction; class CFeatureVFunction; class CFeatureRewardFunction; class CActionStatistics; class CFeature; #define GRADIENTQFUNCTION 1 #define CONTINUOUSACTIONQFUNCTION 2 class CAbstractQFunction : public CActionObject, virtual public CLearnDataObject { protected: int type; public: bool mayDiverge; int getType(); bool isType(int type); void addType(int Type); CAbstractQFunction(CActionSet *actions); virtual ~CAbstractQFunction(); virtual void saveData(FILE *file); virtual void loadData(FILE *file); virtual void printValues (){}; virtual void resetData() {}; void getActionValues(CStateCollection *state, CActionSet *actions, double *actionValues, CActionDataSet *data = NULL); virtual CAction* getMax(CStateCollection *state, CActionSet *availableActions, CActionDataSet *data = NULL); virtual double getMaxValue(CStateCollection *state, CActionSet *availableActions); virtual void getStatistics(CStateCollection *state, CAction *action, CActionSet *actions, CActionStatistics* statistics); virtual void updateValue(CStateCollection *, CAction *, double , CActionData * = NULL) {}; virtual void setValue(CStateCollection *state, CAction *action, double qValue, CActionData *data = NULL); virtual double getValue(CStateCollection *state, CAction *action, CActionData *data = NULL) = 0; virtual CAbstractQETraces *getStandardETraces() {return NULL;}; protected: }; class CQFunctionSum : public CAbstractQFunction { protected: std::map<CAbstractQFunction *, double> *qFunctions; public: CQFunctionSum(CActionSet *actions); virtual ~CQFunctionSum(); virtual double getValue(CStateCollection *state, CAction *action, CActionData *data = NULL); virtual CAbstractQETraces *getStandardETraces() {return NULL;}; double getQFunctionFactor(CAbstractQFunction *qFunction); void setQFunctionFactor(CAbstractQFunction *qFunction, double factor); void addQFunction(CAbstractQFunction *qFunction, double factor); void removeQFunction(CAbstractQFunction *qFunction); void normFactors(double factor); }; class CDivergentQFunctionException : public CMyException { protected: virtual string getInnerErrorMsg(); public: string qFunctionName; CAbstractQFunction *qFunction; CState *state; double value; CDivergentQFunctionException(string qFunctionName, CAbstractQFunction *qFunction, CState *state, double value); virtual ~CDivergentQFunctionException(){}; }; class CGradientQFunction : public CAbstractQFunction, virtual public CGradientUpdateFunction { protected: CFeatureList *localGradientQFunctionFeatures; public: CGradientQFunction(CActionSet *actions); virtual ~CGradientQFunction(); virtual int getWeightsOffset(CAction *) {return 0;}; virtual void getGradient(CStateCollection *state, CAction *action, CActionData *data, CFeatureList *gradient) = 0; virtual void updateValue(CStateCollection *state, CAction *action, double td, CActionData *data = NULL); virtual void resetData() {CAbstractQFunction::resetData();}; virtual void loadData(FILE *stream) {CGradientUpdateFunction::loadData(stream);}; virtual void saveData(FILE *stream) {CGradientUpdateFunction::saveData(stream);}; virtual CAbstractQETraces *getStandardETraces(); virtual void copy(CLearnDataObject *qFunction) {CGradientUpdateFunction::copy(qFunction);}; }; /* class CGradientDelayedUpdateQFunction : public CGradientQFunction, public CGradientDelayedUpdateFunction { protected: virtual void updateWeights(CFeatureList *dParams) {CGradientDelayedUpdateFunction::updateWeights(dParams);}; CGradientQFunction *qFunction; public: CGradientDelayedUpdateQFunction(CGradientQFunction *qFunction); virtual ~CGradientDelayedUpdateQFunction() {}; virtual double getValue(CStateCollection *state, CAction *action, CActionData *data = NULL); virtual void getGradient(CStateCollection *state, CAction *action, CActionData *data, CFeatureList *gradientFeatures); virtual void resetData() {CGradientDelayedUpdateFunction::resetData();}; virtual int getNumWeights(){return CGradientDelayedUpdateFunction::getNumWeights();}; virtual void getWeights(double *parameters) {CGradientDelayedUpdateFunction::getWeights(parameters);}; virtual void setWeights(double *parameters) {CGradientDelayedUpdateFunction::setWeights(parameters);}; virtual void loadData(FILE *stream) {CGradientQFunction::loadData(stream);}; virtual void saveData(FILE *stream) {CGradientQFunction::saveData(stream);}; }; */ class CQFunction : public CGradientQFunction { protected: std::map<CAction *, CAbstractVFunction *> *vFunctions; virtual int getWeightsOffset(CAction *action); virtual void updateWeights(CFeatureList *features); public: CQFunction(CActionSet *actions); virtual ~CQFunction(); virtual void updateValue(CStateCollection *state, CAction *action, double td, CActionData *data = NULL); virtual void setValue(CStateCollection *state, CAction *action, double qValue, CActionData *data = NULL); virtual double getValue(CStateCollection *state, CAction *action, CActionData *data = NULL); virtual void updateValue(CState *state, CAction *action, double td, CActionData *data = NULL); virtual void setValue(CState *state, CAction *action, double qValue, CActionData *data = NULL); virtual double getValue(CState *state, CAction *action, CActionData *data = NULL); virtual void saveData(FILE *file); virtual void loadData(FILE *file); virtual void printValues(); CAbstractVFunction *getVFunction(CAction *action); CAbstractVFunction *getVFunction(int index); void setVFunction(CAction *action, CAbstractVFunction *vfunction, bool bDeleteOld = true); void setVFunction(int index, CAbstractVFunction *vfunction, bool bDeleteOld = true); int getNumVFunctions(); virtual CAbstractQETraces *getStandardETraces(); //virtual CStateProperties *getGradientCalculator(CAction *action); virtual void getGradient(CStateCollection *state, CAction *action, CActionData *data, CFeatureList *gradient); virtual int getNumWeights(); virtual void getWeights(double *weights); virtual void setWeights(double *weights); virtual void resetData(); virtual void copy(CLearnDataObject *qFunction); }; class CQFunctionFromStochasticModel : public CAbstractQFunction, public CStateObject { protected: CFeatureVFunction *vfunction; CAbstractFeatureStochasticModel *model; CStateProperties *discretizer; CFeatureRewardFunction *rewardfunction; CState *discState; public: CQFunctionFromStochasticModel(CFeatureVFunction *vfunction, CAbstractFeatureStochasticModel *model, CFeatureRewardFunction *rewardfunction); virtual ~CQFunctionFromStochasticModel(); // Writes the Action-Values in the actionValues Array. // void getActionValues(CStateCollection *state, double *actionValues, CActionSet *actions); virtual void updateValue(CStateCollection *, CAction *, double , CActionData * = NULL) {}; virtual void setValue(CStateCollection *, CAction *, double , CActionData * = NULL) {}; virtual double getValue(CStateCollection *state, CAction *action, CActionData *data = NULL); virtual double getValue(CState *featState, CAction *action, CActionData *data = NULL); virtual double getValue(int feature, CAction *action, CActionData *data = NULL); virtual CAbstractQETraces *getStandardETraces() {return NULL;}; }; class CFeatureQFunction : public CQFunction { protected: CStateProperties *discretizer; unsigned int features; std::list<CFeatureVFunction *> *featureVFunctions; virtual void init(); void initVFunctions(CFeatureVFunction *vfunction, CAbstractFeatureStochasticModel *model, CFeatureRewardFunction *rewardFunction, double gamma); public: CFeatureQFunction(CActionSet *actions, CStateProperties *discretizer); CFeatureQFunction(CFeatureVFunction *vfunction, CAbstractFeatureStochasticModel *model, CFeatureRewardFunction *rewardFunction,double gamma); virtual ~CFeatureQFunction(); void updateValue(CFeature *state, CAction *action, double td, CActionData *data = NULL); void setValue(int state, CAction *action, double qValue, CActionData *data = NULL); double getValue(int feature, CAction *action, CActionData *data = NULL); void setFeatureCalculator(CStateModifier *discretizer); CStateProperties *getFeatureCalculator(); int getNumFeatures(); void saveFeatureActionValueTable(FILE *stream); void saveFeatureActionTable(FILE *stream); }; class CComposedQFunction : public CGradientQFunction { protected: std::list<CAbstractQFunction *> *qFunctions; virtual int getWeightsOffset(CAction *action); virtual void updateWeights(CFeatureList *features); public: CComposedQFunction(); virtual ~CComposedQFunction(); virtual void saveData(FILE *file); virtual void loadData(FILE *file); virtual void printValues(); virtual void getStatistics(CStateCollection *state, CAction *action, CActionSet *actions, CActionStatistics* statistics); virtual void updateValue(CStateCollection *state, CAction *action, double td, CActionData *data = NULL); virtual void setValue(CStateCollection *state, CAction *action, double qValue, CActionData *data = NULL); virtual double getValue(CStateCollection *state, CAction *action, CActionData *data = NULL); void addQFunction(CAbstractQFunction *qFunction); std::list<CAbstractQFunction *> *getQFunctions(); int getNumQFunctions(); virtual CAbstractQETraces *getStandardETraces(); //virtual CStateProperties *getGradientCalculator(CAction *action); virtual void getGradient(CStateCollection *state, CAction *action, CActionData *data, CFeatureList *gradient); virtual int getNumWeights(); virtual void getWeights(double *weights); virtual void setWeights(double *weights); virtual void resetData(); }; /* class CQTable : public CFeatureQFunction { CAbstractStateDiscretizer *discretizer; virtual void init(int states); public: CQTable(CActionSet *actions, CAbstractStateDiscretizer *state); ~CQTable(); void setDiscretizer(CAbstractStateDiscretizer *discretizer); CAbstractStateDiscretizer *getDiscretizer(); int getNumStates(); };*/ #endif