Reinforcement Learning Toolbox 2.0 last updated:

General Documentation Manual Tutorial Class Reference Master Thesis Examples Related Papers Downloads Links News mailto:webmaster Main Page     Class Hierarchy   Compound List   File List   Compound Members   File Members cvfunction.h Go to the documentation of this file. // Copyright (C) 2003 // Gerhard Neumann (gneumann@gmx.net) // Stephan Neumann (sneumann@gmx.net) // // This file is part of RL Toolbox. // http://www.igi.tugraz.at/ril_toolbox // // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // 3. The name of the author may not be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef C_ABSTRACTVFUNCTION_H #define C_ABSTRACTVFUNCTION_H #include <stdio.h> #include "clearndataobject.h" #include "cmyexception.h" #include "cbaseobjects.h" #include "cgradientfunction.h" #include "cfeaturefunction.h" class CAbstractVETraces; class CFeatureQFunction; class CStochasticPolicy; class CRewardFunction; class CStateCollectionImpl; class CFeatureCalculator; class CAbstractStateDiscretizer; class CStateReward; #define DIVERGENTVFUNCTIONVALUE 1000000 #define CONTINUOUSVFUNCTION 1 #define GRADIENTVFUNCTION 2 class CAbstractVFunction : public CStateObject, virtual public CLearnDataObject { protected: int type; void addType(int newType); public: bool mayDiverge; CAbstractVFunction(CStateProperties *properties); virtual ~CAbstractVFunction(); virtual void resetData() {}; virtual void updateValue(CStateCollection *state, double td); virtual void setValue(CStateCollection *state, double qValue); virtual double getValue(CStateCollection *state); virtual void updateValue(CState *state, double td); virtual void setValue(CState *, double ) {}; virtual double getValue(CState *state) = 0; virtual void saveData(FILE *file); virtual void loadData(FILE *file); virtual void printValues (){}; int getType(); bool isType(int isT); virtual CAbstractVETraces *getStandardETraces(); }; class CZeroVFunction : public CAbstractVFunction { protected: public: CZeroVFunction(); virtual double getValue(CState *state); }; class CVFunctionSum : public CAbstractVFunction { protected: std::map<CAbstractVFunction *, double> *vFunctions; public: CVFunctionSum(); virtual ~CVFunctionSum(); virtual double getValue(CStateCollection *state); virtual double getValue(CState *state) {return getValue((CStateCollection *) state);}; virtual CAbstractVETraces *getStandardETraces() {return NULL;}; double getVFunctionFactor(CAbstractVFunction *vFunction); void setVFunctionFactor(CAbstractVFunction *vFunction, double factor); void addVFunction(CAbstractVFunction *vFunction, double factor = 1.0); void removeVFunction(CAbstractVFunction *vFunction); void normFactors(double factor); }; class CDivergentVFunctionException : public CMyException { protected: virtual string getInnerErrorMsg(); public: string vFunctionName; CAbstractVFunction *vFunction; CState *state; double value; CDivergentVFunctionException(string vFunctionName, CAbstractVFunction *vFunction, CState *state, double value); virtual ~CDivergentVFunctionException(){}; }; class CGradientVFunction : public CAbstractVFunction, virtual public CGradientUpdateFunction { protected: public: CGradientVFunction(CStateProperties *properties); virtual ~CGradientVFunction(); virtual void updateValue(CStateCollection *state, double td); virtual void updateValue(CState *state, double td); virtual void getGradient(CStateCollection *state, CFeatureList *gradientFeatures) = 0; virtual void resetData() = 0; virtual void loadData(FILE *stream) {CGradientUpdateFunction::loadData(stream);}; virtual void saveData(FILE *stream) {CGradientUpdateFunction::saveData(stream);}; virtual CAbstractVETraces *getStandardETraces(); virtual void copy(CLearnDataObject *vFunction) {CGradientUpdateFunction::copy(vFunction);}; }; /* class CGradientDelayedUpdateVFunction : public CGradientVFunction, public CGradientDelayedUpdateFunction { protected: virtual void updateWeights(CFeatureList *dParams) {CGradientDelayedUpdateFunction::updateWeights(dParams);}; CGradientVFunction *vFunction; public: CGradientDelayedUpdateVFunction(CGradientVFunction *vFunction); virtual ~CGradientDelayedUpdateVFunction() {}; virtual double getValue(CState *state); virtual void getGradient(CStateCollection *state, CFeatureList *gradientFeatures); virtual void resetData() {CGradientDelayedUpdateFunction::resetData();}; virtual int getNumWeights(){return CGradientDelayedUpdateFunction::getNumWeights();}; virtual void getWeights(double *parameters) {CGradientDelayedUpdateFunction::getWeights(parameters);}; virtual void setWeights(double *parameters) {CGradientDelayedUpdateFunction::setWeights(parameters);}; virtual void loadData(FILE *stream) {CGradientVFunction::loadData(stream);}; virtual void saveData(FILE *stream) {CGradientVFunction::saveData(stream);}; }; */ class CVFunctionInputDerivationCalculator : virtual public CParameterObject { protected: CStateProperties *modelState; public: CVFunctionInputDerivationCalculator(CStateProperties *modelState); virtual void getInputDerivation( CStateCollection *state, ColumnVector *targetVector) = 0; unsigned int getNumInputs(); }; class CVFunctionNumericInputDerivationCalculator : public CVFunctionInputDerivationCalculator { protected: CAbstractVFunction *vFunction; CStateCollectionImpl *stateBuffer; public: CVFunctionNumericInputDerivationCalculator(CStateProperties *modelState, CAbstractVFunction *vFunction, double stepSize, std::list<CStateModifier *> *modifiers); virtual ~CVFunctionNumericInputDerivationCalculator(); virtual void getInputDerivation( CStateCollection *state, ColumnVector *targetVector); }; class CFeatureVFunction : public CGradientVFunction, public CFeatureFunction { protected: public: CFeatureVFunction(int numFeatures); CFeatureVFunction(CStateProperties *featureFact); CFeatureVFunction(CFeatureQFunction *qfunction, CStochasticPolicy *policy); ~CFeatureVFunction(); virtual void setVFunctionFromQFunction(CFeatureQFunction *qfunction, CStochasticPolicy *policy); virtual void updateWeights(CFeatureList *gradientFeatures); virtual void updateValue(CState *state, double td); virtual void setValue(CState *state, double qValue); virtual double getValue(CState *state); virtual void saveData(FILE *file); virtual void loadData(FILE *file); virtual void printValues(); virtual CAbstractVETraces *getStandardETraces(); virtual void getGradient(CStateCollection *state, CFeatureList *gradientFeatures); virtual int getNumWeights(); virtual void resetData(); virtual void getWeights(double *parameters); virtual void setWeights(double *parameters); void setFeatureCalculator(CFeatureCalculator *featCalc); }; /* class CFeatureVFunctionInputDerivationCalculator : public CVFunctionInputDerivationCalculator { protected: CFeatureVFunction *vFunction; ColumnVector *featureInputDerivation; ColumnVector *fiDerivationSum; public: bool normalizedFeatures; CFeatureVFunctionInputDerivationCalculator(CStateProperties *inputState, CFeatureVFunction *vFunction); ~CFeatureVFunctionInputDerivationCalculator(); virtual void getInputDerivation( CStateCollection *state, ColumnVector *targetVector); };*/ /* Tables are just the same as feature functions. The only difference is the kind of states for the value-table, it uses discrete states. The class CVTable represents tabular value functions, the class is subclass of CFeatureVFunction. Value-Tables can only be used with CAbstractStateDiscretizer. */ class CVTable : public CFeatureVFunction { public: CVTable(CAbstractStateDiscretizer *state); ~CVTable(); void setDiscretizer(CAbstractStateDiscretizer *discretizer); CAbstractStateDiscretizer *getDiscretizer(); int getNumStates(); }; class CRewardAsVFunction : public CAbstractVFunction { protected: CStateReward *reward; public: CRewardAsVFunction(CStateReward *reward); virtual ~CRewardAsVFunction() {}; virtual double getValue(CState *state); }; #endif