Reinforcement Learning Toolbox 2.0 last updated:

General Documentation Manual Tutorial Class Reference Master Thesis Examples Related Papers Downloads Links News mailto:webmaster Main Page     Class Hierarchy   Compound List   File List   Compound Members   File Members ctheoreticalmodel.h Go to the documentation of this file. // Copyright (C) 2003 // Gerhard Neumann (gneumann@gmx.net) // Stephan Neumann (sneumann@gmx.net) // // This file is part of RL Toolbox. // http://www.igi.tugraz.at/ril_toolbox // // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions // are met: // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // 3. The name of the author may not be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef C_DISCRETETHEORETICALMODEL_H #define C_DISCRETETHEORETICALMODEL_H #include "clearndataobject.h" #include "caction.h" #include "cbaseobjects.h" #include "cagentlistener.h" #include "cutility.h" #include <map> #include <list> #define TRANSITION 1 #define SEMIMDPTRANSITION 2 class CFeatureQFunction; class CAbstractStateDiscretizer; class CFeatureCalculator; class CTransitionFunction; class CFeatureList; class CStateCollection; class CState; //class CMyArray2D<CStateActionTransitions *>; class CTransition { protected: int startState; int endState; double propability; int type; public: CTransition(int startState, int endState, double prop); virtual ~CTransition() {}; int getStartState(); int getEndState(); virtual double getPropability(); virtual void setPropability(double prop); virtual void loadASCII(FILE *stream, int fixedState, bool forward); virtual void saveASCII(FILE *stream, bool forward); virtual bool isType(int Type); }; class CSemiMDPTransition : public CTransition { protected: std::map<int, double> *durations; public: CSemiMDPTransition(int startState, int endState, double prop); virtual ~CSemiMDPTransition(); std::map<int, double> *getDurations(); void addDuration(int duration, double factor); void setDuration(int duration, double factor); double getDurationFaktor(int duration); double getDurationPropability(int duration); virtual void loadASCII(FILE *stream, int fixedState, bool forward); virtual void saveASCII(FILE *stream, bool forward); double getSemiMDPFaktor(double gamma); }; /***The transitions are all stored in a CTransitionList object. The transition list stores whether it is a forward or a backward list. The Transitions are stored in a ordered list, the list is ordered by end-states for forward lists and by start-states for backward lists. It provides functions for adding a specific transition, getting the transition given a feature index and determining whether a feature index is member of the transition list. If the list is a forward list the search criteria for get and isMember is obviously the end-State otherwise the start-state of the transitions. */ class CTransitionList : public std::list<CTransition *> { protected: bool forwardList; public: CTransitionList(bool forwardList); bool isMember(int featureIndex); bool isForwardList(); /*Adds a Transition to the sorted list in the right position*/ void addTransition(CTransition *transition); CTransition *getTransition(int featureIndex); CTransitionList::iterator getTransitionIterator(int featureIndex); void clearAndDelete(); }; class CStateActionTransitions { protected: CTransitionList *forwardList; CTransitionList *backwardList; public: CStateActionTransitions(); ~CStateActionTransitions(); CTransitionList* getForwardTransitions(); CTransitionList* getBackwardTransitions(); }; class CAbstractFeatureStochasticModel : public CActionObject { protected: unsigned int numFeatures; CStateModifier *discretizer; bool createdActions; public: CAbstractFeatureStochasticModel(CActionSet *actions, int numStates); CAbstractFeatureStochasticModel(CActionSet *actions, CStateModifier *discretizer); CAbstractFeatureStochasticModel(int numActions, int numFeatures); virtual ~CAbstractFeatureStochasticModel(); virtual double getPropability(int oldFeature, CAction *action, int newFeature); virtual double getPropability(int oldFeature, int action, int newFeature) = 0; virtual double getPropability(int oldFeature, int action, int duration, int newFeature) = 0; virtual double getPropability(CFeatureList *oldList, CAction *action, CFeatureList *newList); virtual double getPropability(CStateCollection *oldState, CAction *action, CStateCollection *newState); virtual double getPropability(CState *oldState, CAction *action, CState *newState); virtual CTransitionList* getForwardTransitions(int action, int state) = 0; virtual CTransitionList* getForwardTransitions(CAction *action, CState *state); virtual CTransitionList* getForwardTransitions(CAction *action, CStateCollection *state); virtual CTransitionList* getBackwardTransitions(int action, int state) = 0; virtual unsigned int getNumFeatures(); }; class CFeatureStochasticModel : public CAbstractFeatureStochasticModel { protected: CMyArray2D<CStateActionTransitions *> *stateTransitions; void loadASCII(FILE *stream); CTransition *getNewTransition(int startState, int endState, CAction *action, double propability); public: CFeatureStochasticModel(CActionSet *actions, int numFeatures, FILE *file); CFeatureStochasticModel(CActionSet *actions, int numFeatures); CFeatureStochasticModel(int numActions, int numFeatures); virtual ~CFeatureStochasticModel(); virtual double getPropability(int oldFeature, int action, int newFeature); // returns the Propability of the transition virtual double getPropability(int oldFeature, int action, int duration, int newFeature); void setPropability(double propability, int oldFeature, int action, int newFeature); void setPropability(double propability, int oldFeature, int action, int duration, int newFeature); virtual CTransitionList* getForwardTransitions(int action, int state); virtual CTransitionList* getBackwardTransitions(int action, int state); virtual void saveASCII(FILE *stream); }; class CStochasticModelAction : public CPrimitiveAction { protected: CAbstractFeatureStochasticModel *model; public: CStochasticModelAction(CAbstractFeatureStochasticModel *model); virtual ~CStochasticModelAction(){}; virtual bool isAvailable(CStateCollection *state); }; /* class CFeatureStateVisitCounter : public CLearnDataObject, public CStateObject, public CSemiMDPListener { protected: }; class CFeatureStateActionVisitCounter : public CLearnDataObject, public CStateObject, public CSemiMDPListener { protected: };*/ class CAbstractFeatureStochasticEstimatedModel : public CFeatureStochasticModel, public CSemiMDPListener, public CStateObject, public CLearnDataObject { protected: CFeatureQFunction *stateActionVisits; virtual void updateStep(int oldFeature, CAction *action, int newFeature, double Faktor); public: CAbstractFeatureStochasticEstimatedModel(CStateProperties *properties, CFeatureQFunction *stateActionVisits, CActionSet *actions, int numFeatures); CAbstractFeatureStochasticEstimatedModel(CStateProperties *properties, CFeatureQFunction *stateActionVisits, CActionSet *actions, int numFeatures, FILE *file); virtual ~CAbstractFeatureStochasticEstimatedModel(); virtual void nextStep(CStateCollection *oldState, CAction *action, CStateCollection *nextState) = 0; virtual void intermediateStep(CStateCollection *oldState, CAction *action, CStateCollection *nextState); virtual void saveData(FILE *stream); virtual void loadData(FILE *stream); virtual void resetData(); double getTransitionsVisits(int oldFeature, CAction *action, int newFeature); double getStateActionVisits(int Feature, int action); double getStateVisits(int Feature); }; class CDiscreteStochasticEstimatedModel : public CAbstractFeatureStochasticEstimatedModel { protected: CAbstractStateDiscretizer *discretizer; public: CDiscreteStochasticEstimatedModel(CAbstractStateDiscretizer *discState, CFeatureQFunction *stateActionVisits, CActionSet *actions); virtual ~CDiscreteStochasticEstimatedModel() {}; virtual void nextStep(CStateCollection *oldState, CAction *action, CStateCollection *nextState); int getStateActionVisits(int Feature, int action); int getStateVisits(int Feature); }; class CFeatureStochasticEstimatedModel : public CAbstractFeatureStochasticEstimatedModel { protected: CFeatureCalculator *featCalc; public: CFeatureStochasticEstimatedModel(CFeatureCalculator *properties, CFeatureQFunction *stateActionVisits, CActionSet *actions); virtual ~CFeatureStochasticEstimatedModel() {}; virtual void nextStep(CStateCollection *oldState, CAction *action, CStateCollection *nextState); }; #endif