RL/valueIteration.cpp at master · nicolasvallee/RL · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
#include <iostream>
#include <vector>
#include <algorithm>

using namespace std;

const int NB_STATES = 5;
const int NB_ACTIONS = 2;
const double gamma = 1;
double value[NB_STATES];
double P[NB_STATES][NB_STATES][NB_ACTIONS] = {
    {{1,0},{0,1},{0,0},{0,0},{0,0}},
    {{1,0},{0,0},{0,1},{0,0},{0,0}},
    {{0,0},{0,0},{0,0},{1,0},{0,0}},
    {{0,0},{0.2,0},{0.4,0},{0.4,0},{0,1}},
    {{0,0},{0,0},{0,0},{0,0},{0,0}}
    };;

double reward[NB_STATES][NB_ACTIONS] = {
    {-1,0},{-1,-2},{-2,0},{1,10},{0,0}
};
const int oo = 1e8;

const double epsilon = 0.1;

void valueIteration()
{
    bool valueHasChanged = true;
    while (valueHasChanged)
    {
        valueHasChanged = false;
        for(int state1 = 0; state1 < NB_STATES; state1++)
        {
            double max_value = value[state1];
            for(int action = 0; action < NB_ACTIONS; action++)
            {
                double sum = 0;
                sum = reward[state1][action];
                for(int state2 = 0; state2 < NB_STATES; state2++)
                {
                    sum += gamma * P[state1][state2][action] * value[state2];
                }
                max_value = max(max_value, sum);
            }
            if(max_value - value[state1] > epsilon)
                valueHasChanged = true;
            value[state1] = max_value;
        }
    }
}

void input()
{
    cout << "probabilities \n";

    for(int i = 0; i < NB_STATES; i++)
        for(int j = 0; j < NB_STATES; j++)
            for(int k = 0; k < NB_ACTIONS; k++)
            {
                cout << "state1 " << i << " state2 " << j << " action " << k << '\n';
                cin >> P[i][j][k];
            }

    cout << "rewards \n";

    for(int i = 0; i < NB_STATES; i++)
        for(int j = 0; j < NB_ACTIONS; j++)
        {
            cout << "state " << i << " action " << j << '\n';
            cin >> reward[i][j];
        }
}

int main()
{

    valueIteration();


    cout << "results \n";
    for(int i = 0; i < NB_STATES; i++)
        cout << "state" << i << " " << value[i] << '\n';

    return 0;
}