ddpg.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Found existing installation: gym 0.26.2\n",
            "Uninstalling gym-0.26.2:\n",
            "  Successfully uninstalled gym-0.26.2\n",
            "Requirement already satisfied: swig in ./myenv/lib/python3.9/site-packages (4.2.1)\n",
            "Collecting gym[box2d]\n",
            "  Using cached gym-0.26.2-py3-none-any.whl\n",
            "Requirement already satisfied: numpy>=1.18.0 in ./myenv/lib/python3.9/site-packages (from gym[box2d]) (1.26.4)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in ./myenv/lib/python3.9/site-packages (from gym[box2d]) (3.0.0)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in ./myenv/lib/python3.9/site-packages (from gym[box2d]) (0.0.8)\n",
            "Requirement already satisfied: importlib-metadata>=4.8.0 in ./myenv/lib/python3.9/site-packages (from gym[box2d]) (7.1.0)\n",
            "Requirement already satisfied: box2d-py==2.3.5 in ./myenv/lib/python3.9/site-packages (from gym[box2d]) (2.3.5)\n",
            "Requirement already satisfied: pygame==2.1.0 in ./myenv/lib/python3.9/site-packages (from gym[box2d]) (2.1.0)\n",
            "Requirement already satisfied: swig==4.* in ./myenv/lib/python3.9/site-packages (from gym[box2d]) (4.2.1)\n",
            "Requirement already satisfied: zipp>=0.5 in ./myenv/lib/python3.9/site-packages (from importlib-metadata>=4.8.0->gym[box2d]) (3.18.1)\n",
            "Installing collected packages: gym\n",
            "Successfully installed gym-0.26.2\n"
          ]
        }
      ],
      "source": [
        "!pip3 uninstall gym --yes\n",
        "!pip3 install swig\n",
        "!pip3 install 'gym[box2d]'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "83PethNIFnG6"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "from collections import deque\n",
        "import random\n",
        "\n",
        "# Ornstein-Ulhenbeck Process\n",
        "# Taken from #https://github.com/vitchyr/rlkit/blob/master/rlkit/exploration_strategies/ou_strategy.py\n",
        "class OUNoise(object):\n",
        "    def __init__(self, action_space, mu=0.0, theta=0.15, max_sigma=0.3, min_sigma=0.3, decay_period=100000):\n",
        "        self.mu           = mu\n",
        "        self.theta        = theta\n",
        "        self.sigma        = max_sigma\n",
        "        self.max_sigma    = max_sigma\n",
        "        self.min_sigma    = min_sigma\n",
        "        self.decay_period = decay_period\n",
        "        self.action_dim   = action_space.shape[0]\n",
        "        self.low          = action_space.low\n",
        "        self.high         = action_space.high\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        self.state = np.ones(self.action_dim) * self.mu\n",
        "\n",
        "    def evolve_state(self):\n",
        "        x  = self.state\n",
        "        dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.action_dim)\n",
        "        self.state = x + dx\n",
        "        return self.state\n",
        "\n",
        "    def get_action(self, action, t=0):\n",
        "        ou_state = self.evolve_state()\n",
        "        self.sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * min(1.0, t / self.decay_period)\n",
        "        return np.clip(action + ou_state, self.low, self.high)\n",
        "\n",
        "class NormalNoiseProcess:\n",
        "    def __init__(self, action_space, var, decay, min_sigma):\n",
        "        action_shape = action_space.shape\n",
        "\n",
        "        self.mean   = np.zeros(action_shape)\n",
        "        self.sigma = var\n",
        "        self.sigma_decay = decay\n",
        "        self.min_sigma = min_sigma\n",
        "\n",
        "    def sample(self):\n",
        "        return np.random.normal(loc = self.mean, scale=self.sigma, size=self.mean.shape)\n",
        "\n",
        "    def decay(self):\n",
        "        self.sigma = max(self.min_sigma, self.sigma - self.sigma_decay)\n",
        "\n",
        "# https://github.com/openai/gym/blob/master/gym/core.py\n",
        "class NormalizedEnv(gym.ActionWrapper):\n",
        "    \"\"\" Wrap action \"\"\"\n",
        "\n",
        "    def action(self, action):\n",
        "        act_k = (self.action_space.high - self.action_space.low)/ 2.\n",
        "        act_b = (self.action_space.high + self.action_space.low)/ 2.\n",
        "        return act_k * action + act_b\n",
        "\n",
        "\n",
        "\n",
        "class Memory:\n",
        "    def __init__(self, max_size):\n",
        "        self.max_size = max_size\n",
        "        self.buffer = deque(maxlen=max_size)\n",
        "\n",
        "    def push(self, state, action, reward, next_state, done):\n",
        "        experience = (state, action, np.array([reward]), next_state, done)\n",
        "        self.buffer.append(experience)\n",
        "\n",
        "    def sample(self, batch_size):\n",
        "        state_batch = []\n",
        "        action_batch = []\n",
        "        reward_batch = []\n",
        "        next_state_batch = []\n",
        "        done_batch = []\n",
        "\n",
        "        batch = random.sample(self.buffer, batch_size)\n",
        "\n",
        "        for experience in batch:\n",
        "            state, action, reward, next_state, done = experience\n",
        "            state_batch.append(state)\n",
        "            action_batch.append(action)\n",
        "            reward_batch.append(reward)\n",
        "            next_state_batch.append(next_state)\n",
        "            done_batch.append(done)\n",
        "\n",
        "        return state_batch, action_batch, reward_batch, next_state_batch, done_batch\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.buffer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 63,
      "metadata": {
        "id": "lD24XhWVF1If"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.autograd\n",
        "from torch.autograd import Variable\n",
        "\n",
        "class Critic(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, output_size):\n",
        "        super(Critic, self).__init__()\n",
        "        self.linear1 = nn.Linear(input_size, hidden_size)\n",
        "        f1 = 1./np.sqrt(self.linear1.weight.data.size()[0])\n",
        "        nn.init.uniform_(self.linear1.weight.data, -f1, f1)\n",
        "        nn.init.uniform_(self.linear1.bias.data, -f1, f1)\n",
        "        self.bn1 = nn.LayerNorm(hidden_size)\n",
        "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
        "        f2 = 1./np.sqrt(self.linear2.weight.data.size()[0])\n",
        "        nn.init.uniform_(self.linear2.weight.data, -f2, f2)\n",
        "        nn.init.uniform_(self.linear2.bias.data, -f2, f2)\n",
        "        self.bn2 = nn.LayerNorm(hidden_size)\n",
        "        self.linear3 = nn.Linear(hidden_size, hidden_size)\n",
        "        f3 = 1./np.sqrt(self.linear3.weight.data.size()[0])\n",
        "        nn.init.uniform_(self.linear3.weight.data, -f3, f3)\n",
        "        nn.init.uniform_(self.linear3.bias.data, -f3, f3)\n",
        "        self.bn3 = nn.LayerNorm(hidden_size)\n",
        "        self.linear4 = nn.Linear(hidden_size, output_size)\n",
        "        f4 = 1./np.sqrt(self.linear4.weight.data.size()[0])\n",
        "        nn.init.uniform_(self.linear4.weight.data, -f4, f4)\n",
        "        nn.init.uniform_(self.linear4.bias.data, -f4, f4)\n",
        "\n",
        "    def forward(self, state, action):\n",
        "        \"\"\"\n",
        "        Params state and actions are torch tensors\n",
        "        \"\"\"\n",
        "        x = torch.cat([state, action], 1)\n",
        "        x = self.bn1(F.relu(self.linear1(x)))\n",
        "        x = F.relu(self.linear2(x))\n",
        "        x = self.linear3(x)\n",
        "\n",
        "        return x\n",
        "\n",
        "class Actor(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, output_size):\n",
        "        super(Actor, self).__init__()\n",
        "        self.linear1 = nn.Linear(input_size, hidden_size)\n",
        "        f1 = 1./np.sqrt(self.linear1.weight.data.size()[0])\n",
        "        nn.init.uniform_(self.linear1.weight.data, -f1, f1)\n",
        "        nn.init.uniform_(self.linear1.bias.data, -f1, f1)\n",
        "        self.bn1 = nn.LayerNorm(hidden_size)\n",
        "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
        "        f2 = 1./np.sqrt(self.linear2.weight.data.size()[0])\n",
        "        nn.init.uniform_(self.linear2.weight.data, -f2, f2)\n",
        "        nn.init.uniform_(self.linear2.bias.data, -f2, f2)\n",
        "        self.bn2 = nn.LayerNorm(hidden_size)\n",
        "        self.linear3 = nn.Linear(hidden_size, output_size)\n",
        "        f3 = 1./np.sqrt(self.linear3.weight.data.size()[0])\n",
        "        nn.init.uniform_(self.linear3.weight.data, -f3, f3)\n",
        "        nn.init.uniform_(self.linear3.bias.data, -f3, f3)\n",
        "\n",
        "    def forward(self, state):\n",
        "        \"\"\"\n",
        "        Param state is a torch tensor\n",
        "        \"\"\"\n",
        "        x = self.bn1(F.relu(self.linear1(state)))\n",
        "        x = F.relu(self.linear2(x))\n",
        "        x = torch.tanh(self.linear3(x))\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xv8Id7pZGDOS",
        "outputId": "af468407-21b0-48f2-d968-d195b6b45d2e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "device cpu\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import torch.autograd\n",
        "import torch.optim as optim\n",
        "import torch.nn as nn\n",
        "# from model import *\n",
        "# from utils import *\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(\"device\",device)\n",
        "\n",
        "class DDPGagent:\n",
        "    def __init__(self, env, hidden_size=256, actor_learning_rate=1e-4, critic_learning_rate=1e-3, gamma=0.99, tau=1e-2, max_memory_size=50000):\n",
        "        # Params\n",
        "        self.num_states = env.observation_space.shape[0]\n",
        "        self.num_actions = env.action_space.shape[0]\n",
        "        self.gamma = gamma\n",
        "        self.tau = tau\n",
        "\n",
        "        # Networks\n",
        "        self.actor = Actor(self.num_states, hidden_size, self.num_actions).to(device)\n",
        "        self.actor_target = Actor(self.num_states, hidden_size, self.num_actions).to(device)\n",
        "        self.critic = Critic(self.num_states + self.num_actions, hidden_size, self.num_actions).to(device)\n",
        "        self.critic_target = Critic(self.num_states + self.num_actions, hidden_size, self.num_actions).to(device)\n",
        "\n",
        "\n",
        "        for target_param, param in zip(self.actor_target.parameters(), self.actor.parameters()):\n",
        "            target_param.data.copy_(param.data)\n",
        "            target_param.requires_grad = False\n",
        "\n",
        "        for target_param, param in zip(self.critic_target.parameters(), self.critic.parameters()):\n",
        "            target_param.data.copy_(param.data)\n",
        "            target_param.requires_grad = False\n",
        "\n",
        "        # Training\n",
        "        self.memory = Memory(max_memory_size)\n",
        "        self.critic_criterion  = nn.MSELoss()\n",
        "        self.actor_optimizer  = optim.Adam(self.actor.parameters(), lr=actor_learning_rate, step_size=1000, gamma=0.9, lr=1e-4, weight_decay=1e-5)\n",
        "        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=critic_learning_rate, step_size=1000, gamma=0.9, lr=1e-3, weight_decay=1e-5)\n",
        "\n",
        "    def get_action(self, state):\n",
        "        state = Variable(torch.from_numpy(state).float().unsqueeze(0)).to(device)\n",
        "        action = self.actor.forward(state)\n",
        "        action = action.detach().cpu().numpy()[0,0]\n",
        "        return action\n",
        "\n",
        "    def update(self, batch_size):\n",
        "        states, actions, rewards, next_states, _ = self.memory.sample(batch_size)\n",
        "        states = torch.FloatTensor(states).to(device)\n",
        "        actions = torch.FloatTensor(actions).to(device)\n",
        "        rewards = torch.FloatTensor(rewards).to(device)\n",
        "        next_states = torch.FloatTensor(next_states).to(device)\n",
        "\n",
        "        # Implement critic loss and update critic\n",
        "        with torch.no_grad():\n",
        "          Q_target = rewards + self.gamma * self.critic_target(next_states,self.actor_target(next_states))\n",
        "\n",
        "        Q_current = self.critic(states,actions)\n",
        "        critic_loss = self.critic_criterion(Q_target,Q_current)\n",
        "        self.critic_optimizer.zero_grad()\n",
        "        critic_loss.backward()\n",
        "        self.critic_optimizer.step()\n",
        "\n",
        "\n",
        "        # Implement actor loss and update actor\n",
        "\n",
        "        # We don't want to update critic in actor update, hence saving some computation\n",
        "        for parameters in self.critic.parameters():\n",
        "            parameters.requires_grad = False\n",
        "\n",
        "        actor_loss = -self.critic(states,self.actor(states)).mean()\n",
        "        self.actor_optimizer.zero_grad()\n",
        "        actor_loss.backward()\n",
        "        self.actor_optimizer.step()\n",
        "\n",
        "        for parameters in self.critic.parameters():\n",
        "            parameters.requires_grad = True\n",
        "\n",
        "        # update target networks\n",
        "        for target_param, param in zip(self.actor_target.parameters(), self.actor.parameters()):\n",
        "            target_param.data.copy_(self.tau*param.data + (1-self.tau)*target_param.data)\n",
        "\n",
        "        for target_param, param in zip(self.critic_target.parameters(), self.critic.parameters()):\n",
        "            target_param.data.copy_(self.tau*param.data + (1-self.tau)*target_param.data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "67dTiw8FGJT-",
        "outputId": "535d149d-aab0-4f83-cc6b-de03e5028b8b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "episode: 0, reward: -273.76, average _reward: nan \n",
            "episode: 1, reward: -612.91, average _reward: -273.7580365963993 \n",
            "episode: 2, reward: -383.87, average _reward: -443.33211943995286 \n",
            "episode: 3, reward: -100.52, average _reward: -423.5120641800716 \n",
            "episode: 4, reward: -178.24, average _reward: -342.76425663592363 \n",
            "episode: 5, reward: -176.42, average _reward: -309.85884041766656 \n",
            "episode: 6, reward: -207.82, average _reward: -287.6188316533285 \n",
            "episode: 7, reward: -99.57, average _reward: -276.2184686594823 \n",
            "episode: 8, reward: -268.38, average _reward: -254.13775280738338 \n",
            "episode: 9, reward: -226.93, average _reward: -255.7198905648151 \n",
            "episode: 10, reward: -272.49, average _reward: -252.8407901782738 \n",
            "episode: 11, reward: -249.28, average _reward: -252.71430575924904 \n",
            "episode: 12, reward: -169.09, average _reward: -216.35208328580688 \n",
            "episode: 13, reward: -420.44, average _reward: -194.87369066675615 \n",
            "episode: 14, reward: -322.61, average _reward: -226.86541432091076 \n",
            "episode: 15, reward: -560.62, average _reward: -241.30222507028853 \n",
            "episode: 16, reward: -642.71, average _reward: -279.72215322599567 \n",
            "episode: 17, reward: -262.41, average _reward: -323.21123234844094 \n",
            "episode: 18, reward: -702.01, average _reward: -339.49515714038944 \n",
            "episode: 19, reward: -632.93, average _reward: -382.8581954422621 \n",
            "episode: 20, reward: -1150.23, average _reward: -423.45855189810874 \n",
            "episode: 21, reward: -1007.89, average _reward: -511.23207881131395 \n",
            "episode: 22, reward: -721.7, average _reward: -587.0923323104283 \n",
            "episode: 23, reward: -738.44, average _reward: -642.3532011933273 \n",
            "episode: 24, reward: -617.15, average _reward: -674.1535453810436 \n",
            "episode: 25, reward: -410.74, average _reward: -703.6082893927725 \n",
            "episode: 26, reward: -328.17, average _reward: -688.6202607040461 \n",
            "episode: 27, reward: -554.79, average _reward: -657.1663351788097 \n",
            "episode: 28, reward: -664.26, average _reward: -686.4039395698911 \n",
            "episode: 29, reward: -703.25, average _reward: -682.6289024065029 \n",
            "episode: 30, reward: -543.66, average _reward: -689.6602535175427 \n",
            "episode: 31, reward: -867.64, average _reward: -629.0035597417648 \n",
            "episode: 32, reward: -670.93, average _reward: -614.9787163799436 \n",
            "episode: 33, reward: -589.23, average _reward: -609.9020045668992 \n",
            "episode: 34, reward: -248.62, average _reward: -594.9809324235205 \n",
            "episode: 35, reward: -225.47, average _reward: -558.1274352894262 \n",
            "episode: 36, reward: -364.74, average _reward: -539.6007198146125 \n",
            "episode: 37, reward: -556.52, average _reward: -543.2583719020298 \n",
            "episode: 38, reward: -467.85, average _reward: -543.431428662665 \n",
            "episode: 39, reward: -99.27, average _reward: -523.7910334282583 \n",
            "episode: 40, reward: -246.87, average _reward: -463.39332771809967 \n",
            "episode: 41, reward: -119.19, average _reward: -433.7143883748161 \n",
            "episode: 42, reward: -80.98, average _reward: -358.86986646092737 \n",
            "episode: 43, reward: -331.5, average _reward: -299.8745448268722 \n",
            "episode: 44, reward: -259.47, average _reward: -274.101825733206 \n",
            "episode: 45, reward: -72.14, average _reward: -275.1870645967607 \n",
            "episode: 46, reward: -560.4, average _reward: -259.8535281502333 \n",
            "episode: 47, reward: -434.54, average _reward: -279.419072084589 \n",
            "episode: 48, reward: -256.12, average _reward: -267.2213961769942 \n",
            "episode: 49, reward: -541.16, average _reward: -246.0480066436387 \n",
            "episode: 50, reward: -360.53, average _reward: -290.23745186507193 \n",
            "episode: 51, reward: -496.45, average _reward: -301.6036305008092 \n",
            "episode: 52, reward: -471.22, average _reward: -339.3295893184055 \n",
            "episode: 53, reward: -358.53, average _reward: -378.35418445063414 \n",
            "episode: 54, reward: -311.1, average _reward: -381.05707585487187 \n",
            "episode: 55, reward: -388.17, average _reward: -386.21991739683244 \n",
            "episode: 56, reward: -457.96, average _reward: -417.82373037595323 \n",
            "episode: 57, reward: -347.51, average _reward: -407.5801548670726 \n",
            "episode: 58, reward: -447.58, average _reward: -398.8772986294892 \n",
            "episode: 59, reward: -460.61, average _reward: -418.0229742026307 \n",
            "episode: 60, reward: -368.83, average _reward: -409.96808928808275 \n",
            "episode: 61, reward: -530.87, average _reward: -410.79721463086446 \n",
            "episode: 62, reward: -330.13, average _reward: -414.2394426512906 \n",
            "episode: 63, reward: -395.43, average _reward: -400.13040321276446 \n",
            "episode: 64, reward: -460.96, average _reward: -403.8199028146984 \n",
            "episode: 65, reward: -514.58, average _reward: -418.8065073182864 \n",
            "episode: 66, reward: -524.49, average _reward: -431.4470676884439 \n",
            "episode: 67, reward: -324.91, average _reward: -438.1001035286787 \n",
            "episode: 68, reward: -334.1, average _reward: -435.83950305341915 \n",
            "episode: 69, reward: -721.68, average _reward: -424.49172497373536 \n",
            "episode: 70, reward: -521.64, average _reward: -450.598597104663 \n",
            "episode: 71, reward: -511.54, average _reward: -465.88053177133617 \n",
            "episode: 72, reward: -379.24, average _reward: -463.9468354975879 \n",
            "episode: 73, reward: -534.02, average _reward: -468.8578638979808 \n",
            "episode: 74, reward: -694.29, average _reward: -482.7166691373408 \n",
            "episode: 75, reward: -531.7, average _reward: -506.04881993434276 \n",
            "episode: 76, reward: -557.1, average _reward: -507.76054672584667 \n",
            "episode: 77, reward: -491.72, average _reward: -511.02106573715264 \n",
            "episode: 78, reward: -392.61, average _reward: -527.702096042419 \n",
            "episode: 79, reward: -230.91, average _reward: -533.5531320042471 \n",
            "episode: 80, reward: -430.27, average _reward: -484.475995174723 \n",
            "episode: 81, reward: -474.3, average _reward: -475.33882733043237 \n",
            "episode: 82, reward: -398.48, average _reward: -471.61497583459834 \n",
            "episode: 83, reward: -136.12, average _reward: -473.53923024305175 \n",
            "episode: 84, reward: -422.15, average _reward: -433.7493735447989 \n",
            "episode: 85, reward: -715.59, average _reward: -406.53567597780585 \n",
            "episode: 86, reward: -541.96, average _reward: -424.9246571594972 \n",
            "episode: 87, reward: -396.4, average _reward: -423.4111025563555 \n",
            "episode: 88, reward: -217.07, average _reward: -413.8794462503917 \n",
            "episode: 89, reward: -780.44, average _reward: -396.3251674733955 \n",
            "episode: 90, reward: -490.09, average _reward: -451.2779834460489 \n",
            "episode: 91, reward: -501.17, average _reward: -457.25927995899127 \n",
            "episode: 92, reward: -337.05, average _reward: -459.9464549528203 \n",
            "episode: 93, reward: -710.74, average _reward: -453.8025581719905 \n",
            "episode: 94, reward: -378.07, average _reward: -511.2648660249861 \n",
            "episode: 95, reward: -418.08, average _reward: -506.85690448633005 \n",
            "episode: 96, reward: -473.74, average _reward: -477.10662259068357 \n",
            "episode: 97, reward: -445.64, average _reward: -470.2842929193034 \n",
            "episode: 98, reward: -542.16, average _reward: -475.20845811012293 \n",
            "episode: 99, reward: -378.12, average _reward: -507.7176738631292 \n",
            "episode: 100, reward: -755.78, average _reward: -467.4853516480228 \n",
            "episode: 101, reward: -336.14, average _reward: -494.05485136561373 \n",
            "episode: 102, reward: -272.32, average _reward: -477.55145107885755 \n",
            "episode: 103, reward: -421.85, average _reward: -471.0791531038153 \n",
            "episode: 104, reward: -443.78, average _reward: -442.19028372942694 \n",
            "episode: 105, reward: -458.97, average _reward: -448.76179493320194 \n",
            "episode: 106, reward: -606.26, average _reward: -452.85065618380287 \n",
            "episode: 107, reward: -408.65, average _reward: -466.1030239470382 \n",
            "episode: 108, reward: -754.81, average _reward: -462.4033056480177 \n",
            "episode: 109, reward: -261.41, average _reward: -483.66821380733575 \n",
            "episode: 110, reward: -327.07, average _reward: -471.9972803231605 \n",
            "episode: 111, reward: -1.58, average _reward: -429.1263585333989 \n",
            "episode: 112, reward: -72.71, average _reward: -395.6710114872645 \n",
            "episode: 113, reward: -357.38, average _reward: -375.7099941044096 \n",
            "episode: 114, reward: -374.72, average _reward: -369.2633236120258 \n",
            "episode: 115, reward: -66.11, average _reward: -362.35714623218956 \n",
            "episode: 116, reward: -189.14, average _reward: -323.0709562130437 \n",
            "episode: 117, reward: -118.13, average _reward: -281.35896305487427 \n",
            "episode: 118, reward: -264.32, average _reward: -252.3078178459195 \n",
            "episode: 119, reward: -443.4, average _reward: -203.2588622629832 \n",
            "episode: 120, reward: -434.85, average _reward: -221.45780730766177 \n",
            "episode: 121, reward: -241.13, average _reward: -232.23608445749215 \n",
            "episode: 122, reward: -310.92, average _reward: -256.1907221224769 \n",
            "episode: 123, reward: -424.44, average _reward: -280.01128287695985 \n",
            "episode: 124, reward: -352.83, average _reward: -286.71649161570343 \n",
            "episode: 125, reward: -339.69, average _reward: -284.52724182111587 \n",
            "episode: 126, reward: -352.74, average _reward: -311.88514711851997 \n",
            "episode: 127, reward: -174.18, average _reward: -328.24438295396595 \n",
            "episode: 128, reward: -252.21, average _reward: -333.84872391342856 \n",
            "episode: 129, reward: -460.4, average _reward: -332.6381351289019 \n",
            "episode: 130, reward: -473.01, average _reward: -334.3382422955584 \n",
            "episode: 131, reward: -312.08, average _reward: -338.15407487185223 \n",
            "episode: 132, reward: -242.39, average _reward: -345.2495684964762 \n",
            "episode: 133, reward: -241.59, average _reward: -338.39647828744114 \n",
            "episode: 134, reward: -215.36, average _reward: -320.1118701747632 \n",
            "episode: 135, reward: -384.31, average _reward: -306.36462065830756 \n",
            "episode: 136, reward: -335.07, average _reward: -310.82625885360903 \n",
            "episode: 137, reward: -372.33, average _reward: -309.0597642471213 \n",
            "episode: 138, reward: -373.3, average _reward: -328.87498721916495 \n",
            "episode: 139, reward: -350.32, average _reward: -340.9837145258762 \n",
            "episode: 140, reward: -404.83, average _reward: -329.9761647551942 \n",
            "episode: 141, reward: -406.93, average _reward: -323.1581277899606 \n",
            "episode: 142, reward: -307.41, average _reward: -332.64255280817986 \n",
            "episode: 143, reward: -557.64, average _reward: -339.1452484279234 \n",
            "episode: 144, reward: -577.97, average _reward: -370.7497352724033 \n",
            "episode: 145, reward: -614.53, average _reward: -407.0106638568858 \n",
            "episode: 146, reward: -435.56, average _reward: -430.03262629304555 \n",
            "episode: 147, reward: -564.9, average _reward: -440.0815856526895 \n",
            "episode: 148, reward: -596.83, average _reward: -459.3382128464933 \n",
            "episode: 149, reward: -909.63, average _reward: -481.69088811916856 \n",
            "episode: 150, reward: -779.98, average _reward: -537.621640657512 \n",
            "episode: 151, reward: -779.85, average _reward: -575.1361415938343 \n",
            "episode: 152, reward: -820.86, average _reward: -612.4282595165432 \n",
            "episode: 153, reward: -826.77, average _reward: -663.7728514777269 \n",
            "episode: 154, reward: -1281.7, average _reward: -690.6861362327448 \n",
            "episode: 155, reward: -845.14, average _reward: -761.0597561239899 \n",
            "episode: 156, reward: -856.74, average _reward: -784.1216765870906 \n",
            "episode: 157, reward: -683.01, average _reward: -826.2396903655163 \n",
            "episode: 158, reward: -412.06, average _reward: -838.0510374408856 \n",
            "episode: 159, reward: -650.73, average _reward: -819.574761022784 \n",
            "episode: 160, reward: -842.78, average _reward: -793.6851110798517 \n",
            "episode: 161, reward: -615.14, average _reward: -799.9652150554623 \n",
            "episode: 162, reward: -941.99, average _reward: -783.4943027989518 \n",
            "episode: 163, reward: -471.05, average _reward: -795.6074447468275 \n",
            "episode: 164, reward: -914.6, average _reward: -760.0355413898635 \n",
            "episode: 165, reward: -842.4, average _reward: -723.3247351289176 \n",
            "episode: 166, reward: -807.88, average _reward: -723.050108647539 \n",
            "episode: 167, reward: -411.32, average _reward: -718.1642583192408 \n",
            "episode: 168, reward: -771.11, average _reward: -690.9948810369219 \n",
            "episode: 169, reward: -1197.64, average _reward: -726.8999102998547 \n",
            "episode: 170, reward: -1140.98, average _reward: -781.5904018736212 \n",
            "episode: 171, reward: -946.13, average _reward: -811.4104974474634 \n",
            "episode: 172, reward: -728.98, average _reward: -844.50933280702 \n",
            "episode: 173, reward: -845.76, average _reward: -823.207931093594 \n",
            "episode: 174, reward: -727.0, average _reward: -860.6791073944817 \n",
            "episode: 175, reward: -612.06, average _reward: -841.91989294771 \n",
            "episode: 176, reward: -782.67, average _reward: -818.8864676847205 \n",
            "episode: 177, reward: -530.01, average _reward: -816.3650776413324 \n",
            "episode: 178, reward: -844.56, average _reward: -828.234359978803 \n",
            "episode: 179, reward: -1208.04, average _reward: -835.5789570077817 \n",
            "episode: 180, reward: -824.2, average _reward: -836.6190275952071 \n",
            "episode: 181, reward: -699.86, average _reward: -804.9412229460345 \n",
            "episode: 182, reward: -425.73, average _reward: -780.3146902031627 \n",
            "episode: 183, reward: -915.22, average _reward: -749.9899804380102 \n",
            "episode: 184, reward: -1110.22, average _reward: -756.9354630403889 \n",
            "episode: 185, reward: -829.36, average _reward: -795.256802803108 \n",
            "episode: 186, reward: -1038.65, average _reward: -816.9868594939073 \n",
            "episode: 187, reward: -730.71, average _reward: -842.5844991876053 \n",
            "episode: 188, reward: -738.12, average _reward: -862.6549156725605 \n",
            "episode: 189, reward: -1040.94, average _reward: -852.0109047314121 \n",
            "episode: 190, reward: -751.72, average _reward: -835.301321357288 \n",
            "episode: 191, reward: -678.65, average _reward: -828.0535004487159 \n",
            "episode: 192, reward: -709.07, average _reward: -825.93161379576 \n",
            "episode: 193, reward: -604.63, average _reward: -854.2651909865319 \n",
            "episode: 194, reward: -1455.49, average _reward: -823.2065063668921 \n",
            "episode: 195, reward: -492.72, average _reward: -857.7342434270406 \n",
            "episode: 196, reward: -801.76, average _reward: -824.0694139922164 \n",
            "episode: 197, reward: -1205.73, average _reward: -800.3813349813071 \n",
            "episode: 198, reward: -610.4, average _reward: -847.8830198660977 \n",
            "episode: 199, reward: -665.77, average _reward: -835.1108758013954 \n",
            "episode: 200, reward: -816.13, average _reward: -797.5930900932011 \n",
            "episode: 201, reward: -744.64, average _reward: -804.0335162870077 \n",
            "episode: 202, reward: -807.61, average _reward: -810.6329635885738 \n",
            "episode: 203, reward: -742.69, average _reward: -820.4868850877807 \n",
            "episode: 204, reward: -1016.19, average _reward: -834.2927973600783 \n",
            "episode: 205, reward: -834.28, average _reward: -790.3619204083125 \n",
            "episode: 206, reward: -683.22, average _reward: -824.5185916260895 \n",
            "episode: 207, reward: -663.91, average _reward: -812.6644482156659 \n",
            "episode: 208, reward: -782.56, average _reward: -758.4824656355706 \n",
            "episode: 209, reward: -734.59, average _reward: -775.6990770059572 \n",
            "episode: 210, reward: -655.91, average _reward: -782.5817157360751 \n",
            "episode: 211, reward: -648.97, average _reward: -766.5596209037104 \n"
          ]
        }
      ],
      "source": [
        "import sys\n",
        "import gym\n",
        "from gym.envs import box2d\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "env = NormalizedEnv(gym.make(\"LunarLander-v2\", continuous= True))\n",
        "#  hidden_size=256, actor_learning_rate=1e-4, critic_learning_rate=1e-3, gamma=0.99, tau=1e-2, max_memory_size=50000\n",
        "# hidden_size=512, actor_learning_rate=5e-8, critic_learning_rate=5e-6, gamma=0.98, tau=5e-2\n",
        "agent = DDPGagent(env, hidden_size=512, actor_learning_rate=1e-4, critic_learning_rate=1e-3, gamma=0.99, tau=1e-1, max_memory_size=50000)\n",
        "noise = OUNoise(env.action_space)\n",
        "batch_size = 64\n",
        "rewards = []\n",
        "avg_rewards = []\n",
        "\n",
        "for episode in range(10000):\n",
        "    state, _ = env.reset()\n",
        "    noise.reset()\n",
        "    episode_reward = 0\n",
        "\n",
        "    while True:\n",
        "        action = agent.get_action(state)\n",
        "        #Add noise to action\n",
        "\n",
        "        action = noise.get_action(action)\n",
        "        new_state, reward, done, info, other = env.step(action)\n",
        "        agent.memory.push(state, action, reward, new_state, done)\n",
        "\n",
        "        if len(agent.memory) > batch_size:\n",
        "            agent.update(batch_size)\n",
        "\n",
        "        state = new_state\n",
        "        episode_reward += reward\n",
        "\n",
        "        if done:\n",
        "            sys.stdout.write(\"episode: {}, reward: {}, average _reward: {} \\n\".format(episode, np.round(episode_reward, decimals=2), np.mean(rewards[-10:])))\n",
        "            torch.save(agent.actor.state_dict(), 'weights/ddpg.pth')\n",
        "            break\n",
        "\n",
        "    rewards.append(episode_reward)\n",
        "    avg_rewards.append(np.mean(rewards[-10:]))\n",
        "\n",
        "plt.plot(rewards)\n",
        "plt.plot(avg_rewards)\n",
        "plt.plot()\n",
        "plt.xlabel('Episode')\n",
        "plt.ylabel('Reward')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tD0lRTe-tXgw"
      },
      "source": [
        "Your Infrence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 449
        },
        "id": "rPIOCJzBGNvv",
        "outputId": "b76e0b3c-df2c-4152-86bf-71c877036497"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Users/jagjitsingh/Desktop/company-projects/RL-implementations/myenv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3504: RuntimeWarning: Mean of empty slice.\n",
            "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
            "/Users/jagjitsingh/Desktop/company-projects/RL-implementations/myenv/lib/python3.9/site-packages/numpy/core/_methods.py:129: RuntimeWarning: invalid value encountered in scalar divide\n",
            "  ret = ret.dtype.type(ret / rcount)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "episode: 0, reward: -166.07, average _reward: nan \n",
            "episode: 1, reward: -165.76, average _reward: -166.06975625048614 \n",
            "episode: 2, reward: -187.01, average _reward: -165.91600317439983 \n",
            "episode: 3, reward: -280.34, average _reward: -172.94886991819658 \n",
            "episode: 4, reward: -508.02, average _reward: -199.7970432075702 \n",
            "episode: 5, reward: -98.85, average _reward: -261.4412221522504 \n",
            "episode: 6, reward: -309.97, average _reward: -234.34259216181795 \n",
            "episode: 7, reward: -122.83, average _reward: -245.14645033970524 \n"
          ]
        },
        {
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "Cell \u001b[0;32mIn[15], line 18\u001b[0m\n\u001b[1;32m     16\u001b[0m state \u001b[38;5;241m=\u001b[39m new_state\n\u001b[1;32m     17\u001b[0m episode_reward \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m reward\n\u001b[0;32m---> 18\u001b[0m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m done:\n\u001b[1;32m     20\u001b[0m     sys\u001b[38;5;241m.\u001b[39mstdout\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mepisode: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, reward: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, average _reward: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(episode, np\u001b[38;5;241m.\u001b[39mround(episode_reward, decimals\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m), np\u001b[38;5;241m.\u001b[39mmean(rewards[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m100\u001b[39m:])))\n",
            "File \u001b[0;32m~/Desktop/company-projects/RL-implementations/myenv/lib/python3.9/site-packages/gym/core.py:329\u001b[0m, in \u001b[0;36mWrapper.render\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    325\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrender\u001b[39m(\n\u001b[1;32m    326\u001b[0m     \u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m    327\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Optional[Union[RenderFrame, List[RenderFrame]]]:\n\u001b[1;32m    328\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Renders the environment.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 329\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m~/Desktop/company-projects/RL-implementations/myenv/lib/python3.9/site-packages/gym/core.py:329\u001b[0m, in \u001b[0;36mWrapper.render\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    325\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrender\u001b[39m(\n\u001b[1;32m    326\u001b[0m     \u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m    327\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Optional[Union[RenderFrame, List[RenderFrame]]]:\n\u001b[1;32m    328\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Renders the environment.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 329\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m~/Desktop/company-projects/RL-implementations/myenv/lib/python3.9/site-packages/gym/wrappers/order_enforcing.py:51\u001b[0m, in \u001b[0;36mOrderEnforcing.render\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     46\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_disable_render_order_enforcing \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_has_reset:\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m ResetNeeded(\n\u001b[1;32m     48\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot call `env.render()` before calling `env.reset()`, if this is a intended action, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     49\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mset `disable_render_order_enforcing=True` on the OrderEnforcer wrapper.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     50\u001b[0m     )\n\u001b[0;32m---> 51\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m~/Desktop/company-projects/RL-implementations/myenv/lib/python3.9/site-packages/gym/wrappers/env_checker.py:55\u001b[0m, in \u001b[0;36mPassiveEnvChecker.render\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m env_render_passive_checker(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m     54\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 55\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m~/Desktop/company-projects/RL-implementations/myenv/lib/python3.9/site-packages/gym/envs/box2d/lunar_lander.py:710\u001b[0m, in \u001b[0;36mLunarLander.render\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    708\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscreen\u001b[38;5;241m.\u001b[39mblit(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msurf, (\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m0\u001b[39m))\n\u001b[1;32m    709\u001b[0m     pygame\u001b[38;5;241m.\u001b[39mevent\u001b[38;5;241m.\u001b[39mpump()\n\u001b[0;32m--> 710\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtick\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmetadata\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrender_fps\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    711\u001b[0m     pygame\u001b[38;5;241m.\u001b[39mdisplay\u001b[38;5;241m.\u001b[39mflip()\n\u001b[1;32m    712\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrender_mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrgb_array\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "env = NormalizedEnv(gym.make(\"LunarLander-v2\", continuous= True,render_mode='human'))\n",
        "# agent = DDPGagent(env)\n",
        "# agent.actor.load_state_dict(torch.load('weights/ddpg.pth', map_location=torch.device(device)))\n",
        "agent.actor.eval()\n",
        "\n",
        "rewards = []\n",
        "avg_rewards = []\n",
        "\n",
        "for episode in range(10):\n",
        "    state, _ = env.reset()\n",
        "    noise.reset()\n",
        "    episode_reward = 0\n",
        "    while True:\n",
        "        action = env.action_space.sample()\n",
        "        new_state, reward, done, info, other = env.step(action)\n",
        "        state = new_state\n",
        "        episode_reward += reward\n",
        "        env.render()\n",
        "        if done:\n",
        "            sys.stdout.write(\"episode: {}, reward: {}, average _reward: {} \\n\".format(episode, np.round(episode_reward, decimals=2), np.mean(rewards[-100:])))\n",
        "            break\n",
        "\n",
        "    rewards.append(episode_reward)\n",
        "    avg_rewards.append(np.mean(rewards[-10:]))\n",
        "env.close()\n",
        "plt.plot(rewards)\n",
        "plt.plot(avg_rewards)\n",
        "plt.plot()\n",
        "plt.xlabel('Episode')\n",
        "plt.ylabel('Reward')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F64fBbxhjbnV"
      },
      "outputs": [],
      "source": [
        "torch.save(agent.actor.state_dict(), 'ddpg.pth')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}