Lab_05_Logistic_Regression.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lab-05 Logistic Regression.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOk4/ZQZ1Tj5c0lR6cJ7rKa"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hZPuYBAwua-7"
      },
      "source": [
        "# Logistic Regression\n",
        "classification 문제"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_oCgqKHMuJ8F"
      },
      "source": [
        "# import\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DX44xW7PxLVx",
        "outputId": "1b886ca3-869f-4461-cca6-fdb6738048d3"
      },
      "source": [
        "# For reproducibility\n",
        "torch.manual_seed(1)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7f553eb0dad0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-nDgrErDxSZd"
      },
      "source": [
        "## Traning Data\n",
        "\n",
        "|x_data| = (6, 2)\n",
        "\n",
        "|y_data| = (6, )"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fazK8VLyxRoO"
      },
      "source": [
        "x_data = [[1, 2], [2, 3], [3, 1], [4, 3], [5, 3], [6, 2]]\n",
        "y_data = [[0], [0], [0], [1], [1], [1]]"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R0MqzJ3lxpy9"
      },
      "source": [
        "x_train = torch.FloatTensor(x_data)\n",
        "y_train = torch.FloatTensor(y_data)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h4wfZWTxxyni",
        "outputId": "1064d762-86e0-453c-d37a-66a9e5061ee9"
      },
      "source": [
        "print(x_train.shape)\n",
        "print(y_train.shape)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([6, 2])\n",
            "torch.Size([6, 1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jv7JJko0x53E"
      },
      "source": [
        "## Computing the Hypothesis\n",
        "\n",
        "We can use `torch.exp` to compute the hypothesis function conviniently."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JNLlpfSWx9cR",
        "outputId": "6be97f4a-b542-482f-bb75-a5fa9c59049b"
      },
      "source": [
        "W = torch.zeros((2, 1), requires_grad = True) # dim = 2\n",
        "b = torch.zeros(1, requires_grad = True)\n",
        "\n",
        "hypothesis = 1 / (1 + torch.exp(-(x_train.matmul(W)) + b))\n",
        "\n",
        "print(hypothesis)\n",
        "print(hypothesis.shape)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000]], grad_fn=<MulBackward0>)\n",
            "torch.Size([6, 1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X5Z9vH4py65f"
      },
      "source": [
        "pytorch에서 sigmoid 함수 `torch.sigmoid()`를 제공하고 있기 때문에 굳이 수식으로 hypothesis를 적을 필요 없음."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WOThaSNTzBvM",
        "outputId": "260e8a80-1d37-4cf1-d2e4-e7090515e0fe"
      },
      "source": [
        "hypothesis = torch.sigmoid(x_train.matmul(W)+b)\n",
        "\n",
        "print(hypothesis)\n",
        "print(hypothesis.shape)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000]], grad_fn=<SigmoidBackward>)\n",
            "torch.Size([6, 1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QvOcSN3ezAR-"
      },
      "source": [
        "We want to measure the difference between `hypothesis` and `y_train`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8oIiIhaczaht",
        "outputId": "a1c67ea0-2784-4fcf-bd68-b6f8d5a35fce"
      },
      "source": [
        "print(hypothesis)\n",
        "print(y_train)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000],\n",
            "        [0.5000]], grad_fn=<SigmoidBackward>)\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [1.],\n",
            "        [1.],\n",
            "        [1.]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sziw-UQezeab"
      },
      "source": [
        "To compute the losses for the entire batch, we can simply input the entire vector."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L-LnvQp6zkyX",
        "outputId": "53181685-abe8-493f-f6cb-40cd19c31061"
      },
      "source": [
        "losses = -(y_train * torch.log(hypothesis) + (1-y_train) * torch.log(1-hypothesis))\n",
        "print(losses)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.6931],\n",
            "        [0.6931],\n",
            "        [0.6931],\n",
            "        [0.6931],\n",
            "        [0.6931],\n",
            "        [0.6931]], grad_fn=<NegBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XcHDFbyCz1rJ"
      },
      "source": [
        "Then, we just `.mean()` to take the mean of these individual losses."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o5uQCkc1z7MB",
        "outputId": "8b203fe1-969e-4a99-85dd-062baff4c392"
      },
      "source": [
        "cost = losses.mean()\n",
        "print(cost)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.6931, grad_fn=<MeanBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yerm1WQg0ENV"
      },
      "source": [
        "위의 긴 수식의 과정들을 `binary_cross_entropy`로 구할 수 있음."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q5iX0MQj0CdX",
        "outputId": "0cb3b23f-c909-4efa-f31c-5e5c00e46676"
      },
      "source": [
        "F.binary_cross_entropy(hypothesis, y_train)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(0.6931, grad_fn=<BinaryCrossEntropyBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uVw-qFwCZOhX"
      },
      "source": [
        "# Whole Training Procedure"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xHkONzQ9ZV9n",
        "outputId": "1067aeb2-1d9d-4a7c-d2d6-c3a983b139e1"
      },
      "source": [
        "# 모델 초기화\n",
        "W = torch.zeros((2, 1), requires_grad = True)\n",
        "b = torch.zeros(1, requires_grad = True)\n",
        "\n",
        "# optimizer 설정\n",
        "optimizer = torch.optim.SGD((W, b), lr = 1)\n",
        "\n",
        "nb_epochs = 1000\n",
        "for epoch in range(nb_epochs + 1):\n",
        "\n",
        "  # cost 계산\n",
        "  hypothesis = torch.sigmoid(x_train.matmul(W) + b)\n",
        "  cost = F.binary_cross_entropy(hypothesis, y_train)\n",
        "\n",
        "  # cost로 H(x) 계산\n",
        "  optimizer.zero_grad() # 초기화를 하지 않으면 기존 grad에 더해지므로 꼭 초기화하기.\n",
        "  cost.backward()\n",
        "  optimizer.step()\n",
        "\n",
        "  # 100번마다 로그 출력\n",
        "  if epoch % 100 == 0:\n",
        "    print('Epoch: {:4d}/{} Cost: {:.6f}'.format(\n",
        "        epoch, nb_epochs, cost.item()\n",
        "    ))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:    0/1000 Cost: 0.693147\n",
            "Epoch:  100/1000 Cost: 0.134722\n",
            "Epoch:  200/1000 Cost: 0.080643\n",
            "Epoch:  300/1000 Cost: 0.057900\n",
            "Epoch:  400/1000 Cost: 0.045300\n",
            "Epoch:  500/1000 Cost: 0.037261\n",
            "Epoch:  600/1000 Cost: 0.031672\n",
            "Epoch:  700/1000 Cost: 0.027556\n",
            "Epoch:  800/1000 Cost: 0.024394\n",
            "Epoch:  900/1000 Cost: 0.021888\n",
            "Epoch: 1000/1000 Cost: 0.019852\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cBoF6ZYway7Y"
      },
      "source": [
        "# Evaluation\n",
        "\n",
        "원래는 `x_test`, 즉 test data set에 대해서 수행을 해줘야함."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NQAAMKDRa3K9",
        "outputId": "5d9182b7-6cc0-4916-e2b0-c005d8b8a701"
      },
      "source": [
        "hypothesis = torch.sigmoid(x_train.matmul(W) + b)\n",
        "print(hypothesis[:5])"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[2.7648e-04],\n",
            "        [3.1608e-02],\n",
            "        [3.8977e-02],\n",
            "        [9.5622e-01],\n",
            "        [9.9823e-01]], grad_fn=<SliceBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BitG1LySbYlt",
        "outputId": "9efeda26-3428-4ae3-874d-156677ff29fd"
      },
      "source": [
        "prediction = hypothesis >= torch.FloatTensor([0.5]) # prediction type : byteTensor\n",
        "print(prediction[:5])"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[False],\n",
            "        [False],\n",
            "        [False],\n",
            "        [ True],\n",
            "        [ True]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dWcC6ov2bj9G",
        "outputId": "a89c8fd6-0d7a-42e0-8592-e65e64fcf061"
      },
      "source": [
        "# compare with y_train\n",
        "print(prediction[:5])\n",
        "print(y_train[:5])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[False],\n",
            "        [False],\n",
            "        [False],\n",
            "        [ True],\n",
            "        [ True]])\n",
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.],\n",
            "        [1.],\n",
            "        [1.]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_j-wYsKDbrmm",
        "outputId": "b7adc32a-c5f1-458e-c15f-ab915a8bfc0c"
      },
      "source": [
        "correct_prediction = prediction.float() == y_train\n",
        "print(correct_prediction[:5])\n",
        "accuracy = torch.mean(correct_prediction.float())\n",
        "print(accuracy)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[True],\n",
            "        [True],\n",
            "        [True],\n",
            "        [True],\n",
            "        [True]])\n",
            "tensor(1.)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5le0F3vecE8P"
      },
      "source": [
        "# Hyper Implementation with class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GFS_xOkmcJRO"
      },
      "source": [
        "class BinaryClassifier(nn.Module):\n",
        "  def __init__(self):\n",
        "    super().__init__()\n",
        "    # W와 b가 들어있는 linear layer\n",
        "    # self.linear = {W, b}\n",
        "    self.linear = nn.Linear(2, 1) # dim = 2이므로 nn.Linear(2, 1)이 맞음.\n",
        "    self.sigmoid = nn.Sigmoid()\n",
        "  \n",
        "  def forward(self, x):\n",
        "    return self.sigmoid(self.linear(x))"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7O_rye1Sclxo",
        "outputId": "9ff0be51-5650-47c5-f648-2ffb3709fa5b"
      },
      "source": [
        "model = BinaryClassifier()\n",
        "\n",
        "# optimizer 설정\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr = 1)\n",
        "\n",
        "nb_epochs = 100\n",
        "for epoch in range(nb_epochs + 1):\n",
        "  # H(x) 계산\n",
        "  hypothesis = model(x_train)\n",
        "\n",
        "  # cost 계산\n",
        "  cost = F.binary_cross_entropy(hypothesis, y_train)\n",
        "\n",
        "  # cost로 H(x) 계산\n",
        "  optimizer.zero_grad() # 초기화를 하지 않으면 기존 grad에 더해지므로 꼭 초기화하기.\n",
        "  cost.backward()\n",
        "  optimizer.step()\n",
        "\n",
        "  # 100번마다 로그 출력\n",
        "  if epoch % 10 == 0:\n",
        "    prediction = hypothesis >= torch.FloatTensor([0.5])\n",
        "    correct_prediction = prediction.float() == y_train\n",
        "    accuracy = correct_prediction.sum().item() / len(correct_prediction)\n",
        "    print('Epoch: {:4d}/{} Cost: {:.6f} Accuracy: {:2.2f}%'.format(\n",
        "        epoch, nb_epochs, cost.item(), accuracy*100\n",
        "    ))"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:    0/100 Cost: 0.590113 Accuracy: 50.00%\n",
            "Epoch:   10/100 Cost: 0.442518 Accuracy: 66.67%\n",
            "Epoch:   20/100 Cost: 0.412972 Accuracy: 83.33%\n",
            "Epoch:   30/100 Cost: 0.346455 Accuracy: 83.33%\n",
            "Epoch:   40/100 Cost: 0.292822 Accuracy: 83.33%\n",
            "Epoch:   50/100 Cost: 0.244709 Accuracy: 100.00%\n",
            "Epoch:   60/100 Cost: 0.201719 Accuracy: 100.00%\n",
            "Epoch:   70/100 Cost: 0.168851 Accuracy: 100.00%\n",
            "Epoch:   80/100 Cost: 0.150460 Accuracy: 100.00%\n",
            "Epoch:   90/100 Cost: 0.139416 Accuracy: 100.00%\n",
            "Epoch:  100/100 Cost: 0.130253 Accuracy: 100.00%\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}