Race_Removal.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Race_Removal.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOPyKcSHB70yu+EmumIpogK",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b1607f03f92942f98f8e4e3829e0bb70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1cb82436b8844802b8e6261fbf437129",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5802970c8b494d8c825889e3207e8d9d",
              "IPY_MODEL_8f9aeaa2b5c24e79a10cc16d16a91426"
            ]
          }
        },
        "1cb82436b8844802b8e6261fbf437129": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5802970c8b494d8c825889e3207e8d9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d6b8f2e6401547a8931a00aefc8d50d3",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_63db77e36110414c82175c5c64169b7e"
          }
        },
        "8f9aeaa2b5c24e79a10cc16d16a91426": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a613aed2c5a14fc9a7207671d4ca7c5c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 2.06MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4fd6e142e17d428a83ed68cae9f5efc1"
          }
        },
        "d6b8f2e6401547a8931a00aefc8d50d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "63db77e36110414c82175c5c64169b7e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a613aed2c5a14fc9a7207671d4ca7c5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4fd6e142e17d428a83ed68cae9f5efc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "872ab89745a747feb03909e48d9bbc75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ade5cf1ba6024d318700c7e7ece119dd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b168fb722f6545b6a5704bd573862efb",
              "IPY_MODEL_204295f2de9142efba349d256ed438da"
            ]
          }
        },
        "ade5cf1ba6024d318700c7e7ece119dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b168fb722f6545b6a5704bd573862efb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e371c209aca449c78006a6d25d8cc66f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 443,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 443,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7af7414c39864a4080f8a9d57d8e75bd"
          }
        },
        "204295f2de9142efba349d256ed438da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2037eb3fbed14efca41cefdd2f42dcc1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 443/443 [00:55&lt;00:00, 7.93B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0e73cd230e124c87a68373d58c0fc3bd"
          }
        },
        "e371c209aca449c78006a6d25d8cc66f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7af7414c39864a4080f8a9d57d8e75bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2037eb3fbed14efca41cefdd2f42dcc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0e73cd230e124c87a68373d58c0fc3bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a07493433f7346d2b312c91ce321027d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3ed6c296ac0e49b4870101020b7dc8d9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b2bac57f84be49f6bac3e75e94971c04",
              "IPY_MODEL_cd4097dd7aed4b209505f35c29d5d40f"
            ]
          }
        },
        "3ed6c296ac0e49b4870101020b7dc8d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b2bac57f84be49f6bac3e75e94971c04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ded4cf81bc89432ca0751fc26f0e9ab9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1340675298,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1340675298,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ad21f0314025422281ffefcb48baaa58"
          }
        },
        "cd4097dd7aed4b209505f35c29d5d40f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2131d1bd8f0549a6b5525fc335479b58",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.34G/1.34G [00:55&lt;00:00, 24.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_95c9e8919a694ebbbcdf40c8a499e4d5"
          }
        },
        "ded4cf81bc89432ca0751fc26f0e9ab9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ad21f0314025422281ffefcb48baaa58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2131d1bd8f0549a6b5525fc335479b58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "95c9e8919a694ebbbcdf40c8a499e4d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/RudyVenguswamy/Portfolio/blob/master/Race_Removal.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qk2kHuLAExo8",
        "colab_type": "text"
      },
      "source": [
        "# Removing Race from Text Documents\n",
        "The following is a new version of a tool built by Rudy Venguswamy to remove race and gender from a document.\n",
        "\n",
        "Disclaimer:\n",
        "Simply removing race from documents does not go far enough to undo the systemic problems that lead to bias in decision making and the ways race are used to evaluate and judge humans in the United States. This tool is just one part of a larger change institutions should make to provide equity, justice and rights to those disenfranchised via the normative and oppressive systems of our country.\n",
        "\n",
        "I hope this tool can be used to not turn a blind eye to racism and bias, but to highlight its subtle presence and to consider what we must do to stop it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5lobV5GREtGg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "b1607f03f92942f98f8e4e3829e0bb70",
            "1cb82436b8844802b8e6261fbf437129",
            "5802970c8b494d8c825889e3207e8d9d",
            "8f9aeaa2b5c24e79a10cc16d16a91426",
            "d6b8f2e6401547a8931a00aefc8d50d3",
            "63db77e36110414c82175c5c64169b7e",
            "a613aed2c5a14fc9a7207671d4ca7c5c",
            "4fd6e142e17d428a83ed68cae9f5efc1",
            "872ab89745a747feb03909e48d9bbc75",
            "ade5cf1ba6024d318700c7e7ece119dd",
            "b168fb722f6545b6a5704bd573862efb",
            "204295f2de9142efba349d256ed438da",
            "e371c209aca449c78006a6d25d8cc66f",
            "7af7414c39864a4080f8a9d57d8e75bd",
            "2037eb3fbed14efca41cefdd2f42dcc1",
            "0e73cd230e124c87a68373d58c0fc3bd",
            "a07493433f7346d2b312c91ce321027d",
            "3ed6c296ac0e49b4870101020b7dc8d9",
            "b2bac57f84be49f6bac3e75e94971c04",
            "cd4097dd7aed4b209505f35c29d5d40f",
            "ded4cf81bc89432ca0751fc26f0e9ab9",
            "ad21f0314025422281ffefcb48baaa58",
            "2131d1bd8f0549a6b5525fc335479b58",
            "95c9e8919a694ebbbcdf40c8a499e4d5"
          ]
        },
        "outputId": "79f78237-e376-4a1a-f411-8772df124414"
      },
      "source": [
        "! git clone https://github.com/huggingface/transformers\n",
        "%cd transformers\n",
        "! pip install .\n",
        "! pip install -r ./examples/requirements.txt\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "from transformers import BertTokenizer, BertForQuestionAnswering #BertQA\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "modelqa = BertForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad', output_hidden_states= True)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 22, done.\u001b[K\n",
            "remote: Counting objects: 100% (22/22), done.\u001b[K\n",
            "remote: Compressing objects: 100% (15/15), done.\u001b[K\n",
            "remote: Total 29300 (delta 7), reused 10 (delta 3), pack-reused 29278\u001b[K\n",
            "Receiving objects: 100% (29300/29300), 26.60 MiB | 25.87 MiB/s, done.\n",
            "Resolving deltas: 100% (20299/20299), done.\n",
            "/content/transformers\n",
            "Processing /content/transformers\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (1.18.5)\n",
            "Collecting tokenizers==0.7.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/e5/a26eb4716523808bb0a799fcfdceb6ebf77a18169d9591b2f46a9adb87d9/tokenizers-0.7.0-cp36-cp36m-manylinux1_x86_64.whl (3.8MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8MB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (20.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (4.41.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (2019.12.20)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 27.9MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 46.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from transformers==2.11.0) (0.7)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==2.11.0) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==2.11.0) (1.12.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.11.0) (2020.4.5.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.11.0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.11.0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.11.0) (2.9)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.11.0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.11.0) (0.15.1)\n",
            "Building wheels for collected packages: transformers, sacremoses\n",
            "  Building wheel for transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-2.11.0-cp36-none-any.whl size=688471 sha256=a814c8ff8426b74ea842ad800d6934725d22e11b90fcb5906884bd82ac7ca9ae\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-nhcen9xf/wheels/23/19/dd/2561a4e47240cf6b307729d58e56f8077dd0c698f5992216cf\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=fcd9c3aaaf668bff13e77682b8d514957eac0c049dfe8b0896e641bdcfd88c6a\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built transformers sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.7.0 transformers-2.11.0\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.6/dist-packages (from -r ./examples/requirements.txt (line 1)) (2.2.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from -r ./examples/requirements.txt (line 2)) (0.22.2.post1)\n",
            "\u001b[33mWARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))': /simple/seqeval/\u001b[0m\n",
            "Collecting seqeval\n",
            "  Downloading https://files.pythonhosted.org/packages/34/91/068aca8d60ce56dd9ba4506850e876aba5e66a6f2f29aa223224b50df0de/seqeval-0.0.12.tar.gz\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (from -r ./examples/requirements.txt (line 4)) (5.4.8)\n",
            "Collecting sacrebleu\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/da/4b/6c7a0b26a48d88f56573d11aa5058808fe0d36ba40951287894f943556b5/sacrebleu-1.4.10-py3-none-any.whl (60kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 2.9MB/s \n",
            "\u001b[?25hCollecting rouge-score\n",
            "  Downloading https://files.pythonhosted.org/packages/1f/56/a81022436c08b9405a5247b71635394d44fe7e1dbedc4b28c740e09c2840/rouge_score-0.0.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: tensorflow_datasets in /usr/local/lib/python3.6/dist-packages (from -r ./examples/requirements.txt (line 7)) (2.1.0)\n",
            "Collecting pytorch-lightning==0.7.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4e/53/0549dd9c44c90e96d217592e094e9c53ef39ae2fed0c5cdb7e57aca65af6/pytorch_lightning-0.7.3-py3-none-any.whl (203kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 7.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from -r ./examples/requirements.txt (line 9)) (3.2.1)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (47.1.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (1.6.0.post3)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (1.7.2)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (0.34.2)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (0.9.0)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (3.10.0)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (2.23.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (1.0.1)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (1.12.0)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (1.29.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (3.2.2)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (0.4.1)\n",
            "Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard->-r ./examples/requirements.txt (line 1)) (1.18.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->-r ./examples/requirements.txt (line 2)) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->-r ./examples/requirements.txt (line 2)) (0.15.1)\n",
            "Requirement already satisfied: Keras>=2.2.4 in /usr/local/lib/python3.6/dist-packages (from seqeval->-r ./examples/requirements.txt (line 3)) (2.3.1)\n",
            "Collecting portalocker\n",
            "  Downloading https://files.pythonhosted.org/packages/53/84/7b3146ec6378d28abc73ab484f09f47dfa008ad6f03f33d90a369f880e25/portalocker-1.7.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (from rouge-score->-r ./examples/requirements.txt (line 6)) (3.2.5)\n",
            "Requirement already satisfied: tensorflow-metadata in /usr/local/lib/python3.6/dist-packages (from tensorflow_datasets->-r ./examples/requirements.txt (line 7)) (0.22.1)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.6/dist-packages (from tensorflow_datasets->-r ./examples/requirements.txt (line 7)) (1.1.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.6/dist-packages (from tensorflow_datasets->-r ./examples/requirements.txt (line 7)) (1.12.1)\n",
            "Requirement already satisfied: attrs>=18.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_datasets->-r ./examples/requirements.txt (line 7)) (19.3.0)\n",
            "Requirement already satisfied: promise in /usr/local/lib/python3.6/dist-packages (from tensorflow_datasets->-r ./examples/requirements.txt (line 7)) (2.3)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from tensorflow_datasets->-r ./examples/requirements.txt (line 7)) (0.3.1.1)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from tensorflow_datasets->-r ./examples/requirements.txt (line 7)) (0.16.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from tensorflow_datasets->-r ./examples/requirements.txt (line 7)) (4.41.1)\n",
            "Requirement already satisfied: torch>=1.1 in /usr/local/lib/python3.6/dist-packages (from pytorch-lightning==0.7.3->-r ./examples/requirements.txt (line 8)) (1.5.0+cu101)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r ./examples/requirements.txt (line 9)) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r ./examples/requirements.txt (line 9)) (2.4.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r ./examples/requirements.txt (line 9)) (1.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r ./examples/requirements.txt (line 9)) (2.8.1)\n",
            "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard->-r ./examples/requirements.txt (line 1)) (3.1.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard->-r ./examples/requirements.txt (line 1)) (0.2.8)\n",
            "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard->-r ./examples/requirements.txt (line 1)) (4.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard->-r ./examples/requirements.txt (line 1)) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard->-r ./examples/requirements.txt (line 1)) (2.9)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard->-r ./examples/requirements.txt (line 1)) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard->-r ./examples/requirements.txt (line 1)) (2020.4.5.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard->-r ./examples/requirements.txt (line 1)) (1.6.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard->-r ./examples/requirements.txt (line 1)) (1.3.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->-r ./examples/requirements.txt (line 3)) (1.0.8)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->-r ./examples/requirements.txt (line 3)) (2.10.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->-r ./examples/requirements.txt (line 3)) (1.1.2)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->-r ./examples/requirements.txt (line 3)) (3.13)\n",
            "Requirement already satisfied: googleapis-common-protos in /usr/local/lib/python3.6/dist-packages (from tensorflow-metadata->tensorflow_datasets->-r ./examples/requirements.txt (line 7)) (1.52.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard->-r ./examples/requirements.txt (line 1)) (0.4.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard->-r ./examples/requirements.txt (line 1)) (3.1.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard->-r ./examples/requirements.txt (line 1)) (3.1.0)\n",
            "Building wheels for collected packages: seqeval\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-0.0.12-cp36-none-any.whl size=7424 sha256=47a1dc06391e73818fc4002b76fa6615740e466b2af4644e3f3fea4af46abd22\n",
            "  Stored in directory: /root/.cache/pip/wheels/4f/32/0a/df3b340a82583566975377d65e724895b3fad101a3fb729f68\n",
            "Successfully built seqeval\n",
            "\u001b[31mERROR: rouge-score 0.0.4 has requirement six>=1.14.0, but you'll have six 1.12.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: pytorch-lightning 0.7.3 has requirement future>=0.17.1, but you'll have future 0.16.0 which is incompatible.\u001b[0m\n",
            "Installing collected packages: seqeval, portalocker, sacrebleu, rouge-score, pytorch-lightning\n",
            "Successfully installed portalocker-1.7.0 pytorch-lightning-0.7.3 rouge-score-0.0.4 sacrebleu-1.4.10 seqeval-0.0.12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b1607f03f92942f98f8e4e3829e0bb70",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "872ab89745a747feb03909e48d9bbc75",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=443.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a07493433f7346d2b312c91ce321027d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1340675298.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D0rpP3SKE3gd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#choose a topic by commenting one out.\n",
        "topic = 'race'\n",
        "#topic = 'gender'\n",
        "\n",
        "#what are some words you do not want removed even if they are related to the topic?\n",
        "exception_list = ['racism','racist','race']\n",
        "\n",
        "#Threshold for closeness of embeddings\n",
        "THRESHOLD = 0.85"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iOGy6o6e5Ir8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a3e0e258-35c2-4d8a-d5d3-3e6ff3c225f8"
      },
      "source": [
        "'''\n",
        "Generates the sentence embedding from BERT, a 1x1024 vector representing a sentence\n",
        "'''  \n",
        "def generate_vector(hidden): \n",
        "  #Found empirically that the 23 last layers performs the best for a sentence embedding representation\n",
        "  layer_depth = -23\n",
        "  vector = []\n",
        "  #shapes the embeddings into a 1x1024 vector\n",
        "  for i in hidden[layer_depth:-1]:\n",
        "    vector.append(i.detach().numpy()[0])\n",
        "  vector = np.array(vector).sum(axis =(0,1)).reshape(1024)\n",
        "  return vector\n",
        "\n",
        "def compute_corpus(corpus):\n",
        "#Precompute the embedding representation of the topic of interest to save time on comparisons later.\n",
        "\n",
        "  corpus_embeddings = []\n",
        "  for sentence in corpus:\n",
        "    input_ids = tokenizer.encode(sentence)\n",
        "    token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))]\n",
        "    #does prediction in order to generate hidden embeddings of each statement\n",
        "    _, _, hiddens_race = modelqa(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids]))\n",
        "    corpus_embeddings.append(generate_vector(hiddens_race))\n",
        "\n",
        "  #appends to corpus\n",
        "  corpus_embeddings = np.array(corpus_embeddings)\n",
        "  return corpus_embeddings\n",
        "\n",
        "if topic=='race':\n",
        "    corpus = [\"The person's race is White.\",\"The person's race is Black.\",\"The person's race is Hispanic.\",\"The person's race is Asian.\",\"The person's race is American Indian.\"]\n",
        "corpus_embeddings = compute_corpus(corpus)\n",
        "print(corpus_embeddings.shape)\n",
        "\n",
        "\n",
        "general_exception_list = np.array(['masked']) #necessary for code to function\n",
        "exception_list = np.append(np.array(exception_list), general_exception_list) #combines exception list specified by user along with code necessary list\n",
        "  "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(5, 1024)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h7um_Fwm4Gvs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "\n",
        "def sim_criteria(hiddens, topic):\n",
        "  candidate = generate_vector(hiddens)\n",
        "  similarity = cosine_similarity(corpus_embeddings, candidate.reshape(1,-1)).mean()\n",
        "  print('sim: ', similarity)\n",
        "  if topic is not 'race':\n",
        "    return True\n",
        "  return similarity > THRESHOLD #HYPER PARAM"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VqX0GlaTuWYR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def remove_term(text, topic):\n",
        "\n",
        "  #selects question to ask to remove race in document\n",
        "  if topic == 'race':\n",
        "    question = \"What is the subjects ethnicity?\"\n",
        "  elif topic == 'name':\n",
        "    question = \"Name?\"\n",
        "  elif topic == 'location':\n",
        "    question = \"Where are they from?\"\n",
        "  else:\n",
        "    return 'Topic Unsupported at this time.'\n",
        "\n",
        "  question = question.lower()\n",
        "\n",
        "  #processes text, first combining question and text as BERTQA expects. Second, it converts it into IDs\n",
        "  q_text = question+' ' + text\n",
        "  input_ids = tokenizer.encode(question,text)\n",
        "  token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))]\n",
        "\n",
        "  #does prediction\n",
        "  start_scores, end_scores, hiddens = modelqa(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids]))\n",
        "\n",
        "  #converts ids to tokens that are word like\n",
        "  all_tokens = tokenizer.convert_ids_to_tokens(input_ids)\n",
        "  start = torch.argmax(start_scores)\n",
        "  end = torch.argmax(end_scores)+1\n",
        "\n",
        "  #answer of question\n",
        "  answer = ' '.join(all_tokens[start : end])\n",
        "  print('Answer: ', answer)\n",
        "\n",
        "  exception_criteria = False\n",
        "  for word in exception_list:\n",
        "    if word in answer:\n",
        "      exception_criteria = True\n",
        "\n",
        "#is this worth including?\n",
        "  # if word not in general_exception_list[0]:\n",
        "  #   print(\"Stopping because algorithm was instructed not to remove word: \", word)\n",
        "\n",
        "\n",
        "  #only continue removing words if the embedding sim is close to a race related word and is not in exception list\n",
        "  if sim_criteria(hiddens, topic) and not exception_criteria and len(answer) > 0: \n",
        "    \n",
        "    for i in range(start,end):\n",
        "      all_tokens[i] = general_exception_list[0]\n",
        "    new_id = tokenizer.convert_tokens_to_ids(all_tokens)\n",
        "    re_q_text = tokenizer.decode(new_id, skip_special_tokens= True)\n",
        "  \n",
        "    re_text = re_q_text.split(question)[1]\n",
        "    #iterate again and remove more words\n",
        "    \n",
        "    return remove_term(re_text, topic)\n",
        "\n",
        "  else:\n",
        "    new_id = tokenizer.convert_tokens_to_ids(all_tokens)\n",
        "    return tokenizer.decode(new_id, skip_special_tokens= True).split(question)[1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f5lhuRX3urqH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "9c59232d-d2da-45b7-fff7-899cc5df7e48"
      },
      "source": [
        "#paragraph from sample text\n",
        "text = \"Royce is lowkey racist. He constantly goes around virtue signaling, but when push comes to shove, he isn't a true ally to the movement. \\\n",
        "I think it's pretty telling- that Roy comes from an upper middle class, white family. He's from the uptown area of Manhattan. His parents own an entire floor there.\"\n",
        "res = remove_term(text, 'race')\n",
        "res"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Answer:  white\n",
            "sim:  0.8501217\n",
            "Answer:  upper middle class , masked family . he ' s from the uptown area of manhattan\n",
            "sim:  0.849675\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\" roy is lowkey racist. he constantly goes around virtue signaling, but when push comes to shove, he isn't a true ally to the movement. i think it's pretty telling - that roy comes from an upper middle class, masked family. he's from the uptown area of manhattan. his parents own an entire floor there.\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iK1iL7hPuuOX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "outputId": "ac06c563-967a-4ffd-b375-2463de24a247"
      },
      "source": [
        "res2 = remove_term(res, 'name')\n",
        "res2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Answer:  roy\n",
            "sim:  0.8418867\n",
            "Answer:  roy\n",
            "sim:  0.8421305\n",
            "Answer:  masked\n",
            "sim:  0.8439621\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\" masked is lowkey racist. he constantly goes around virtue signaling, but when push comes to shove, he isn't a true ally to the movement. i think it's pretty telling - that masked comes from an upper middle class, masked family. he's from the uptown area of manhattan. his parents own an entire floor there.\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ySvMyGgQOHGN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "b4d0746e-8e23-4898-d657-6ea5a92a5ebb"
      },
      "source": [
        "res3 = remove_term(res2, 'location')\n",
        "res3"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Answer:  uptown area of manhattan\n",
            "sim:  0.8506276\n",
            "Answer:  masked masked masked masked\n",
            "sim:  0.8481865\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\" masked is lowkey racist. he constantly goes around virtue signaling, but when push comes to shove, he isn't a true ally to the movement. i think it's pretty telling - that masked comes from an upper middle class, masked family. he's from the masked masked masked masked. his parents own an entire floor there.\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    }
  ]
}