NLP_Primer_twitter_challenge.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP Primer twitter challenge.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "7bfc92373971404c82c893f6110ee02f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_06b1e0e6273044e7923923a9835f0bdb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8af370330541427db3bab98008001207",
              "IPY_MODEL_8dfd25a5820747559299ec4ab8ec7e48"
            ]
          }
        },
        "06b1e0e6273044e7923923a9835f0bdb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8af370330541427db3bab98008001207": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6801510baeef42189b06c3190c88b22e",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 8999,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 8999,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9eb71605fb1a46c987e830a6373f8931"
          }
        },
        "8dfd25a5820747559299ec4ab8ec7e48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1937a47f310c4e92a6c2cd931fe6dc9c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8999/8999 [04:33&lt;00:00, 32.85it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a6bf9b1468104506a9153cdb5d043a22"
          }
        },
        "6801510baeef42189b06c3190c88b22e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9eb71605fb1a46c987e830a6373f8931": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1937a47f310c4e92a6c2cd931fe6dc9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a6bf9b1468104506a9153cdb5d043a22": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d7e804c6097492dae849e9f6f17dfea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_928db950860f4d7eb1244976e07340ec",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c21e200a0eaa42559ecd60c9f43c6c41",
              "IPY_MODEL_a98245ca84b948ec9d93a1f56d2d93cc"
            ]
          }
        },
        "928db950860f4d7eb1244976e07340ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c21e200a0eaa42559ecd60c9f43c6c41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2e817807da2e4d93ac8dc05ad4481c81",
            "_dom_classes": [],
            "description": "Epoch 3 of 3: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_60e82ae75dff4eec978ef8821fa7ed59"
          }
        },
        "a98245ca84b948ec9d93a1f56d2d93cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a402f09ca15f48478ef3ebef9d8deede",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [04:29&lt;00:00, 89.83s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_84b959b97c494e119eb44dd729d494fc"
          }
        },
        "2e817807da2e4d93ac8dc05ad4481c81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "60e82ae75dff4eec978ef8821fa7ed59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a402f09ca15f48478ef3ebef9d8deede": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "84b959b97c494e119eb44dd729d494fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a599f37980174e1fac1cf52979c245c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_99d0a0cb7263435f950f396778bcd7dc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7156685b440f4b9e8ebf08b7aadbd833",
              "IPY_MODEL_95ca19de8c9544ccb60664f695466c9f"
            ]
          }
        },
        "99d0a0cb7263435f950f396778bcd7dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7156685b440f4b9e8ebf08b7aadbd833": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_eee873822e4940a0b57f10622c31436a",
            "_dom_classes": [],
            "description": "Epochs 0/3. Running Loss:    0.8864: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1125,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1125,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e9d48e11bae24b0185dab499900bcd0d"
          }
        },
        "95ca19de8c9544ccb60664f695466c9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_60ad1420aa434290aba6b1b1666cedce",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1125/1125 [04:29&lt;00:00,  4.17it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bba22b3ab5304851aca8fc6e369de611"
          }
        },
        "eee873822e4940a0b57f10622c31436a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e9d48e11bae24b0185dab499900bcd0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "60ad1420aa434290aba6b1b1666cedce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bba22b3ab5304851aca8fc6e369de611": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "89d634f4c697419192bbca8ba1c7a00c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a7597d4690c247df8d405c5ac949d5da",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bfced04a7c6c4192a03d77d4fa455b57",
              "IPY_MODEL_38602ece6def4541ae784224f7502e2c"
            ]
          }
        },
        "a7597d4690c247df8d405c5ac949d5da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bfced04a7c6c4192a03d77d4fa455b57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_88ab8cf638c3415baba06d3bdbf404df",
            "_dom_classes": [],
            "description": "Epochs 1/3. Running Loss:    0.7346: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1125,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1125,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_347846c981364e81977e8e8507c07833"
          }
        },
        "38602ece6def4541ae784224f7502e2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_854581df260344b1bb8ccc05586e9882",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1125/1125 [03:02&lt;00:00,  6.18it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7c6ce8c73c4b4d23bebc46f9afb55816"
          }
        },
        "88ab8cf638c3415baba06d3bdbf404df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "347846c981364e81977e8e8507c07833": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "854581df260344b1bb8ccc05586e9882": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7c6ce8c73c4b4d23bebc46f9afb55816": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4dbbdf27eaff473e8e2b51e6325ea40f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8f32c914655242f5ba8d36a163dc87a7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ce8f2d8a8b67463d85993ca0aece6baa",
              "IPY_MODEL_ac446200ebee4256b013faaeb2b59fa9"
            ]
          }
        },
        "8f32c914655242f5ba8d36a163dc87a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce8f2d8a8b67463d85993ca0aece6baa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b31517bc0bc247e7ac8a42e6f6a5381d",
            "_dom_classes": [],
            "description": "Epochs 2/3. Running Loss:    0.0219: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1125,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1125,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ff4132f18677435289b13e2db78bf44e"
          }
        },
        "ac446200ebee4256b013faaeb2b59fa9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e38110db81274fc1a6b3513c8e3ff095",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1125/1125 [01:33&lt;00:00, 12.07it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_757dcc97d2404e39a3bcb258ea2f9277"
          }
        },
        "b31517bc0bc247e7ac8a42e6f6a5381d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ff4132f18677435289b13e2db78bf44e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e38110db81274fc1a6b3513c8e3ff095": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "757dcc97d2404e39a3bcb258ea2f9277": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8f26cf7b54af474b90feb0a033227bb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b3d74d636c5446e48a996388040ff428",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fd6de9c69886427ba37c9b1f1b248efa",
              "IPY_MODEL_8e3a8aa1a6cd4f1da417723697f17ba5"
            ]
          }
        },
        "b3d74d636c5446e48a996388040ff428": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd6de9c69886427ba37c9b1f1b248efa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_021cba019ed643a0918f06f4e4b05070",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a159cc22f15c47a38754873ee0e4d9ca"
          }
        },
        "8e3a8aa1a6cd4f1da417723697f17ba5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a2669d9b06504677900f062e1874f51b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1000/1000 [00:05&lt;00:00, 171.15it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6b0728e3a5474e859ae4b9f58f0c5f0c"
          }
        },
        "021cba019ed643a0918f06f4e4b05070": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a159cc22f15c47a38754873ee0e4d9ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a2669d9b06504677900f062e1874f51b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6b0728e3a5474e859ae4b9f58f0c5f0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0749fef36bab4a8c8a2274874ee7d2e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3846517229694c89834eacbf7dc611cd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7f3e59b13f2b45e2b758cddc2fa5fd96",
              "IPY_MODEL_4e825c21b2e44b6399fa7b4f77b02b96"
            ]
          }
        },
        "3846517229694c89834eacbf7dc611cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7f3e59b13f2b45e2b758cddc2fa5fd96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_280479ca5c4d4ff79f7bc50f82b022f4",
            "_dom_classes": [],
            "description": "Running Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 125,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 125,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_68f56100fd4f4ac6ac094af47a5e6799"
          }
        },
        "4e825c21b2e44b6399fa7b4f77b02b96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b8c2d92d57254af7af30d0e5ec7797c2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 125/125 [00:04&lt;00:00, 29.82it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9025128f32f049b39ee238049971475e"
          }
        },
        "280479ca5c4d4ff79f7bc50f82b022f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "68f56100fd4f4ac6ac094af47a5e6799": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b8c2d92d57254af7af30d0e5ec7797c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9025128f32f049b39ee238049971475e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b9a456bb440a47d580bb57dde7e2a671": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5087a1d42d5c4c4188014be3b3aaa822",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8dff72db6a3a48ae811ae8c714070041",
              "IPY_MODEL_e1fea716e5da4cf2861ccd1179a3bd77"
            ]
          }
        },
        "5087a1d42d5c4c4188014be3b3aaa822": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8dff72db6a3a48ae811ae8c714070041": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_11281e73b4ee4822b748ef9f0484342a",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5177,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5177,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7ef816fb26e44cd9832422a6afcdfdaf"
          }
        },
        "e1fea716e5da4cf2861ccd1179a3bd77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b34f58920f74497e94a6f93f2c067a7f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5177/5177 [00:01&lt;00:00, 2811.77it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5983754c3eff48979d40be82d165a507"
          }
        },
        "11281e73b4ee4822b748ef9f0484342a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7ef816fb26e44cd9832422a6afcdfdaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b34f58920f74497e94a6f93f2c067a7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5983754c3eff48979d40be82d165a507": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c556108d7f654a6498b358f733775483": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7e0d65d4f98741519a03ce63cfc91a77",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_56c41d26a1754a17b9757269658015cb",
              "IPY_MODEL_de36c2e593e24abba3374ae79df1b872"
            ]
          }
        },
        "7e0d65d4f98741519a03ce63cfc91a77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "56c41d26a1754a17b9757269658015cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0bf46d6513a5495496042e39e27dfd51",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 648,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 648,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fd13451cc9054116886d995daeb3e571"
          }
        },
        "de36c2e593e24abba3374ae79df1b872": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d15cc24c4be9496381f9125313d8b5c2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 648/648 [00:11&lt;00:00, 57.16it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7c76f6d506c44cedbc191048bb97cda5"
          }
        },
        "0bf46d6513a5495496042e39e27dfd51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fd13451cc9054116886d995daeb3e571": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d15cc24c4be9496381f9125313d8b5c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7c76f6d506c44cedbc191048bb97cda5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IucN-YUVlwQu",
        "colab_type": "text"
      },
      "source": [
        "# Introduction\n",
        "\n",
        "When the '#ZindiWeekendz Learning: To Vaccinate or Not to Vaccinate: It’s not a Question' originally ran as a hackathon, someone linked a notebook on Kaggle as a getting started resource: https://www.kaggle.com/abhishek/approaching-almost-any-nlp-problem-on-kaggle. It's some good info, but reading that and trying to place myself in the shoes of a beginner I felt like it was a) too much to take in and b) not even the best approach given current tools. So, I started a new discussion titled 'The Lazy NLP Route'. Here's what I said:\n",
        "\n",
        "*Thanks @overfitting_PLB for sharing the kaggle getting started with NLP notebook, but wow is that a lot of code. TF-IDF, word vectors, custom models, cross-validation, ensembles of different models.... I thought I'd share an alternate view.*\n",
        "\n",
        "*Deep learning has pretty much taken over NLP. Language models like those available through fastai or huggingface are able to capture nuances of text, and can be trained with very little effort. They handle the tokenization etc, and I find them super easy to use.*\n",
        "\n",
        "*I tried two different approaches, each ~10 lines of code, training time under 15 minutes. Both ~0.6 scores. Both have PLENTY of room for improvement since I did almost no optimising. I'm not going to share code for this one (maybe in the future) but here are some places to get started:*\n",
        "\n",
        "*1) Fastai text. The docs are decent: https://docs.fast.ai/text.html. I didn't do any language model tuning (there's a place to look for improvements!) but went straight to training a `text_classifier_learner(data_clas,AWD_LSTM,drop_mult=0.3, metrics=[rmse])` - give it a validation set and you get RMSE (like the Zindi score) as it trains!*\n",
        "\n",
        "*2) Huggingface transformers via the simpletransformers library. The github has docs including a regression example: https://github.com/ThilinaRajapakse/simpletransformers#minimal-start-for-regression. Hugginface do amazing work, but if you look for tutorials many of them have lots of code to copy and paste - I like the simpletransformers library as it simplifies a lot of that and gets out of the way. You specify some parameters, pick a model architecture (I chose DistilBERT) and basically hit go :)*\n",
        "\n",
        "*The reason I ran these models and am sharing this: a lot of smart people have tried very hard to make it easy to solve new challenges in the field of NLP. But there are so many options, and it's hard to know where to start. These are two ideas for you to research and play with. They're not hobbled beginner methods, they're the real deal. And it's possible to make good predictions with them. They've given me good results in the workplace and my hobby projects. So if you're not sure where to start, pick one and dig in, and see if you can get it working. You'll be playing with the cutting edge of NLP research, and hopefully, it'll let you get up there on the 'board without needing a masters degree in ML :) Good luck!*\n",
        "\n",
        "*PS: Disagree, and think you should start from the basics and work up? Let's chat! I'm hoping this will spark some interesting discussion about SOTA in NLP, how to learn, using first vs bottom up... Drop your view in the discussion here :)*\n",
        "\n",
        "So, now that this is open as a knowledge competition, I figured it's time to share the actual code! The winners blog and code repositories show that transformers won the day - score one for fancy new tools :) Let's dive in and see how we can use them ourselves.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_O6s11nLnVEv",
        "colab_type": "text"
      },
      "source": [
        "# 1) Quick LSTM with fastai\n",
        "\n",
        "Here's a minimal solution with fastai, using the AWD_LSTM language model to solve this task. TO run this, make sure you've uploaded the csv files from Zindi into Colab using the files pane on the left."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iQ-8b6eLnrYk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np \n",
        "import pandas as pd \n",
        "from pathlib import Path\n",
        "from fastai.text import *"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ymf7qs79nvQZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "outputId": "ed32dc22-f8c3-4fac-fd00-9d805042a900"
      },
      "source": [
        "# Load the data\n",
        "train = pd.read_csv('Train.csv').dropna(0) # Read in train, ignoring one row with missing data\n",
        "test = pd.read_csv('Test.csv').fillna('') # Read in test\n",
        "test['label']=0 # We'll fill this in with predictions later\n",
        "train.head(3) # Take a peek at the data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>safe_text</th>\n",
              "      <th>label</th>\n",
              "      <th>agreement</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CL1KWCMY</td>\n",
              "      <td>Me &amp;amp; The Big Homie meanboy3000 #MEANBOY #M...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>E3303EME</td>\n",
              "      <td>I'm 100% thinking of devoting my career to pro...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>M4IVFSMS</td>\n",
              "      <td>#whatcausesautism VACCINES, DO NOT VACCINATE Y...</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   tweet_id  ... agreement\n",
              "0  CL1KWCMY  ...       1.0\n",
              "1  E3303EME  ...       1.0\n",
              "2  M4IVFSMS  ...       1.0\n",
              "\n",
              "[3 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y1TlPEUmoSgE",
        "colab_type": "text"
      },
      "source": [
        "Fastai uses something called a databunch to store the data. The docs show how to create one. Here, we add our test data with test_df=test, and split our training data into df_train and df_valid (to let us see scores on a validation set while it trains)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8JEEdb_oL7s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "bfe885b1-1b16-492d-f35c-077754b6393c"
      },
      "source": [
        "# Build the databunch, and keep 1000 rows for validation\n",
        "df_valid = train.sample(1000)\n",
        "df_train = train.loc[~train.tweet_id.isin(df_valid.tweet_id.values)]\n",
        "print(df_valid.shape, df_train.shape)\n",
        "data_clas=TextClasDataBunch.from_df(path=Path(''),train_df=df_train, \n",
        "                                    valid_df=df_valid,\n",
        "                                    test_df=test,\n",
        "                                    label_cols='label',\n",
        "                                    text_cols='safe_text')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1000, 4) (8999, 4)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iiFWYKWvonJA",
        "colab_type": "text"
      },
      "source": [
        "Now we have the data ready, we can create a model to train:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dE5u8ARMoqxK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "206b375c-9a2a-4166-edd1-d47c96609ed4"
      },
      "source": [
        "# Learner\n",
        "clas = text_classifier_learner(data_clas,AWD_LSTM,drop_mult=0.3, metrics=[rmse])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://s3.amazonaws.com/fast-ai-modelzoo/wt103-fwd.tgz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X_3s-JopoxiU",
        "colab_type": "text"
      },
      "source": [
        "There are things we could do t pick learning rates etc, but this is a minimal example. Let's train our model! I'm running it for 20 epochs as a fairly arbitrary choice."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u10h2Ol4o7R5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        },
        "outputId": "9245b00d-b5cc-45d3-c09c-07f02cada964"
      },
      "source": [
        "clas.fit_one_cycle(20)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>root_mean_squared_error</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.564282</td>\n",
              "      <td>0.465141</td>\n",
              "      <td>0.679848</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.462321</td>\n",
              "      <td>0.436267</td>\n",
              "      <td>0.657814</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.432291</td>\n",
              "      <td>0.427791</td>\n",
              "      <td>0.651200</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.421621</td>\n",
              "      <td>0.427525</td>\n",
              "      <td>0.651112</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.424048</td>\n",
              "      <td>0.436113</td>\n",
              "      <td>0.657987</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.424395</td>\n",
              "      <td>0.432718</td>\n",
              "      <td>0.655117</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.423630</td>\n",
              "      <td>0.429524</td>\n",
              "      <td>0.652540</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.421932</td>\n",
              "      <td>0.430618</td>\n",
              "      <td>0.653448</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.417334</td>\n",
              "      <td>0.428009</td>\n",
              "      <td>0.651377</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.420241</td>\n",
              "      <td>0.431765</td>\n",
              "      <td>0.654545</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.413076</td>\n",
              "      <td>0.425135</td>\n",
              "      <td>0.649148</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.410114</td>\n",
              "      <td>0.426605</td>\n",
              "      <td>0.650154</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.408655</td>\n",
              "      <td>0.424962</td>\n",
              "      <td>0.649133</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.413460</td>\n",
              "      <td>0.421472</td>\n",
              "      <td>0.646095</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.408621</td>\n",
              "      <td>0.421888</td>\n",
              "      <td>0.646548</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.411027</td>\n",
              "      <td>0.421755</td>\n",
              "      <td>0.646552</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.408274</td>\n",
              "      <td>0.419568</td>\n",
              "      <td>0.644795</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.411849</td>\n",
              "      <td>0.419302</td>\n",
              "      <td>0.644528</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.412453</td>\n",
              "      <td>0.420188</td>\n",
              "      <td>0.645280</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.416901</td>\n",
              "      <td>0.419886</td>\n",
              "      <td>0.644907</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IhVVkm3ZpDN9",
        "colab_type": "text"
      },
      "source": [
        "You can see the RMSE decrease as it trains. This is the metric used in the competition, and a score of ~0.6 is pretty good looking at the leaderboard.We'll do better than 0.64 later, but for now let's save predictions:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qxN-Vo09pAVA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "f0d3ac79-f09d-4458-b564-c87c156f5493"
      },
      "source": [
        "# Get predictions\n",
        "preds, y = clas.get_preds(DatasetType.Test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YqJG0szrpa_B",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "e2ad4b11-c685-4694-af30-090370d8c2bb"
      },
      "source": [
        "# Make a submission dataframe\n",
        "sub = pd.DataFrame({\n",
        "    'tweet_id':test['tweet_id'],\n",
        "    'label':[p[0] for p in preds.numpy()]\n",
        "})\n",
        "sub.to_csv('first_try_fastai_20_epochs.csv', index=False)\n",
        "sub.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>00BHHHP1</td>\n",
              "      <td>0.249226</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>00UNMD0E</td>\n",
              "      <td>0.271966</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>01AXPTJF</td>\n",
              "      <td>0.288489</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>01HOEQJW</td>\n",
              "      <td>0.336241</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>01JUKMAO</td>\n",
              "      <td>0.250573</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   tweet_id     label\n",
              "0  00BHHHP1  0.249226\n",
              "1  00UNMD0E  0.271966\n",
              "2  01AXPTJF  0.288489\n",
              "3  01HOEQJW  0.336241\n",
              "4  01JUKMAO  0.250573"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lpg0QEZdpjIM",
        "colab_type": "text"
      },
      "source": [
        "This scores 0.64 on the LB. Not bad, but we'll keep on improving. Bu tthis isn't bad for such a quick starting point!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qNwtv8P6qDTF",
        "colab_type": "text"
      },
      "source": [
        "# 1.2 - Fastai with some better tuning\n",
        "\n",
        "Building on the previous example, let's now follow the steps as taught in the fastai course for text. First, we'll re-train our language model on our data, then we'll train a classifier."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liZkhfN3orq6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 236
        },
        "outputId": "3e808499-fff7-4fd7-d784-d605516ab4e3"
      },
      "source": [
        "# Creating a databunch for the language model\n",
        "data_lm = TextLMDataBunch.from_df(path='', train_df=df_train, \n",
        "                                    valid_df=df_valid,\n",
        "                                    text_cols='safe_text')\n",
        "# And the learner\n",
        "learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.5)\n",
        "\n",
        "# Some very quick training - could do much more here\n",
        "learn.fit_one_cycle(2, 1e-2)\n",
        "learn.unfreeze()\n",
        "learn.fit_one_cycle(3, 1e-3)\n",
        "\n",
        "# We save the encoder for later use\n",
        "learn.save_encoder('ft_enc')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>4.748188</td>\n",
              "      <td>3.706317</td>\n",
              "      <td>0.330357</td>\n",
              "      <td>00:05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.950920</td>\n",
              "      <td>3.573325</td>\n",
              "      <td>0.346726</td>\n",
              "      <td>00:05</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.413620</td>\n",
              "      <td>3.338416</td>\n",
              "      <td>0.380729</td>\n",
              "      <td>00:07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.228122</td>\n",
              "      <td>3.260442</td>\n",
              "      <td>0.394122</td>\n",
              "      <td>00:07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.051455</td>\n",
              "      <td>3.243684</td>\n",
              "      <td>0.394494</td>\n",
              "      <td>00:07</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CAZ75KBSqmF8",
        "colab_type": "text"
      },
      "source": [
        "Now we have a language model trained on tweets, we can use the encoder as part of our text classifier:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fpChSjN0qgub",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        },
        "outputId": "615e750b-1018-45fe-b5cf-9e70e8575e96"
      },
      "source": [
        "# Creating the learner\n",
        "learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5, metrics=[rmse])\n",
        "# Loading the encoder we just saved\n",
        "learn.load_encoder('ft_enc')\n",
        "# Using lr_find to pick a learning rate:\n",
        "learn.lr_find()\n",
        "learn.recorder.plot(suggestion=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/1 00:00<00:00]\n",
              "    </div>\n",
              "    \n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>root_mean_squared_error</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>\n",
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='83' class='' max='140' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      59.29% [83/140 00:02<00:01 0.6125]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
            "Min numerical gradient: 6.31E-07\n",
            "Min loss divided by 10: 3.02E-02\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEGCAYAAABYV4NmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUZfbA8e9JJyGhhgBJgAChVwlIsSAqokuzIdjLWldd665uYV3d4q7154quZdG1LSoooqJYQFCK9CK9Q6ihJ4H08/tjbnAICUnI3MxMcj7PMw8zd947c14myZn33veeV1QVY4wxxtdC/B2AMcaYmskSjDHGGFdYgjHGGOMKSzDGGGNcYQnGGGOMK8L8HYCvNG7cWFu1auXvMIwxJqgsWrRon6rGu/HaNSbBtGrVioULF/o7DGOMCSoistWt17ZDZMYYY1xhCcYYY4wrLMEYY4xxhSUYY4wxrrAEY4wxxhWWYIwxxrjCEowxxhhXuJpgRGSIiKwVkQ0i8kgpzz8nIkud2zoROVTi+TgRSReRF92MMxh8+dMuvvxpF8fyCv0dijHGVIhrF1qKSCgwDrgQSAcWiMgUVV1V3EZV7/dqfw/Qs8TLPAHMcivGYHEgO4+73l1MkUKd8FDO6xDPkC7NGNwpgajwUH+HZ4wxpXJzBNMH2KCqm1Q1D5gAjDhF+zHA/4ofiEgvIAH4ysUYg8LMdXspUnh8RGcu75XIgi0Hufd/S7jt7UXYgnHGmEDlZoJJBLZ7PU53tp1ERFoCKcB053EI8AzwkIvxBY1vVu8lPjaSa89syV9GdmXeo+fz8EXtmbUug69X7fF3eMYYU6pAOck/GpioqsUnGO4Cpqpq+ql2EpHbRGShiCzMyMhwPUh/yC8sYtbaDAa1b0JIiAAQGiLcdk5rUpvU5S+frya3wM7LGGMCj5sJZgeQ7PU4ydlWmtF4HR4D+gF3i8gW4GngehF5suROqvqqqqapalp8vCvFQP1uwZYDZOYWMKhjkxO2h4eGMHZYJ7YdOMr4H7b49D0PZOfx2JSVtP/DF1z7+o/MWLOXoiI7FGeMqRw3qykvAFJFJAVPYhkNXF2ykYh0ABoAc4u3qeo1Xs/fCKSp6kmz0GqD6av3EhEawlltG5/03Nmp8VzQMYEXp6/n8jMSaRIXVaX3yskvZPzszbw8YyPZeQUM6dKURVsPctObC2gdH8NNA1K4sleSTSwwxlSIayMYVS0A7gamAauBD1R1pYg8LiLDvZqOBiaona0u1fQ1e+nXphExkaV/F/jDLzqSV1jEP6etrdL7bD9wlPOfmck/v1xLn5SGTLvvHF66phc//HYQ/ze6B3Ujw/jj5J8Y89o8MjJzq/RexpjaQWrK3/W0tDStaevBbMrIYtAzM3l8RGeu79eqzHZ//2I1r8zcxCe/GkD35Pqn9V4PvL+UqT/tYvyNvenf5uTRkqoydcVuHvxwKY1iIvnPjWl0aBp3Wu9ljAkcIrJIVdPceO1AOclvSjF9zV4Azmvf5JTt7hmUSnxsJGM/+YmCwqJKv8+mjCwmL93BdX1blppcAESEX3Rrxoe39ye/sIjLX5rD9DU2g80YUzZLMAHs29V7aZ8QS3LD6FO2qxsZxp+GdWJZ+mFembWp0u/z4owNRISFcNs5bcpt2zWpHp/cPYBWjWP45X8X8snSsuZtGGNqO0swAerwsXwWbDlw0uyxsgzt1pyh3Zrx/DfrWLXzSIXfZ/O+bCYv2cG1Z7YkPjayQvs0q1eHD+/oR/fk+jz+6Soyc/Ir/H7GmNrDEkyA+n59BgVFygUVTDAAT4zoQr06ETzwwVLyCip2qOzF6RsIDw3htnNbVyq+6IgwHhvWmf3Zebwys/KjJmNMzWcJJkBNX72XBtHh9EhuUOF9GsRE8ORlXVmzO5P/+3Zdue237Mtm8tIdXNu3JU1iKz/FuXtyfYZ3b87rP2xi1+Fjld7fGFOzWYIJQAWFRcxYu5fz2jch1Ll6v6Iu6JTAlb2SePm7jSzZdvCUbV+csYGwEOH2So5evD18UXuKiuCZr8pPaMaY2sUSTACasTaDg0fzuahL09Pa/4/DOtE0Lop7/reEbfuPltpm/Z5MPl6yg2vOPL3RS7HkhtHcOKAVkxanV+rcjzGm5rMEE4De/XErCXGRnN+h4udfvMVFhfPytb3Iyi3g8n/PYfWuE//wf78+gytfmUvdyDDuqMLopdivBrYlLiqcv3+xusqvZYypOSzBBJjtB44yc10Go3u3ICz09D+e7sn1+fD2foSFCKNemcuPm/ajqvx75kZuGD+fhNgoPvnVgCqXlwGoFx3Oveen8v36fcxcVzOLjhpjKs8STIB5b/42QkQY3Se5/MblSE2IZeKd/WkSG8l14+dz3X/m8+QXa7i4azM+uqs/rRrH+CBij+v6tqRlo2j++vmq07rYM1Coqk27NsZHLMEEkNyCQj5YsJ3zOzShWb06PnnNxPp1+PCO/nRsFsecjft49OIOvDimZ5m1zU5XRFgIv7ukI+v2ZPHuj9t8+trV5dDRPK77z3z6/306+7Os3poxVeVmNWVTSdNW7mF/dh7X9m3p09dtGBPBB7f3Zc/hXFo0OnVVgKoY3CmBAW0b8ezX6xjevTkNYiJcey9fW7s7k1vfWsiuw8fIL1Q+XrKDX55d9fNTxtRmNoIJIO/M20rLRtGlluavqsiwUFeTC3jqlY0d2pnMnHye+yZ4pi1/+dNuLn1pNsfyC5lwWz96tqjP+wu223LUxlSRJZgAsX5PJvM3H+DqPi2Or1wZjNo3jeXavi15Z95W1uwO/GnLb8/byh3vLCI1IZZP7z6LXi0bMLp3Muv3ZrF42yF/h2dMULMEEyDe/XEbEaEhXNEryd+hVNkDF7Yjrk44f56yKqBHASvSD/P4pysZ1KEJ79/Wl6b1PDPqhnZrTkxEKO8vCM5zScYECkswAeBYXiGTFqdzSdemNKpbsYKTgax+dAQPXNiOuZv2M23lbn+HU6qs3ALu+d9i4utG8uyo7ies0hkTGcaw7s35dNkum1FmTBVYggkAX6/eQ2ZOAaP7tPB3KD5zdZ8WtEuoyz+/XEtRkX9GMarKuBkbeHH6enLyC094buzkn9h24CjPj+5J/eiTJyOM6p3MsfxCPlu+q7rCNabGsQQTAD5fvpOEuEj6tGro71B8Jiw0hF+d15ZN+7KZsXavX2IYP3sLT01by9NfrWPI87OY5VwE+tHidD5asoN7z0+lT0rp/+c9k+vTLqEu7y/YXp0hG1OjWILxs8ycfGaszeCSrs2C+uR+aS7p2oxm9aJ4/fvN1f7e36zaw18+X8WQzk15+5Y+iAjXj5/PHW8v4o+Tf6JPSkPuGZRa5v4iwlW9W7B0+6GgmKxgTCCyBONn36zeQ15BEUO7Nfd3KD4XHhrCDf1bMXfTflbuPFxt7/vTjsPcO2EJXRPr8dxVPTg7NZ4vfn0291/Qjulr9xIWGsLzV/Uot1L1pT0TiQgNsVGMMafJ1QQjIkNEZK2IbBCRR0p5/jkRWerc1onIIWd7DxGZKyIrRWS5iFzlZpz+9NmyXTSvF0XP5Pr+DsUVY3q3oE54KP/5oXpGMbsP5/DL/y6kfp1wXr8+jToRnpP3UeGh/PqCVKY/eC5T7h5A8/rlV0poGBPB4M4JTFqUzovT1/PJ0h0s3naQg9l5bnfDmAr7ZOkOPlgYmF+CXLuSX0RCgXHAhUA6sEBEpqjqquI2qnq/V/t7gJ7Ow6PA9aq6XkSaA4tEZJqq1qgLEw4fy2fW+gxu7N+qxh0eK1YvOpxRaUm8N38bjwzpUOXimit3HuZgdj75RUUUFip5hUXsOpzD1v3ZbN1/lJU7j3Asr8BTg62U90pqULmLTe84tw1Lth3iaa/1bsJChJev7cWFnRKq1BdjfOGDhdvJyS9iVFrV6xf6mpulYvoAG1R1E4CITABGAKvKaD8G+BOAqh7/bVbVnSKyF4gHalSC+WrlbvILtUYeHvN204AU3pq3lbfmbuWhi9qX2S6/sIhl2w/RI7n+SZWkD2TnMfaTn8qc1RUbGUaLRtH0SWnAjf1T6Ngsziexd0msx+xHBnE0r4D0g8fYfuAoz369jocnLuPLX59z/NoZY/wlK6eAeqXMhAwEbiaYRMB73JYOnFlaQxFpCaQA00t5rg8QAWws5bnbgNsAWrQIvim+ny3fRXLDOnRLqufvUFzVqnEMF3RM4N0ft/Kr89oeP2xV0l8/X82bc7aQ1KAONw1I4areydSNDGPayt38/uMVHD6Wz/0XtKNfm0aEhQrhISGEhQoJcVE0iA5HxL1RYHREGO0SYmmXEEurxjEMfeEHHvxwKW/ffGaNHX2a4JCVW0BiA98Ux/W1QCl2ORqYqKonXKwgIs2At4EbVPWkGvCq+irwKkBaWlrgXjJeioPZeczesI9fnt3a1T+MgeKWs1L4etUePlqSzjVnnlzMc96m/bw5ZwuDOyVw6Gg+T3y2iue/Xke35HrM3rCfTs3iePuWM302MqmKNvF1+dOwTjzy0Qpe+34Tt5/bxt8hmVosO7eQmIhA+VN+Ijej2gF4HxRMcraVZjTwK+8NIhIHfA78XlXnuRKhH01buZuCImVot2b+DqVanJnSkC6Jcbw0YyPntW9ywkn27NwCHp64jFaNonl+dA+iI8JYtv0Qr/+wme/XZ3Dv+ancfV5bIsICZ9LjVb2Tmbkug6emraVfm0Z0S6qZkzRM4MvOLaBuVGAmGDd/YxcAqSKSIiIReJLIlJKNRKQD0ACY67UtAvgYeEtVJ7oYo998tnwXrRpF07m5/7+RVwcR4c/Du3DkWD5XvDyHDXuzjj/35BdrSD94jKeu7E60802se3J9/jWmJ0vHDuaBC9sFVHIBT3/+fllX4mMj+fWEpRyxkjLGD1SVrLwC6vp4fSdfce23VlULgLuBacBq4ANVXSkij4vIcK+mo4EJemJVxFHAOcCNXtOYe/g8yI0b4a67IC4OQkI8/951l2e7i/Zl5TJn4z6GdmteKw6PFevVsgETbu9LXqFy5b/nsGTbQWZv2Mfb87Zyy4AUegdZJYP60RE8O6oHW/dnc/4zM5m4KP2ksjgHs/N4f8E2ZqzdW2bJnN2Hc9hzJKc6QjY1zNG8QlTx+QKCviKBXO22MtLS0nThwoUV3+GLL+CKKyA/33MrFh7uuU2cCBdf7PtAgd9MXMaHi9L56r5zSE2IdeU9AtnW/dlc95/5ZGTmEhsVRt3IMKb++uwTCk4GkyXbDvLnT1exdPshuifV4w9DO3HkWD4TF6Xzzeo95Bd6fseSG9bh2jNbMiotmdBQ4csVu/l4yQ7mbd5P47qRfPvgucRFhfu5NyaY7D2SQ5+/fcsTI7tw3WkuVCgii1Q1zceheV67ViaYjRuhWzc4erTsNtHRsHw5tPHtCdwvVuzizncX86vz2vDwRR18+trBZO+RHG54YwFrdx/hwzv606tlA3+HVCVFRcrkpTv4x5dr2HPEs9xyo5gIRvZM5NKeiWzZn81bc7cyf/OB44f78gqKSGkcw3ntm/DGnM3c1D+FscM6+bMbJshsyshi0DMzef6qHozsmXhar+FmggnMcZXbnnnmxFFLafLz4bnn4MUXT3rqnXlb2ZeVy50D2xAZVvFv3bsOH+ORj1bQLake913QrrJR1yhN4qKYdGc/dh46RtsmwT+KCwkRLjsjiYs6N2XS4nSa1avDwPbxhDvX83RJrMfQbs1ZuzuT/833rDMzsmci3ZPqISLkFBTy37lbGNU7iQ5Na8d5OVN1WbkFgB0ic12lRjBxcZCZWbF2h0+soTVn4z6uef1HVKF9QizPXtWdzs3Lv46lqEi5bvyPLN56iM/vPYvW8XUrFqupFQ5m5zHome9IbRLL+7f3rVXn5szpm7NxH1e/9iPv3Xom/duc3lLrbo5gAmtqTnXJyiq/TSntDmbn8cD7y0hpHMO/rz2Dg0fzGDluNuNmbKCg8KTLdE7wnx82M3vDfsYO62TJxZykQUwEvx3SgflbDjB5aVmz+Y05UXau59LBWjeLLKDVreAfeK92qsqjH61gf3YuL4zuyZAuzZh23zkM7tyUp6at5eb/LixzeeDVu47w1LS1DO6UwOjegVcvyASGUWnJdE+uz9+mrrFpz6ZCsgP8EFntTDDXXuuZKXYq4eFw3XXHH76/YDtfrtzNQ4Pb0yXRc0isQUwE464+gz/8oiOz1mUwZdnOk15GVfnj5J+IjQrjycu72aEPU6aQEOGJEZ3Zl5XL09PWlvmFxZhimU6CibUEE0AefLBiCeZ+T7HnjRlZ/PnTVQxo24hbz259UtObB6TQuXkc//hiDcfyTlyad/LSHSzcepDfDGlPw5jALEhnAke3pPpc37clb83dyh3vLGJfVq6/QzIBzEYwgahNG891LtHRJyUaDQ/3bJ84Edq0obBIeeD9pUSGh/DMlT1KLWwYEiL8cWgndh7O4bXvNx3fnpVbwN+nrqFbUj2u7GWHxkzFjB3Wmd9d0oEZazMY/NwsvlhRegVpY7JzCxCB6DIKyPpb7Uww4LmIcvlyuO02iItDRciMiGb5xaM8252LLN+Zt5Vl6Yf58/DOpyzN3rd1Iy7u0pSXv9t4/Krsf01fz97MXP48vLNV3DUVFhoi3HZOGz6/5yySGtThzncX8+AHy8qsBGBqr6zcAmIiwgL20HvtTTDgGcm8+CIcPowUFXHLC9/wwDm/RFt7DoPtOZLDU9PWcnZqY4Z3L3/Nlkcv7khhkfLPL9eyMSOL8T9s5opeSfRsEdwXERr/SE2IZdKd/bn9nNZMWpzOrPUZ/g7JBJjs3MCtQwa1PcGUMKJHczZmZLNq1xEAHv9sFXmFRTwxokuFviG0aBTNzWelMGlxOne/t4SosFB+O6T2Xq1vqi48NIQHB7enSWwkb8ze4u9wTIDJyi0gJjIwD4+BJZgTXNKlGWEhwpSlO/lu7V4+X76Le85rS6vGMRV+jV+d14bGdSNYvesIv74glfjYSBcjNrVBRFgI1/Vtycx1GSdUoTYmK7fQRjDBokFMBOe2i2fKsp2M/WQlreNjuO3ck2eNnUpsVDj/uLwbl/ZM5Ib+rdwJ1NQ6V5/ZgoiwEN6cs9nfoZgAkp1bELAzyMASzEmG92jOrsM5bDtwlL+M7FKpWmPFzu+YwHNX9Theh8qYqmpUN5KRPZozadEODh3N83c4JkDYOZggc2GnBOKiwriiV9Jp1/Yxxg03DUjhWH4hExZs93coJkBk5gR2ggncyPwkOiKMbx8cSP1oW5fDBJaOzeLo17oRb83Zwi/PSiHMRsi1XnaeHSILOvGxkXZ4ywSkm89KYefhHKat3OPvUEwAsHMwxhifGdShCS0aRvPGbDvZX9vlFhSSX6jERlmCMcb4QGiIcGP/VizcepBXZ230dzjGj7JynDpkAVomBuwcjDFB59q+LVm09SB/m7qGw8fyeWhw+4AtFWLcU7wWTK09RCYiQ0RkrYhsEJFHSnn+ORFZ6tzWicghr+duEJH1zu0GN+M0JphEhIXwwpiejO6dzLgZGxn7yUqrU1YLFS+XXCtnkYlIKDAOuBBIBxaIyBRVXVXcRlXv92p/D9DTud8Q+BOQBiiwyNn3oFvxGhNMQkOEv1/WlXp1wnll1iaO5OTz9JXdbXJKLZKd5ySYWnoOpg+wQVU3qWoeMAEYcYr2Y4D/OfcvAr5W1QNOUvkaGOJirMYEHRHh0Us68psh7flk6U7uencxuQWF5e9oaoTj52ACeATjZoJJBLyvCEt3tp1ERFoCKcD0yuwrIreJyEIRWZiRYZVmTe1018C2/Hl4Z75etYfb315ETr4lmdogGA6RBcp4ejQwUVUr9Zuhqq+qapqqpsXHx7sUmjGB74b+rXjysq7MXJfBzW8u4Khz+MTUXIG+miW4m2B2AN7LOCY520ozmp8Pj1V2X2MMMLpPC565sjvzNu3nhvHzrWZZDXd8BBNROxPMAiBVRFJEJAJPEplSspGIdAAaAHO9Nk8DBotIAxFpAAx2thljTuGyM5J4YUxPlmw7xKBnZjJh/jabYVZDZR0fwQTudTCuJRhVLQDuxpMYVgMfqOpKEXlcRIZ7NR0NTFBV9dr3APAEniS1AHjc2WaMKcfQbs2ZcvdZtImP4ZGPVnDpS7NZtv1Q+TuaoJKdW0BUeEhA16QTr7/rQS0tLU0XLlzo7zCMCRiqyuSlO/jb1DXsy8rln5d348q05PJ3NEHh0Y9W8PWq3Sz8w4VVeh0RWaSqaT4K6wSBm/qMMVUiIlzaM4npD55Lz+T6/HPaWo7l2QyzmiLQC12CJRhjarzYqHB+O6QDGZm5vDNvq7/DMT4S6IuNgSUYY2qFM1s34qy2jfn3zI3Hp7ea4JZpIxhjTKB4YHA79mfn8d+5W/wdivEBG8EYYwLGGS0acF77eF6dtYnMnHx/h2OqyM7BGGMCyv0XtuPQ0XzG/7Dl+La1uzP5zcRljJuxwX+BmUrLyi0M+BFMYEdnjPGpbkn1ubBTAq//sIluSfV4e95Wpq/ZS4hAkUJi/TqM7FlqyUATYLJy86kbwBdZgo1gjKl17r+gHZk5Bdz05gKWbj/EAxe248ffXUCfVg155KPlrNx52N8hmnIUFBaRk18U8IfIAjs6Y4zPdWoexxMjOqPAlb2SqeMsufviNT0Z9q8fuOOdRXx691nUj44A4EB2Hv+avp7tB47x/OgeAX9YpjbIdq5nCvTPwkYwxtRC1/VrxfX9Wh1PLgBNYqN4+dpe7D6cw70TlnIsr5BXZm7k3Kdm8N85W5i+Zg/3vLeYgsIiP0Zu4OdKypZgjDFB44wWDXhseGdmrcugz1+/4e9frKF3q4ZMu+8c/jKyKzPWZjB2ykpqSompYJUVBKX6wQ6RGWNKuLpPCzbuzWZZ+iEevLAd/ds2BiA1IZbtB4/y8ncbSW4QzZ0D2/g50torGBYbA0swxpgSRISxwzqV+tzDg9uTfvAY//hyDYkN6jC8e/Nqjs5AcCw2BnaIzBhTCSEhwlNXdKN3qwY88P5Snv16HXkFdk6mutk5GGNMjRQVHsrrN/RmWPfmvPDteob96weWp9t6M9UpMyc4EkxgR2eMCUj16oTz3FU9GNqtGb/7eAUjx83m2r4tSYiLIje/kGP5hYSIcNOAFJrWi/J3uDVOdhCsZgmWYIwxVXB+xwS+atWQv09dzVtzPUsBiEBUWCj5hUV8tnwXb9/Sh9bxdf0cac1SfB1MoJ+DCezojDEBr16dcJ68vBt/GtaZkBCICA1BRFiRfpgb35jPFf+ey5s39aZbUn1/h1pjZOUWEBYiRIYF9lmOwI7OGBM06kSEEhkWiogA0DWpHhPv7E90RChjXp3HD+v3+TnCmiMrp4C6UWHH/68DlSUYY4xrUhrHMOnO/iQ3jOamN+ezYMsBf4dUI2TnFhATEfgHoFxNMCIyRETWisgGEXmkjDajRGSViKwUkfe8tv/T2bZaRF6QQE/VxphSJcRF8f7t/ahXJ5z/fL/Z3+HUCFlBsNgYuJhgRCQUGAdcDHQCxohIpxJtUoFHgQGq2hm4z9neHxgAdAO6AL2Bc92K1Rjjrnp1whnZI5Fv1+zhQHaev8MJetl5BQE/gwzcHcH0ATao6iZVzQMmACNKtLkVGKeqBwFUda+zXYEoIAKIBMKBPS7Gaoxx2eW9ksgvVKYs3eHvUIJeVm4hdaPC/R1GudxMMInAdq/H6c42b+2AdiIyW0TmicgQAFWdC8wAdjm3aaq6uuQbiMhtIrJQRBZmZGS40gljjG90bBZH5+ZxTFyc7u9Qgl5WTuAvNgb+P8kfBqQCA4ExwGsiUl9E2gIdgSQ8SWmQiJxdcmdVfVVV01Q1LT4+vhrDNsacjit6JfHTjiOs3nXE36EEtezcwppzkl9EYkQkxLnfTkSGi0h547MdQLLX4yRnm7d0YIqq5qvqZmAdnoRzKTBPVbNUNQv4AuhXkViNMYFrRI9EwkOFSYtsFFMV2bkFAX+RJVR8BDMLiBKRROAr4DrgzXL2WQCkikiKiEQAo4EpJdpMxjN6QUQa4zlktgnYBpwrImFOIjsXOOkQmTEmuDSMiWBQhyZMXrqDfFu47LSoKll5BcRG1ZwEI6p6FLgMeElVrwQ6n2oHVS0A7gam4UkOH6jqShF5XESGO82mAftFZBWecy4Pq+p+YCKwEVgBLAOWqeqnleybMSYAXdErmX1Zecxca+dNT8fRvEJUA79MDFS8VIyISD/gGuAWZ1u5Z5hUdSowtcS2sV73FXjAuXm3KQRur2BsxpggMrB9PI1iIpi4KJ0LOiX4O5ygEyxrwUDFRzD34ble5WNnFNIaz4jDGGMqJTw0hJE97ZqY0/XzapY1ZBaZqs5U1eGq+g/nZP8+Vb3X5diMMTXUFc41MZOX2DUxlZWd66mkXDeyhlwHIyLviUiciMQAPwGrRORhd0MzxtRUHZvF0T25Pm/P20pRkfo7nKCSmZsPBP5aMFDxQ2SdVPUIMBLPlOEUPDPJjDHmtNw8oBWb92Uzc52d7K+Mn0cwNeccTLgzXXgkznUreMq5GGPMabm4SzOaxEYyfrYVwKyMmniS/xVgCxADzBKRloBdimuMOW0RYSFc17cl36/fx4a9mf4OJ2gUn+SPrSkJRlVfUNVEVb1EPbYC57kcmzGmhrv6zBZEhIXwxuwt/g4laGTVtBGMiNQTkWeLC0uKyDN4RjPGGHPaGtWNZET35ny0eAeHj+b7O5ygkJ1bgAhER9Sck/zjgUxglHM7ArzhVlDGmNrjpgEpHMsvZMKCbf4OJShkOatZBsMajBVNMG1U9U/O2i6bVPXPQGs3AzPG1A6dmsfRt3VD3pq7lQKrT1YuT6HLwB+9QMUTzDEROav4gYgMAI65E5Ixpra5aUAKOw4d4+tVtq5gebJzC4NiijJUvBbZHcBbIlLPeXwQuMGdkIwxtc0FHRNIrF+H9+Zv4+KuzfwdTkDLzC0ImgRT0Vlky1S1O9AN6KaqPYFBrkZmjKk1QkOEkT2bM2fjfvZl5fo7nIAWLGvBQCVXtFTVI84V/VCiArIxxlTF8O6JFBYpU1fs8ncoAa3GJpgSAn8KgzEmaLRvGku7hLpMWbrT36EEtBmQX/UAABezSURBVMycgqC4yBKqlmCsVIwxxqeGd2/Owq0H2XHI5hCVJTuvhoxgRCRTRI6UcssEmldTjMaYWmJYd8+flc+W2SimNKpKVk4BdYNguWQoJ8GoaqyqxpVyi1XV4OihMSZotGwUQ/fk+kyxBFOq3IIiCoq0Zs0iM8aY6jK8e3NW7jzCxowsf4cScH5ezdISjDHGVNrQbs0QwU72lyLbEszPRGSIiKwVkQ0i8kgZbUaJyCoRWSki73ltbyEiX4nIauf5Vm7GaowJDAlxUfRNacSny3aianOJvGXmBE8lZXAxwYhIKDAOuBjoBIwRkU4l2qQCjwIDVLUzcJ/X028BT6lqR6APsNetWI0xgWVY9+Zs2pfNyp227JS34hFMbE04yV9FfYANTnHMPGACMKJEm1uBcap6EEBV9wI4iShMVb92tmep6lEXYzXGBJCLuzQlLETsZH8JwbQWDLibYBKB7V6P051t3toB7URktojME5EhXtsPichHIrJERJ5yRkQnEJHbiteoyciwdb2NqSkaxEQwsH08Hy/ZQV6BVVguZif5KycMSAUGAmOA10SkvrP9bOAhoDeepQFuLLmzqr6qqmmqmhYfH19dMRtjqsE1Z7YkIzOXr1bt9ncoAcMSzM92AMlej5Ocbd7SgSmqmq+qm4F1eBJOOrDUObxWAEwGznAxVmNMgDmnXTzJDevwzryt/g4lYByfRWbnYFgApIpIiohEAKOBKSXaTMYzekFEGuM5NLbJ2be+iBQPSwYBq1yM1RgTYEJDhKv7tGTepgOs35Pp73ACQpYziyw6vGYtOFZpzsjjbmAasBr4QFVXisjjIjLcaTYN2C8iq4AZwMOqul9VC/EcHvtWRFbgKaz5mluxGmMC06i0JCJCQ3j3R1tOGSDLWWwsJCQ4ag27Os5S1anA1BLbxnrdVzxl/08q/e/MIOvmZnzGmMDWqG4kl3RtyqRF6Tx8UfugmT3llqzc/KBZLhn8f5LfGGNO6bp+LcnMLbApywTXcslgCcYYE+DOaNGADk1jeXvu1lp/ZX8wLZcMlmCMMQFORLiuX0tW7TrC4m2H/B2OX2XnBk+pfrAEY4wJAiN7JFI3MqzWT1nOyikgJsISjDHG+ExMZBgjezZn6opdHMnJ93c4fpNlIxhjjPG9K3olk1tQxOfLd/k7FL/JsnMwxhjje92T6tEmPoZJi9L9HYpfqKrnHIwlGGOM8S0R4fJeSSzcepAt+7L9HU61K14uOZiuBbIEY4wJGpf2TEQEPlpc+0YxWUG2FgxYgjHGBJFm9epwVtvGTFq8g6Ki2nVNTHEdMptFZowxLrn8jCR2HDrGj5sP+DuUapUVZJWUwRKMMSbIXNS5KXUjw5hUyw6TBdtaMGAJxhgTZOpEhPKLrs34YsUujuYV+DucapNtCcYYY9x3ea8ksvMK+fKn2rPaZfEIxmaRGWOMi3q3akCLhtFMrEXXxNgsMmOMqQYiwqi0JOZs3M/ibQf9HU61KJ5FZofIjDHGZTcNSCE+NpInPltVK8r4Z+cWIALREbbgmDHGuComMoyHB7dnybZDtWIxsszcAupGhCESHMslgyUYY0wQu7xXEp2axfGPL9aQk1/o73BclZ1bEFQn+MESjDEmiIWGCH8c2omdh3N4/ftN/g7HVcFWqh9cTjAiMkRE1orIBhF5pIw2o0RklYisFJH3SjwXJyLpIvKim3EaY4JXvzaNGNwpgZe+28jeIzn+Dsc1WbmFNoIpJiKhwDjgYqATMEZEOpVokwo8CgxQ1c7AfSVe5glgllsxGmNqht9d0pH8wiKe/mqtv0NxTVZOPrGWYI7rA2xQ1U2qmgdMAEaUaHMrME5VDwKo6t7iJ0SkF5AAfOVijMaYGqBV4xhu6NeKDxels2rnEX+H44rs3EJiIoNnBhm4m2ASge1ej9Odbd7aAe1EZLaIzBORIQAiEgI8Azx0qjcQkdtEZKGILMzIyPBh6MaYYHPPoFTiosJ58ss1/g7FFZ7VLMP9HUal+PskfxiQCgwExgCviUh94C5gqqqe8jJdVX1VVdNUNS0+Pt71YI0xgatedDj3DGrLrHUZfL++5n3h9CQYG8EU2wEkez1OcrZ5SwemqGq+qm4G1uFJOP2Au0VkC/A0cL2IPOlirMaYGuC6fi1JalCHv09dU6PWi1FVm0VWwgIgVURSRCQCGA1MKdFmMp7RCyLSGM8hs02qeo2qtlDVVngOk72lqqXOQjPGmGKRYaE8fFF7Vu06wuSlJb/PBq+c/CIKg2y5ZHAxwahqAXA3MA1YDXygqitF5HERGe40mwbsF5FVwAzgYVXd71ZMxpiab1i35nRNrMfT09bWmIsvjxe6tATzM1WdqqrtVLWNqv7V2TZWVac491VVH1DVTqraVVUnlPIab6rq3W7GaYypOUJChEcv6cDOwzm8OWeLv8PxiWAs1Q/+P8lvjDE+179NY85rH8+4GRs4mJ3n73CqLBgXGwNLMMaYGuqRizuSlVvAS99t8HcoVZYZhKX6wRKMMaaGat80lsvPSOK/c7ay49Axf4dTJcdHMDaLzBhjAsP9F7YDgee+XufvUKrEzsEYY0yASaxfhxv6tWTS4nTW7A7eEjI2i8wYYwLQXQPbUjcyjKe+DN5CmDaCMcaYANQgJoI7B7bh2zV7mb/5gL/DOS3BuFwyWIIxxtQCN/VPISEukie/WM3hY/kcyfHcjuYV+Du0CsnMCb7lksFTbNIYY2q0OhGh3HdBOx79aAXd//zzCiChIcI/Lu/GFb2S/Bhd+bKDsA4ZWIIxxtQSo9KSiY4IJSMz9/i2Kct28uQXq7m4S9OAPr+RlVsQ0PGVJfgiNsaY0xAaIozoceKSVL1aNuDSl+Yw/ofN3HN+qp8iK5+nVH/w/bm2czDGmFqrZ4sGXNQ5gVdnbeJAAJeUsQRjjDFB6KHB7cnOK+DlAC4pk20Jxhhjgk9qglNSZu5WdgZoSZmsnOA8yW8JxhhT6913YTtQ+L9v1vs7lFLZITJjjAlSifXrcF2/lny4aDsb9mb5O5wTHF8u2RKMMcYEp1+d15Y64aG8MnOjv0M5QU5+EUUafGViwBKMMcYA0DAmgqHdmvP5il3Hy+MHgszcfCD4SvWDJRhjjDluVO8kjuYV8vmKXf4O5bjs3EIA6kYGVx0ysARjjDHHndGiAa3jY5i4MN3foRyXdXw1y3A/R1J5riYYERkiImtFZIOIPFJGm1EiskpEVorIe862HiIy19m2XESucjNOY4wBEBGu6JXE/C0H2Lwv29/hAN6l+m0Ec5yIhALjgIuBTsAYEelUok0q8CgwQFU7A/c5Tx0Frne2DQGeF5H6bsVqjDHFLj8jiRCBiYu2+zsUwHuxMRvBeOsDbFDVTaqaB0wARpRocyswTlUPAqjqXuffdaq63rm/E9gLxLsYqzHGAJAQF8W57eKZtGgHhUXq73COTziwEcyJEgHvrwDpzjZv7YB2IjJbROaJyJCSLyIifYAI4KS5gyJym4gsFJGFGRkZPgzdGFObXZmWzO4jOfywYZ+/QyHTSTA2i6zywoBUYCAwBnjN+1CYiDQD3gZuUtWikjur6quqmqaqafHxNsAxxvjG+R2bUD86nA8W+v8wWfEIxi60PNEOINnrcZKzzVs6MEVV81V1M7AOT8JBROKAz4Hfq+o8F+M0xpgTRIaFMrJHIl+v3MOho/6tspyVU0CIQJ1wO0TmbQGQKiIpIhIBjAamlGgzGc/oBRFpjOeQ2San/cfAW6o60cUYjTGmVFemJZFXWMSHfp6yXLzYWLAtlwwuJhhVLQDuBqYBq4EPVHWliDwuIsOdZtOA/SKyCpgBPKyq+4FRwDnAjSKy1Ln1cCtWY4wpqXPzepyZ0pC/Tl3NXz9fRW5BoV/iyMotIDYID48BiKr/Z0n4Qlpami5cuNDfYRhjapBjeYX8bepq3p63lQ5NY/m/0T1p3zS2WmO4851FbNibxdcPnOvK64vIIlVNc+O1/X2S3xhjAladiFCeGNmF8TemsS8rl2Ev/sC/vl3P4aP51RZDVm5wrgUDlmCMMaZcgzok8OV95zCwXTzPfL2Ofk9+y2NTVrJ1v/tX+wdrqX6wBGOMMRXSuG4kr16fxtR7z2ZIl6a8++NWBj79HY9MWk6RixdkZuVYgjHGmFqhU/M4nh3Vgx9+O4jr+7ZkwoLt/Gv6BtfeL9uZRRaMgjNqY4zxs4S4KB4b3pnMnAKe/3YdXZPiGNQhwafvUVBYxKFj+TaCMcaY2kZE+NtlXenYNI77Jixli48rMH+/YR9H8wrp27qRT1+3uliCMcaYKogKD+WV63oREiLc8c4ijub5bjXMjxbvoH50OOd1CM5SWJZgjDGmipIbRvPC6J6s25PJwxN9c9L/SE4+X63czbBuzYkMC74yMWAJxhhjfOKcdvH8ZkgHPl++i7FTfqKqF7F/sWIXuQVFXHZGySL0wSM4zxwZY0wAuv2c1hzMzuOVWZuICgvl97/oeNo1xCYt3kHrxjH0SA7etRYtwRhjjI+ICI9c3IHcgiJe/2EzUeGhPHRRewBUlTW7M9l24CiDOjQhPLTsA0jbDxxl/uYDPDS4XVAWuSxmCcYYY3xIRBg7tBM5+YW8OGMD+7PzOJpXwOwN+9mXlQvA2amNefnaXmVOP/54iWdlk5E9g/fwGFiCMcYYnwsJEf56aVfyCor43/xtNK4bwYC2jRnQtjHH8gp5/LNVXPXKXN64qTdNYqNO2FdV+WhxOn1bNySpQbSfeuAblmCMMcYFoSHCM6O685shHWgSG0lIyM+Hulo0iuaudxZz2Utz+O/NfWgTX/f4c4u3HWLL/qPcdV5bf4TtUzaLzBhjXCIiNK0XdUJyATivfRPev70vOfmFXP7yHB6bspIpy3aSfvAoHy1OJyo8hIu7NPVT1L5jIxhjjPGDbkn1mXRnf/4w+ScmLNjGm3O2HH9uRI/mxEaF+y84H7EEY4wxftKyUQxv33Im+YVFrN2dyeJtB1m96wg3DUjxd2g+YQnGGGP8LDw0hC6J9eiSWM/fofiUnYMxxhjjCkswxhhjXOFqghGRISKyVkQ2iMgjZbQZJSKrRGSliLzntf0GEVnv3G5wM05jjDG+59o5GBEJBcYBFwLpwAIRmaKqq7zapAKPAgNU9aCINHG2NwT+BKQBCixy9j3oVrzGGGN8y80RTB9gg6puUtU8YAIwokSbW4FxxYlDVfc62y8CvlbVA85zXwNDXIzVGGOMj7mZYBKB7V6P051t3toB7URktojME5EhldgXEblNRBaKyMKMjAwfhm6MMaaq/H2SPwxIBQYCY4DXRKTCtalV9VVVTVPVtPj44FzxzRhjaio3E8wOINnrcZKzzVs6MEVV81V1M7AOT8KpyL7GGGMCmFR11bUyX1gkDE/COB9PclgAXK2qK73aDAHGqOoNItIYWAL0wDmxD5zhNF0M9FLVA6d4vwxga4nN9YDDldxW3v3GwL6y4ihHae9dmTYV6U919aW8WMtrU9m+lHxcfN97m302FYu1vDb22fj3b8Cp2rnRlxhVdecQkKq6dgMuwZNkNgK/d7Y9Dgx37gvwLLAKWAGM9tr3ZmCDc7vpNN//1cpuK+8+sLAK/x8nvXdl2lSkP9XVl6r2p7J9OUUfvLfZZ2OfTUB/NhXpiy8/G7d/zsq7uVoqRlWnAlNLbBvrdV+BB5xbyX3HA+OrGMKnp7GtIvd9GU9l2lSkP9XVl4q+TlltKtuXko8/LaPN6bLP5tTb7bOpvr8Bp2oXSH0pl2uHyGoqEVmoqmn+jsMXalJfoGb1pyb1BWpWf6wvFefvWWTB6FV/B+BDNakvULP6U5P6AjWrP9aXCrIRjDHGGFfYCMYYY4wrLMEYY4xxRa1OMCIyXkT2ishPp7FvLxFZ4VSKfkFExOu5e0RkjVMh+p++jbrMeHzeFxF5TER2iMhS53aJ7yMvMyZXPhvn+QdFRJ1rr1zn0mfzhIgsdz6Xr0Skue8jLzUeN/rylPP7slxEPq5MNY+qcqk/Vzq/+0Ui4vpkgKr0oYzXK7WSfXm/V6Vycw50oN+Ac/BczPnTaew7H+iL51qeL4CLne3nAd8Akc7jJkHcl8eAh2rKZ+M8lwxMw3NRbuNg7QsQ59XmXuDfQdyXwUCYc/8fwD+C+ecM6Ai0B74D0gK1D058rUpsawhscv5t4NxvcKr+nupWq0cwqjoLOKE6gIi0EZEvRWSRiHwvIh1K7icizfD8gs9Tz//8W8BI5+k7gSdVNdd5j70l93eDS33xGxf78xzwGzzVIqqFG31R1SNeTWOopv641JevVLXAaToPT2moauFSf1ar6trqiN95v9PqQxlKrWR/un8nanWCKcOrwD2q2gt4CHiplDaJeOqoFfOu9twOOFtEfhSRmSLS29VoT62qfQG42zl0MV5EGrgXaoVUqT8iMgLYoarL3A60Aqr82YjIX0VkO3ANMBb/8cXPWbGb8Xw79idf9sdfKtKH0pRVyf60+uvqlfzBRkTqAv2BD70OL0ZW8mXC8Awv+wK9gQ9EpLWT9auNj/ryMvAEnm/HTwDP4PkDUO2q2h8RiQZ+h+dwjF/56LNBVX8P/F5EHgXuxrNIX7XyVV+c1/o9UAC865voTisGn/XHX07VBxG5Cfi1s60tMFVE8oDNqnqpr2OxBHOiEOCQqvbw3iie1TkXOQ+n4PnD6z2M9672nA585CSU+SJShKegXHUvWFPlvqjqHq/9XgM+czPgclS1P22AFGCZ80uXBCwWkT6qutvl2Evyxc+Zt3fxlGSq9gSDj/oiIjcCQ4Hzq/vLWAm+/mz8odQ+AKjqG8AbACLyHXCjqm7xarIDz/IpxZLwnKvZwen01+0TUIF+A1rhdXIMmANc6dwXoHsZ+5U84XWJs/0O4HHnfjs8w00J0r4082pzPzAhmD+bEm22UE0n+V36bFK92twDTAzivgzBU/A2vjp/vtz+OaOaTvKfbh8o+yT/Zjwn+Bs49xtWpL+lxuWPDzRQbsD/gF1APp6Rxy14vuV+CSxzfujHlrFvGvATnkrRL/JzVYQI4B3nucXAoCDuy9t4qlwvx/OtrVl19MWt/pRos4Xqm0Xmxmczydm+HE/hwsQg7ssGPF/Eljq3apkR52J/LnVeKxfYA0wLxD5QSoJxtpdayb4iv1clb1YqxhhjjCtsFpkxxhhXWIIxxhjjCkswxhhjXGEJxhhjjCsswRhjjHGFJRhTo4lIVjW/3xwfvc5AETksnmrJa0Tk6QrsM1JEOvni/Y3xBUswxlSCiJyy+oWq9vfh232vnquxewJDRWRAOe1HApZgTMCwBGNqnbIqzYrIMKdI6RIR+UZEEpztj4nI2yIyG3jbeTxeRL4TkU0icq/Xa2c5/w50np/ojEDeLV4/Q0QucbYtctbVOGUJHlU9hucCxOKinbeKyAIRWSYik0QkWkT6A8OBp5xRT5sqVNQ1xicswZjaqKxKsz8AfVW1JzABT1n/Yp2AC1R1jPO4A57S5n2AP4lIeCnv0xO4z9m3NTBARKKAV/CspdELiC8vWKeKdSowy9n0kar2VtXuwGrgFlWdg6fawsOq2kNVN56in8ZUCyt2aWqVcqrlJgHvO2tfROCpw1RsijOSKPa5etb8yRWRvUACJ5YzB5ivqunO+y7FUy8qC9ikqsWv/T/gtjLCPVtEluFJLs/rz0U5u4jIX4D6QF08C6hVpp/GVAtLMKa2KbPSLPAv4FlVnSIiA/Gs6Fksu0TbXK/7hZT+u1SRNqfyvaoOFZEUYJ6IfKCqS4E3gZGqusypQjywlH1P1U9jqoUdIjO1inpWgtwsIlcCiEd35+l6/FyC/IbS9veBtUBrEWnlPL6qvB2c0c6TwG+dTbHALuew3DVeTTOd58rrpzHVwhKMqemiRSTd6/YAnj/KtziHn1YCI5y2j+E5pLQI2OdGMM5htruAL533yQQOV2DXfwPnOInpj8CPwGxgjVebCcDDziSFNpTdT2OqhVVTNqaaiUhdVc1yZpWNA9ar6nP+jssYX7MRjDHV71bnpP9KPIflXvFzPMa4wkYwxhhjXGEjGGOMMa6wBGOMMcYVlmCMMca4whKMMcYYV1iCMcYY44r/B30uyn3SCAPrAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ciDPSPJZrAap",
        "colab_type": "text"
      },
      "source": [
        "We could do this in stages, picking a learning rate at each stage, gradually unfreezing and training our model. But here I'll just do a rough first pass with some learning rates that are pretty much just guesses:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wz0T8wtwrMoN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 516
        },
        "outputId": "6aa19b95-a15a-4a5a-a9de-ad1c882d6983"
      },
      "source": [
        "# Lots of room to optimize here\n",
        "learn.fit_one_cycle(2, 1e-2)\n",
        "learn.freeze_to(-2)\n",
        "learn.fit_one_cycle(3, slice(5e-3/2., 5e-3))\n",
        "learn.unfreeze()\n",
        "learn.fit_one_cycle(8, slice(2e-3/100, 2e-3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>root_mean_squared_error</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.408922</td>\n",
              "      <td>0.388217</td>\n",
              "      <td>0.618519</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.372963</td>\n",
              "      <td>0.376649</td>\n",
              "      <td>0.610002</td>\n",
              "      <td>00:03</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>root_mean_squared_error</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.364128</td>\n",
              "      <td>0.378104</td>\n",
              "      <td>0.610888</td>\n",
              "      <td>00:04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.341824</td>\n",
              "      <td>0.362447</td>\n",
              "      <td>0.596669</td>\n",
              "      <td>00:04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.308411</td>\n",
              "      <td>0.355083</td>\n",
              "      <td>0.590309</td>\n",
              "      <td>00:04</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>root_mean_squared_error</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.289866</td>\n",
              "      <td>0.352724</td>\n",
              "      <td>0.587997</td>\n",
              "      <td>00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.279483</td>\n",
              "      <td>0.354678</td>\n",
              "      <td>0.589670</td>\n",
              "      <td>00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.263848</td>\n",
              "      <td>0.356244</td>\n",
              "      <td>0.590417</td>\n",
              "      <td>00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.232595</td>\n",
              "      <td>0.348897</td>\n",
              "      <td>0.584643</td>\n",
              "      <td>00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.209547</td>\n",
              "      <td>0.353752</td>\n",
              "      <td>0.588064</td>\n",
              "      <td>00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.190320</td>\n",
              "      <td>0.349932</td>\n",
              "      <td>0.584656</td>\n",
              "      <td>00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.174233</td>\n",
              "      <td>0.356348</td>\n",
              "      <td>0.589636</td>\n",
              "      <td>00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.165692</td>\n",
              "      <td>0.360438</td>\n",
              "      <td>0.593117</td>\n",
              "      <td>00:08</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xgKskOj0rdJ-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "f39a16d6-855d-4c7a-bc07-3778b488ce37"
      },
      "source": [
        "# Save and see how we do\n",
        "preds, y = learn.get_preds(DatasetType.Test)\n",
        "sub = pd.DataFrame({\n",
        "    'tweet_id':test['tweet_id'],\n",
        "    'label':[p[0] for p in preds.numpy()]\n",
        "})\n",
        "sub.to_csv('fastai_2nd_try_lm.csv', index=False)\n",
        "sub.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>00BHHHP1</td>\n",
              "      <td>0.352108</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>00UNMD0E</td>\n",
              "      <td>0.577899</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>01AXPTJF</td>\n",
              "      <td>0.398042</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>01HOEQJW</td>\n",
              "      <td>1.351928</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>01JUKMAO</td>\n",
              "      <td>0.114119</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   tweet_id     label\n",
              "0  00BHHHP1  0.352108\n",
              "1  00UNMD0E  0.577899\n",
              "2  01AXPTJF  0.398042\n",
              "3  01HOEQJW  1.351928\n",
              "4  01JUKMAO  0.114119"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "txpwWKc3rke1",
        "colab_type": "text"
      },
      "source": [
        "Now we're talking! By tuning the language model first, we get a learner more suited to our task and we end up scoring 0.588. This already puts us top 30 out of 250+ entrants, and this model is a single LSTM that can make predictions VERY quickly compared to the larger transformers used by the winning entrants. Note we spent almost no time training, guessed some numbers for lr etc, and basically just threw this together. I think that with a bit more time spent this could get a competitive model going. BUT transformer models are the rage, and so let's move on to trying some of those to see how much better we can get."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MOFFs5M8sixg",
        "colab_type": "text"
      },
      "source": [
        "# 2) Transformers Assemble!\n",
        "\n",
        "You can start here if you want - this is independant from Section 1.\n",
        "\n",
        "Background on transformers... [I guess you can google it]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c7xl_8FvsojM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Install the simpletransformers library:\n",
        "!pip install simpletransformers -q"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VUb8ASkjtABi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "outputId": "804bdd16-107e-4e54-ed27-2a9927753bcd"
      },
      "source": [
        "# Imports and load the data\n",
        "import pandas as pd \n",
        "from simpletransformers.classification import ClassificationModel\n",
        "\n",
        "train = pd.read_csv('Train.csv').dropna(0) # Read in train, ignoring one row with missing data\n",
        "test = pd.read_csv('Test.csv').fillna('') # Read in test\n",
        "test['label']=0 # We'll fill this in with predictions later\n",
        "\n",
        "# Get some local validation going\n",
        "df_valid = train.sample(1000)\n",
        "df_train = train.loc[~train.tweet_id.isin(df_valid.tweet_id.values)]\n",
        "\n",
        "train.head(3) # Remind ourselves what the data looks like"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>safe_text</th>\n",
              "      <th>label</th>\n",
              "      <th>agreement</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CL1KWCMY</td>\n",
              "      <td>Me &amp;amp; The Big Homie meanboy3000 #MEANBOY #M...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>E3303EME</td>\n",
              "      <td>I'm 100% thinking of devoting my career to pro...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>M4IVFSMS</td>\n",
              "      <td>#whatcausesautism VACCINES, DO NOT VACCINATE Y...</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   tweet_id  ... agreement\n",
              "0  CL1KWCMY  ...       1.0\n",
              "1  E3303EME  ...       1.0\n",
              "2  M4IVFSMS  ...       1.0\n",
              "\n",
              "[3 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XJ2FkfCctcQ1",
        "colab_type": "text"
      },
      "source": [
        "Simpletransformers expects a dataframe with a text column and a label column. We tell it what model we want (there is a wide selection) and specify things like the number of epochs and the learning rate with a dictionary of arguments. This example comes pretty much vebatim from their readme on github..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C3ka5BTssvRp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 601,
          "referenced_widgets": [
            "7bfc92373971404c82c893f6110ee02f",
            "06b1e0e6273044e7923923a9835f0bdb",
            "8af370330541427db3bab98008001207",
            "8dfd25a5820747559299ec4ab8ec7e48",
            "6801510baeef42189b06c3190c88b22e",
            "9eb71605fb1a46c987e830a6373f8931",
            "1937a47f310c4e92a6c2cd931fe6dc9c",
            "a6bf9b1468104506a9153cdb5d043a22",
            "4d7e804c6097492dae849e9f6f17dfea",
            "928db950860f4d7eb1244976e07340ec",
            "c21e200a0eaa42559ecd60c9f43c6c41",
            "a98245ca84b948ec9d93a1f56d2d93cc",
            "2e817807da2e4d93ac8dc05ad4481c81",
            "60e82ae75dff4eec978ef8821fa7ed59",
            "a402f09ca15f48478ef3ebef9d8deede",
            "84b959b97c494e119eb44dd729d494fc",
            "a599f37980174e1fac1cf52979c245c0",
            "99d0a0cb7263435f950f396778bcd7dc",
            "7156685b440f4b9e8ebf08b7aadbd833",
            "95ca19de8c9544ccb60664f695466c9f",
            "eee873822e4940a0b57f10622c31436a",
            "e9d48e11bae24b0185dab499900bcd0d",
            "60ad1420aa434290aba6b1b1666cedce",
            "bba22b3ab5304851aca8fc6e369de611",
            "89d634f4c697419192bbca8ba1c7a00c",
            "a7597d4690c247df8d405c5ac949d5da",
            "bfced04a7c6c4192a03d77d4fa455b57",
            "38602ece6def4541ae784224f7502e2c",
            "88ab8cf638c3415baba06d3bdbf404df",
            "347846c981364e81977e8e8507c07833",
            "854581df260344b1bb8ccc05586e9882",
            "7c6ce8c73c4b4d23bebc46f9afb55816",
            "4dbbdf27eaff473e8e2b51e6325ea40f",
            "8f32c914655242f5ba8d36a163dc87a7",
            "ce8f2d8a8b67463d85993ca0aece6baa",
            "ac446200ebee4256b013faaeb2b59fa9",
            "b31517bc0bc247e7ac8a42e6f6a5381d",
            "ff4132f18677435289b13e2db78bf44e",
            "e38110db81274fc1a6b3513c8e3ff095",
            "757dcc97d2404e39a3bcb258ea2f9277",
            "8f26cf7b54af474b90feb0a033227bb2",
            "b3d74d636c5446e48a996388040ff428",
            "fd6de9c69886427ba37c9b1f1b248efa",
            "8e3a8aa1a6cd4f1da417723697f17ba5",
            "021cba019ed643a0918f06f4e4b05070",
            "a159cc22f15c47a38754873ee0e4d9ca",
            "a2669d9b06504677900f062e1874f51b",
            "6b0728e3a5474e859ae4b9f58f0c5f0c",
            "0749fef36bab4a8c8a2274874ee7d2e2",
            "3846517229694c89834eacbf7dc611cd",
            "7f3e59b13f2b45e2b758cddc2fa5fd96",
            "4e825c21b2e44b6399fa7b4f77b02b96",
            "280479ca5c4d4ff79f7bc50f82b022f4",
            "68f56100fd4f4ac6ac094af47a5e6799",
            "b8c2d92d57254af7af30d0e5ec7797c2",
            "9025128f32f049b39ee238049971475e"
          ]
        },
        "outputId": "a2acc75f-0ccb-4d9f-cd45-5ea89ed610f7"
      },
      "source": [
        "# Model arguments:\n",
        "args = {\"reprocess_input_data\": True, \"overwrite_output_dir\": True, \n",
        "        'fp16':False,\n",
        "        \"num_train_epochs\": 3,\n",
        "        \"learning_rate\": 1e-4,\n",
        "        \"max_seq_length\": 128, \n",
        "        'regression': True} # Regression is this simple! \n",
        "\n",
        "# Just the text and labels\n",
        "df_train = df_train[['safe_text', 'label']]\n",
        "df_valid = df_valid[['safe_text', 'label']]\n",
        "\n",
        "# Create a ClassificationModel\n",
        "model = ClassificationModel(\n",
        "    \"distilbert\", \"distilbert-base-uncased\",num_labels=1,args=args\n",
        ")\n",
        "\n",
        "# Train the model\n",
        "model.train_model(df_train)\n",
        "\n",
        "# Evaluate the model\n",
        "result, model_outputs, wrong_predictions = model.eval_model(df_valid)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:275: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7bfc92373971404c82c893f6110ee02f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=8999.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4d7e804c6097492dae849e9f6f17dfea",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a599f37980174e1fac1cf52979c245c0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 3', max=1125.0, style=ProgressStyle(de…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:231: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
            "  warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:200: UserWarning: Please also save or load the state of the optimzer when saving or loading the scheduler.\n",
            "  warnings.warn(SAVE_STATE_WARNING, UserWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "89d634f4c697419192bbca8ba1c7a00c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 3', max=1125.0, style=ProgressStyle(de…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4dbbdf27eaff473e8e2b51e6325ea40f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 3', max=1125.0, style=ProgressStyle(de…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:745: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8f26cf7b54af474b90feb0a033227bb2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0749fef36bab4a8c8a2274874ee7d2e2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=125.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QgxAA_rToj8_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "91d56796-3ca1-447b-eb7a-9907a429d7c2"
      },
      "source": [
        "# RMSE scoring on our validation set:\n",
        "from sklearn.metrics import mean_squared_error as skmse\n",
        "skmse(df_valid['label'], model_outputs)**0.5"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5860418579448166"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hudTydcuv9A2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "b9a456bb440a47d580bb57dde7e2a671",
            "5087a1d42d5c4c4188014be3b3aaa822",
            "8dff72db6a3a48ae811ae8c714070041",
            "e1fea716e5da4cf2861ccd1179a3bd77",
            "11281e73b4ee4822b748ef9f0484342a",
            "7ef816fb26e44cd9832422a6afcdfdaf",
            "b34f58920f74497e94a6f93f2c067a7f",
            "5983754c3eff48979d40be82d165a507",
            "c556108d7f654a6498b358f733775483",
            "7e0d65d4f98741519a03ce63cfc91a77",
            "56c41d26a1754a17b9757269658015cb",
            "de36c2e593e24abba3374ae79df1b872",
            "0bf46d6513a5495496042e39e27dfd51",
            "fd13451cc9054116886d995daeb3e571",
            "d15cc24c4be9496381f9125313d8b5c2",
            "7c76f6d506c44cedbc191048bb97cda5"
          ]
        },
        "outputId": "b0dc8882-68e1-4c3b-f480-48f9b4894b3e"
      },
      "source": [
        "# Saving Predictions\n",
        "sub = pd.DataFrame({\n",
        "    'tweet_id':test['tweet_id'],\n",
        "    'label':model.predict(test['safe_text'].values)[0]\n",
        "})\n",
        "sub.to_csv('transfomer_1.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b9a456bb440a47d580bb57dde7e2a671",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5177.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c556108d7f654a6498b358f733775483",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=648.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s5Bts95Ixb7K",
        "colab_type": "text"
      },
      "source": [
        "This scores ~0.6 on the LB. Again, we've picked an arbitrary lr and num_epocs based on our desire to try something quickly. Some ways to improve:\n",
        "\n",
        "\n",
        "*   Try a different model. Roberta seems to be the favourite\n",
        "*   Train a bit more.\n",
        "*   Ensemble! Why use just one model...\n",
        "\n",
        "Want to use fastai's goodness with transformers? You can mash the two together! Check out https://github.com/morganmcg1/fasthugs or seach around for examples. I think this will be the way to go to really nail this type of contest.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vOsYIvyjwpHU",
        "colab_type": "text"
      },
      "source": [
        "# 3) Ensemble Like The Cool Kids\n",
        "\n",
        "An ensemble built from a few different models can do better than any of its constituents. Let's take our three submissions and combine, paying more attention to the better ones, and submit that to see if it improves our situation..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3zmNtbcOexNr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "s1 = pd.read_csv('/content/first_try_fastai_20_epochs.csv')\n",
        "s2 = pd.read_csv('/content/fastai_2nd_try_lm.csv')\n",
        "s3 = pd.read_csv('/content/transfomer_1.csv')\n",
        "sub = pd.DataFrame({\n",
        "    'tweet_id':test['tweet_id'],\n",
        "    'label':s1['label']*0.15 + s2['label']*0.35 + s3['label']*0.5\n",
        "})\n",
        "sub.to_csv('fancy_ensemble.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EVfYel8Mx-a2",
        "colab_type": "text"
      },
      "source": [
        "### 0.56! Just for combining three different sets of predictions!!! \n",
        "\n",
        "You can see why this is a popular approach. Imagine if we actually put effort into our models, and trained 5 or 10 different ones in different ways.... "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CHbojwdGybFx",
        "colab_type": "text"
      },
      "source": [
        "# 4) Conclusion\n",
        "\n",
        "So, there are a few solutions to the challenge. Unsuprisingly, an ensemble of deep learning models does very well. But I'm always conflicted by this - in the real world, no-one wants to deplot 15 models if one simple one will do almost as well. A single well-tuned LSTM does decently - and while the competition framework encourages hoards of fancy transformers I really think that for most real problems that approach is overkill. \n",
        "\n",
        "Musings aside, I hope this is helpful :) Please let me know if you use this, and reach out if you have any questions. I am @johnowhitaker on Zindi, twitter, gmail, yahoo.... \n",
        "\n",
        "Happy Hacking :)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P4pk01ZSgsXL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}