process_twitter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#1 change file name\n",
    "df = pd.read_csv(\"./input/raw/twitter_golf_day1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>contributors</th>\n",
       "      <th>coordinates</th>\n",
       "      <th>created_at</th>\n",
       "      <th>entities</th>\n",
       "      <th>extended_entities</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>favorited</th>\n",
       "      <th>geo</th>\n",
       "      <th>id</th>\n",
       "      <th>...</th>\n",
       "      <th>possibly_sensitive</th>\n",
       "      <th>quoted_status</th>\n",
       "      <th>quoted_status_id</th>\n",
       "      <th>quoted_status_id_str</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>truncated</th>\n",
       "      <th>user</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sat Apr 20 03:09:28 +0000 2019</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1119437903040479232</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>@realDonaldTrump Still les than your golf outi...</td>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 35229046, 'id_str': '35229046', 'name':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sat Apr 20 03:09:27 +0000 2019</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1119437897135001600</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Top golf is definitely not my thing 🤦🏼‍♀️😅</td>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 555301112, 'id_str': '555301112', 'name...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sat Apr 20 03:09:27 +0000 2019</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>{'media': [{'id': 1116147527877660674, 'id_str...</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1119437896358850561</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;a href=\"http://www.aainc.co.jp/\" rel=\"nofollo...</td>\n",
       "      <td>@atarutoiina1 ご参加ありがとうございます！\\r\\n抽選結果はムービーをご覧くだ...</td>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 1057921974020632576, 'id_str': '1057921...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sat Apr 20 03:09:24 +0000 2019</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1119437885923581955</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>@realDonaldTrump A.  It has cost taxpayers mor...</td>\n",
       "      <td>True</td>\n",
       "      <td>{'id': 1100353448233574400, 'id_str': '1100353...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sat Apr 20 03:09:24 +0000 2019</td>\n",
       "      <td>{'hashtags': [{'text': 'OCUgolf', 'indices': [...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1119437885604868096</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;a href=\"http://www.sidearmsports.com\" rel=\"no...</td>\n",
       "      <td>Stars stand in second place in SAC Championshi...</td>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 53120013, 'id_str': '53120013', 'name':...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  contributors coordinates                      created_at  \\\n",
       "0           0           NaN         NaN  Sat Apr 20 03:09:28 +0000 2019   \n",
       "1           1           NaN         NaN  Sat Apr 20 03:09:27 +0000 2019   \n",
       "2           2           NaN         NaN  Sat Apr 20 03:09:27 +0000 2019   \n",
       "3           3           NaN         NaN  Sat Apr 20 03:09:24 +0000 2019   \n",
       "4           4           NaN         NaN  Sat Apr 20 03:09:24 +0000 2019   \n",
       "\n",
       "                                            entities  \\\n",
       "0  {'hashtags': [], 'symbols': [], 'user_mentions...   \n",
       "1  {'hashtags': [], 'symbols': [], 'user_mentions...   \n",
       "2  {'hashtags': [], 'symbols': [], 'user_mentions...   \n",
       "3  {'hashtags': [], 'symbols': [], 'user_mentions...   \n",
       "4  {'hashtags': [{'text': 'OCUgolf', 'indices': [...   \n",
       "\n",
       "                                   extended_entities  favorite_count  \\\n",
       "0                                                NaN               0   \n",
       "1                                                NaN               0   \n",
       "2  {'media': [{'id': 1116147527877660674, 'id_str...               0   \n",
       "3                                                NaN               0   \n",
       "4                                                NaN               0   \n",
       "\n",
       "   favorited  geo                   id  ...  possibly_sensitive quoted_status  \\\n",
       "0      False  NaN  1119437903040479232  ...                 NaN           NaN   \n",
       "1      False  NaN  1119437897135001600  ...                 NaN           NaN   \n",
       "2      False  NaN  1119437896358850561  ...               False           NaN   \n",
       "3      False  NaN  1119437885923581955  ...                 NaN           NaN   \n",
       "4      False  NaN  1119437885604868096  ...               False           NaN   \n",
       "\n",
       "   quoted_status_id  quoted_status_id_str  retweet_count  retweeted  \\\n",
       "0               NaN                   NaN              0      False   \n",
       "1               NaN                   NaN              0      False   \n",
       "2               NaN                   NaN              0      False   \n",
       "3               NaN                   NaN              0      False   \n",
       "4               NaN                   NaN              0      False   \n",
       "\n",
       "                                              source  \\\n",
       "0  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2  <a href=\"http://www.aainc.co.jp/\" rel=\"nofollo...   \n",
       "3  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "4  <a href=\"http://www.sidearmsports.com\" rel=\"no...   \n",
       "\n",
       "                                                text truncated  \\\n",
       "0  @realDonaldTrump Still les than your golf outi...     False   \n",
       "1         Top golf is definitely not my thing 🤦🏼‍♀️😅     False   \n",
       "2  @atarutoiina1 ご参加ありがとうございます！\\r\\n抽選結果はムービーをご覧くだ...     False   \n",
       "3  @realDonaldTrump A.  It has cost taxpayers mor...      True   \n",
       "4  Stars stand in second place in SAC Championshi...     False   \n",
       "\n",
       "                                                user  \n",
       "0  {'id': 35229046, 'id_str': '35229046', 'name':...  \n",
       "1  {'id': 555301112, 'id_str': '555301112', 'name...  \n",
       "2  {'id': 1057921974020632576, 'id_str': '1057921...  \n",
       "3  {'id': 1100353448233574400, 'id_str': '1100353...  \n",
       "4  {'id': 53120013, 'id_str': '53120013', 'name':...  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove duplicate tweets\n",
    "df = df.drop_duplicates(subset=\"text\", keep=\"first\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[\"text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for writing all text to file\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = df.to_string(index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#2 change file name\n",
    "file = open(\"input/mr/twitter_golf_day1.txt\", \"w\", encoding='utf-8')\n",
    "file.write(text)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}