youtube_hashtag_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f8ae3ad9-b965-4665-b5f6-d724ad954efd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/mf/mn2tnfld4370tjxj00fxczxm0000gn/T/ipykernel_7484/1368333510.py:8: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver = webdriver.Chrome('/Users/kittiwin/Local-Documents/MSc-Com-Sci/Social-Network-Analysis/chromedriver')\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from bs4 import BeautifulSoup \n",
    "import json\n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "driver = webdriver.Chrome('/Users/kittiwin/Local-Documents/MSc-Com-Sci/Social-Network-Analysis/chromedriver') "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecaf7cee-40e8-4b13-94c2-8e2b2fd47cac",
   "metadata": {},
   "source": [
    "# Streamer channels\n",
    "- Heartrocer\n",
    "- Doyser\n",
    "- Bay Riffer\n",
    "- zbing z\n",
    "- SkizzTV\n",
    "- CGGG\n",
    "- Lanna Jr.\n",
    "- Xcroxz\n",
    "- TeamGarry MovieThai\n",
    "- Maser gamer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72fc5848-5bef-4542-a6d7-73583a84b0fe",
   "metadata": {},
   "source": [
    "# Hashtag\n",
    "- #eldenring\n",
    "- #farcry6\n",
    "- #freefire\n",
    "- #GTA\n",
    "- #rov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "be7e7af8-63be-47b5-9db0-eaac33525a9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_video_from_hashtags(hashtag):\n",
    "    driver.get(f'https://www.youtube.com/hashtag/{hashtag}')\n",
    "    \n",
    "    print(f'Collecting {hashtag}...')\n",
    "    \n",
    "    for i in range(3):\n",
    "        # print('Scrolling Down')\n",
    "        for i in range(20):\n",
    "            driver.execute_script(\"window.scrollTo(0, 1000000000)\")\n",
    "            time.sleep(1)\n",
    "\n",
    "        # time.sleep(5)\n",
    "        soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
    "        \n",
    "        title_list = []\n",
    "        length_list = []\n",
    "        channel_list = []\n",
    "        view_list = []\n",
    "        age_list = []\n",
    "        link_list = []\n",
    "        \n",
    "        videos = soup.find_all('div', {'class': 'style-scope ytd-rich-grid-media'})\n",
    "        \n",
    "        for v in videos:\n",
    "            title = v.find_all('yt-formatted-string', {'id': 'video-title'})\n",
    "            length = v.find_all('span', {'class': 'style-scope ytd-thumbnail-overlay-time-status-renderer'})\n",
    "            channel = v.find_all('a', {'class': 'yt-simple-endpoint style-scope yt-formatted-string'})\n",
    "            info = v.find_all('span', {'class': 'style-scope ytd-video-meta-block'})\n",
    "            link = v.find_all('a', {'class': 'yt-simple-endpoint focus-on-expand style-scope ytd-rich-grid-media'})\n",
    "            \n",
    "            if len(title) == 0 or len(length) == 0 or len(channel) == 0 or len(info) == 0 or len(link) == 0:\n",
    "                continue\n",
    "                \n",
    "            if title[0].get_text() == '' or channel[0].get_text() == '' or channel[0] is None:\n",
    "                continue\n",
    "                \n",
    "            if len(info) != 2:\n",
    "                continue\n",
    "            \n",
    "            if 'Schedule' in info[1].get_text():\n",
    "                continue\n",
    "            \n",
    "            title_list.append(title[0].get_text())\n",
    "            length_list.append(length[0].get_text().strip())\n",
    "            channel_list.append(channel[0].get_text())\n",
    "            view_list.append(info[0].get_text())\n",
    "            age_list.append(info[1].get_text())\n",
    "            link_list.append(f\"https://www.youtube.com{link[0]['href']}\")\n",
    "        \n",
    "    print(f'{len(title_list)} videos is recorded...')\n",
    "    \n",
    "    return pd.DataFrame({\n",
    "        'hashtag': [hashtag] * len(title_list),\n",
    "        'title': title_list,\n",
    "        'length': length_list,\n",
    "        'channel': channel_list,\n",
    "        'views': view_list,\n",
    "        'age': age_list,\n",
    "        'url': link_list\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2d1922b5-5ef9-4584-8a50-1895dd1b6d1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "hashtags = ['eldenring', 'farcry6', 'freefire', 'roblox', 'pubgmobile', 'overwatch', 'hshsurvive', 'genshinimpact', 'farlight84', 'apexlegendsmobile', 'theforest']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a0c2d111-7ee8-452a-99e4-add35f0b1db4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting eldenring...\n",
      "535 videos is recorded...\n",
      "Collecting farcry6...\n",
      "522 videos is recorded...\n",
      "Collecting freefire...\n",
      "459 videos is recorded...\n",
      "Collecting roblox...\n",
      "510 videos is recorded...\n",
      "Collecting pubgmobile...\n",
      "513 videos is recorded...\n",
      "Collecting overwatch...\n",
      "514 videos is recorded...\n",
      "Collecting hshsurvive...\n",
      "524 videos is recorded...\n",
      "Collecting genshinimpact...\n",
      "555 videos is recorded...\n",
      "Collecting farlight84...\n",
      "569 videos is recorded...\n",
      "Collecting apexlegendsmobile...\n",
      "547 videos is recorded...\n",
      "Collecting theforest...\n",
      "556 videos is recorded...\n"
     ]
    }
   ],
   "source": [
    "hashtags_df = pd.DataFrame(columns=['hashtag', 'title', 'length', 'channel', 'views', 'age'])\n",
    "for h in hashtags:\n",
    "    df = get_video_from_hashtags(h)\n",
    "    hashtags_df = pd.concat([hashtags_df, df], ignore_index=True)\n",
    "    hashtags_df.to_csv('youtube_hashtags.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}