notebooks/1_Bulletines_4_Monthly_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weekly Scraper (Bulletines)\n",
    "\n",
    "This is a Scraper that uses Github Actions to run monthly and retrieve updated data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from playwright.async_api import async_playwright\n",
    "import asyncio\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from time import sleep\n",
    "import re\n",
    "from datetime import datetime, timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'30 settembre 2024'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mapping of month numbers to Italian month names\n",
    "italian_months = [\n",
    "    \"gennaio\", \"febbraio\", \"marzo\", \"aprile\", \"maggio\", \"giugno\",\n",
    "    \"luglio\", \"agosto\", \"settembre\", \"ottobre\", \"novembre\", \"dicembre\"\n",
    "]\n",
    "\n",
    "# Get the current date\n",
    "today = datetime.now()\n",
    "\n",
    "# Calculate the first day of the current month\n",
    "first_day_of_current_month = today.replace(day=1)\n",
    "#  Calculate the last day of the previous month\n",
    "last_day_of_previous_month = first_day_of_current_month - timedelta(days=1)\n",
    "\n",
    "# Format the date in Italian manually\n",
    "day = last_day_of_previous_month.day\n",
    "month = italian_months[last_day_of_previous_month.month - 1]  # Adjust for zero-based index\n",
    "year = last_day_of_previous_month.year\n",
    "\n",
    "# Create the final formatted date string\n",
    "formatted_date = f\"{day} {month} {year}\"\n",
    "formatted_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2024-09-30'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date_value = last_day_of_previous_month.strftime('%Y-%m-%d')\n",
    "date_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.giustizia.it/giustizia/page/it/statistiche'\n",
    "to_search= f'Detenuti italiani e stranieri presenti e capienze per istituto - aggiornamento al {formatted_date}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Opening up the browser...\n",
      "Going to https://www.giustizia.it/giustizia/page/it/statistiche\n",
      "Searching for Detenuti italiani e stranieri presenti e capienze per istituto - aggiornamento al 30 settembre 2024\n",
      "Clicked on the link: Detenuti italiani e stranieri presenti e capienze per istituto - aggiornamento al 30 settembre 2024\n"
     ]
    }
   ],
   "source": [
    "# # \"Hey, open up a browser\"\n",
    "playwright = await async_playwright().start()\n",
    "browser = await playwright.firefox.launch()\n",
    "context = await browser.new_context(viewport={'width': 1280, 'height': 800})\n",
    "page = await context.new_page()\n",
    "\n",
    "# \"Hey, open up a browser\"\n",
    "# playwright = await async_playwright().start()\n",
    "# browser = await playwright.chromium.launch(headless=False)\n",
    "# page = await browser.new_page()\n",
    "\n",
    "print(\"Opening up the browser...\")\n",
    "\n",
    "# Tell it to go to this page\n",
    "await page.goto(url, timeout=0)\n",
    "\n",
    "print(f\"Going to {url}\")\n",
    "\n",
    "search_input = page.locator('form#searchForm input[aria-label=\"Cerca\"]')\n",
    "\n",
    "await search_input.fill(to_search)\n",
    "await page.wait_for_timeout(2000)\n",
    "await search_input.press('Enter')\n",
    "print(f\"Searching for {to_search}\")\n",
    "\n",
    "# Wait for the results to load\n",
    "await page.wait_for_selector('ol.resultVivisimo', timeout=0)\n",
    "\n",
    "# Locate all the search results\n",
    "search_results = page.locator('ol.resultVivisimo li a')\n",
    "\n",
    "# Check if any link matches the search text\n",
    "links = await search_results.all_text_contents()\n",
    "\n",
    "if to_search in links:\n",
    "\n",
    "    data = []\n",
    "\n",
    "    # Click the link if found\n",
    "    await page.click(f'li:has-text(\"{to_search}\") >> a')\n",
    "    print(f\"Clicked on the link: {to_search}\")\n",
    "\n",
    "    await page.wait_for_selector('table', timeout=0)  # Adjust the selector to match the table or content you expect to appear\n",
    "    await page.wait_for_load_state('networkidle', timeout=0)\n",
    "    content = await page.content()\n",
    "\n",
    "    target_link = page.url\n",
    "    print(f\"Current page URL: {target_link}\")\n",
    "    match = re.search(r'contentId=([^&]+)', target_link)\n",
    "    id_value = match.group(1) if match else None\n",
    "\n",
    "\n",
    "    soup = BeautifulSoup(content, \"html.parser\")\n",
    "    rows = soup.find_all(\"tr\")\n",
    "\n",
    "\n",
    "    for row in rows[2:]:  # Skipping the header rows\n",
    "        cells = row.find_all(\"td\")\n",
    "        row_data = [cell.get_text(separator=\" \").strip() for cell in cells]\n",
    "\n",
    "        # Append the Date and ID values to the row_data\n",
    "        row_data.extend([date_value, id_value])\n",
    "        data.append(row_data)\n",
    "    print(f\"Scraped page\")\n",
    "    print(\"#######\")\n",
    "\n",
    "    # Creating df\n",
    "    df = pd.DataFrame(data)\n",
    "\n",
    "    # Adding data to the csv file\n",
    "    bulletines_csv = '../outputs/raw/bulletines_raw.csv'\n",
    "    df.to_csv(bulletines_csv, mode='a', index=False, header=False)\n",
    "    print('New data added to outputs/raw/bulletines_raw.csv')\n",
    "    print('Closing the browser')\n",
    "    await browser.close()\n",
    "\n",
    "\n",
    "else:\n",
    "    # Close the browser if the link is not found\n",
    "    print(f\"Link not found for: {to_search}\")\n",
    "    await browser.close()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}