notebooks/1_Bulletines_1_Links_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Links Scraper\n",
    "\n",
    "This notebook scrapes the [website](https://www.giustizia.it/giustizia/page/it/statistiche) of the Italian Ministry of Justice and retrieve information available regarding inmates' presence in Italian detention center. The scraper looks for entries containing the string `Detenuti italiani e stranieri presenti e capienze per istituto` and collects all links in a `csv` file stored in `outputs/raw/bulletines_links.csv`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from playwright.async_api import async_playwright\n",
    "import asyncio\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from time import sleep\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.giustizia.it/giustizia/page/it/statistiche'\n",
    "to_search= 'Detenuti italiani e stranieri presenti e capienze per istituto'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"Hey, open up a browser\"\n",
    "playwright = await async_playwright().start()\n",
    "browser = await playwright.chromium.launch(headless=False)\n",
    "\n",
    "# Create a new browser window\n",
    "page = await browser.new_page()\n",
    "print(\"Opening up the browser...\")\n",
    "\n",
    "# Tell it to go to this page\n",
    "await page.goto(url)\n",
    "print(f\"Going to {url}\")\n",
    "\n",
    "await page.wait_for_timeout(2000)\n",
    "\n",
    "search_input = page.locator('form#searchForm input[aria-label=\"Cerca\"]')\n",
    "\n",
    "await search_input.fill(to_search)\n",
    "await page.wait_for_timeout(2000)\n",
    "await search_input.press('Enter')\n",
    "print(f\"Searching for {to_search}\")\n",
    "\n",
    "# Wait for the results to load\n",
    "await page.wait_for_selector('ol.resultVivisimo', timeout=5000)\n",
    "\n",
    "# Data storage\n",
    "data = []\n",
    "n = 1\n",
    "\n",
    "\n",
    "while True:\n",
    "    content = await page.content()\n",
    "    await page.wait_for_timeout(5000)\n",
    "    soup = BeautifulSoup(content, 'html.parser')\n",
    "    links = soup.find_all('a', href=True)\n",
    "\n",
    "    # Filter and extract the relevant links\n",
    "    filtered_links = [\n",
    "        link for link in links \n",
    "        if \"contentId\" in link['href'] and \"Detenuti italiani e stranieri presenti e capienze per istituto\" in link.get_text()\n",
    "    ]\n",
    "\n",
    "    for link in filtered_links:\n",
    "            href = link['href']\n",
    "            text = link.get_text(strip=True)\n",
    "            content_id = re.search(r\"contentId=(\\w+)\", href).group(1)\n",
    "            last_update = text.split(\"aggiornamento al\")[-1].strip()\n",
    "            data.append([content_id, last_update, href])\n",
    "    print(f'got link from page {n}')\n",
    "    print(f\"Total number of links: {len(data)}\")\n",
    "    print(\"##################\")\n",
    "    n = n+1\n",
    "\n",
    "    next_button = await page.query_selector('img[alt=\"Vai alla pagina successiva\"]')  # Select the image by alt text\n",
    "    await page.wait_for_timeout(5000)\n",
    "\n",
    "    # Adding a sleep to give the page some time\n",
    "    await page.wait_for_timeout(5000)  # Wait for 1 second before checking for the button\n",
    "\n",
    "    if next_button:\n",
    "        print('next button found')\n",
    "        # Check if the button is visible and can be clicked\n",
    "        is_visible = await next_button.is_visible()\n",
    "        is_enabled = await next_button.is_enabled()  # Check if the button is enabled\n",
    "\n",
    "        if is_visible and is_enabled:\n",
    "            print(f\"Clicking on page {n}\")\n",
    "            # Click the \"Next\" button\n",
    "            await next_button.click()\n",
    "            await page.wait_for_selector('ol.resultVivisimo', timeout=5000)  # Additional wait time for the new page to load\n",
    "        else:\n",
    "            print('button not visible or enabled')\n",
    "            break  # If the button is not visible or not enabled, break the loop\n",
    "    else:\n",
    "        print('button not found')\n",
    "        break  # If no next button, break the loop\n",
    "\n",
    "# Finally close the browser after everything is done\n",
    "await browser.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a DataFrame and convert dates\n",
    "df = pd.DataFrame(data, columns=[\"ID\", \"Ultimo aggiornamento\", \"Hyperlink\"])\n",
    "\n",
    "# Month mapping for conversion to datetime\n",
    "month_mapping = {\n",
    "    \"Gennaio\": \"January\", \"gennaio\": \"January\",\n",
    "    \"Febbraio\": \"February\", \"febbraio\": \"February\",\n",
    "    \"Marzo\": \"March\", \"marzo\": \"March\",\n",
    "    \"Aprile\": \"April\", \"aprile\": \"April\",\n",
    "    \"Maggio\": \"May\", \"maggio\": \"May\",\n",
    "    \"Giugno\": \"June\", \"giugno\": \"June\",\n",
    "    \"Luglio\": \"July\", \"luglio\": \"July\",\n",
    "    \"Agosto\": \"August\", \"agosto\": \"August\",\n",
    "    \"Settembre\": \"September\", \"settembre\": \"September\",\n",
    "    \"Ottobre\": \"October\", \"ottobre\": \"October\",\n",
    "    \"Novembre\": \"November\", \"novembre\": \"November\",\n",
    "    \"Dicembre\": \"December\", \"dicembre\": \"December\"\n",
    "}\n",
    "\n",
    "for italian, english in month_mapping.items():\n",
    "    df['Ultimo aggiornamento'] = df['Ultimo aggiornamento'].str.replace(italian, english, regex=True)\n",
    "\n",
    "df['Ultimo aggiornamento'] = pd.to_datetime(df['Ultimo aggiornamento'], format='%d %B %Y')\n",
    "df = df.sort_values(by='Ultimo aggiornamento', ascending=False)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Ultimo aggiornamento'].dt.year.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the df as csv\n",
    "df.to_csv('../outputs/clean/bulletines_links.csv', index=False, encoding='UTF-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After a review of the file we've found a set of duplicate values. Dataset [SST365607](https://www.giustizia.it/giustizia/it/mg_1_14_1.page?contentId=SST365607) and [SST360932](https://www.giustizia.it/giustizia/it/mg_1_14_1.page?contentId=SST360932) contain the same data from December 2021. Instead, October 2021 is missing (it used to be stored with id [SST352771](https://www.giustizia.it/giustizia/en/mg_1_14_1.page?contentId=SST352771)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove one duplicate (SST365607 = SST360932)\n",
    "duplicate_id = 'SST365607'\n",
    "df = df[df['ID'] != 'SST365607']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the df as csv\n",
    "df.to_csv('../outputs/clean/bulletines_links.csv', index=False, encoding='UTF-8')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}