examples/notebooks/Run_your_first_transform_colab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a98e6dac-e368-4310-a0cb-751faf1e3b9b",
   "metadata": {},
   "source": [
    "The following notebook example will allow you to test DPK, without cloning the repo. You can run it either on Google Colab or you can use your local environment (by downloading just the notebook). We use a temporary folder for input and output, but users are encouraged to use their own input folder. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c4f9c952-cb3b-40f1-bfb5-00d9a43a5715",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install \"data-prep-toolkit-transforms[pdf2parquet]==1.0.0a2\" \n",
    "!pip install pandas\n",
    "import urllib.request\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d447b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "shutil.os.makedirs(\"tmp/input\", exist_ok=True)\n",
    "urllib.request.urlretrieve(\"https://raw.githubusercontent.com/IBM/data-prep-kit/dev/transforms/language/pdf2parquet/test-data/input/archive1.zip\", \"tmp/input/archive1.zip\")\n",
    "urllib.request.urlretrieve(\"https://raw.githubusercontent.com/IBM/data-prep-kit/dev/transforms/language/pdf2parquet/test-data/input/redp5110-ch1.pdf\", \"tmp/input/redp5110-ch1.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "76469c87-cfd0-4acd-b57f-36edc52018ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dpk_pdf2parquet.transform_python import Pdf2Parquet\n",
    "from dpk_pdf2parquet.transform import pdf2parquet_contents_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "90277e74-f223-434d-b256-94079bd6c5bc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16:06:08 INFO - pdf2parquet parameters are : {'batch_size': -1, 'artifacts_path': None, 'contents_type': <pdf2parquet_contents_types.JSON: 'application/json'>, 'do_table_structure': True, 'do_ocr': True, 'ocr_engine': <pdf2parquet_ocr_engine.EASYOCR: 'easyocr'>, 'bitmap_area_threshold': 0.05, 'pdf_backend': <pdf2parquet_pdf_backend.DLPARSE_V2: 'dlparse_v2'>, 'double_precision': 8}\n",
      "16:06:08 INFO - pipeline id pipeline_id\n",
      "16:06:08 INFO - code location None\n",
      "16:06:08 INFO - data factory data_ is using local data access: input_folder - /tmp/input output_folder - /tmp/output\n",
      "16:06:08 INFO - data factory data_ max_files -1, n_sample -1\n",
      "16:06:08 INFO - data factory data_ Not using data sets, checkpointing False, max files -1, random samples -1, files to use ['.pdf', '.zip'], files to checkpoint ['.parquet']\n",
      "16:06:08 INFO - orchestrator pdf2parquet started at 2025-01-09 16:06:08\n",
      "16:06:08 INFO - Number of files is 2, source profile {'max_file_size': 0.3013172149658203, 'min_file_size': 0.2757863998413086, 'total_file_size': 0.5771036148071289}\n",
      "16:06:08 INFO - Initializing models\n",
      "Fetching 9 files: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 55188.21it/s]\n",
      "16:06:11 INFO - Processing archive_doc_filename='2305.03393v1-pg9.pdf' \n",
      "16:06:12 INFO - Processing archive_doc_filename='2408.09869v1-pg1.pdf' \n",
      "16:06:13 INFO - Completed 1 files (50.0%) in 0.028 min\n",
      "16:06:16 INFO - Completed 2 files (100.0%) in 0.086 min\n",
      "16:06:16 INFO - Done processing 2 files, waiting for flush() completion.\n",
      "16:06:16 INFO - done flushing in 0.0 sec\n",
      "16:06:16 INFO - Completed execution in 0.139 min, execution result 0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Pdf2Parquet(input_folder= \"tmp/input\", \n",
    "               output_folder= \"tmp/output\", \n",
    "               data_files_to_use=['.pdf', '.zip'],\n",
    "               pdf2parquet_contents_type=pdf2parquet_contents_types.JSON).transform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e2bee8da-c566-4e45-bca1-354dfd04b0df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>contents</th>\n",
       "      <th>num_pages</th>\n",
       "      <th>num_tables</th>\n",
       "      <th>num_doc_elements</th>\n",
       "      <th>document_id</th>\n",
       "      <th>document_hash</th>\n",
       "      <th>ext</th>\n",
       "      <th>hash</th>\n",
       "      <th>size</th>\n",
       "      <th>date_acquired</th>\n",
       "      <th>pdf_convert_time</th>\n",
       "      <th>source_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2305.03393v1-pg9.pdf</td>\n",
       "      <td>{\"schema_name\":\"DoclingDocument\",\"version\":\"1....</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>ad108517-c497-43cf-b0a7-6299ff7eb2f8</td>\n",
       "      <td>3463920545297462180</td>\n",
       "      <td>pdf</td>\n",
       "      <td>696963c75a92efe2297cc6d34d2a8078528e2a7d488c24...</td>\n",
       "      <td>39757</td>\n",
       "      <td>2025-01-09T16:06:12.636553</td>\n",
       "      <td>1.184641</td>\n",
       "      <td>archive1.zip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2408.09869v1-pg1.pdf</td>\n",
       "      <td>{\"schema_name\":\"DoclingDocument\",\"version\":\"1....</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>b9843e5e-e1d5-421e-aee4-173fb382d107</td>\n",
       "      <td>582377908831471240</td>\n",
       "      <td>pdf</td>\n",
       "      <td>a3e1b4c0978b2fc9b8f9d215253cc196044e8c046a6bea...</td>\n",
       "      <td>8304</td>\n",
       "      <td>2025-01-09T16:06:13.154192</td>\n",
       "      <td>0.516265</td>\n",
       "      <td>archive1.zip</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               filename                                           contents  \\\n",
       "0  2305.03393v1-pg9.pdf  {\"schema_name\":\"DoclingDocument\",\"version\":\"1....   \n",
       "1  2408.09869v1-pg1.pdf  {\"schema_name\":\"DoclingDocument\",\"version\":\"1....   \n",
       "\n",
       "   num_pages  num_tables  num_doc_elements  \\\n",
       "0          1           1                 9   \n",
       "1          1           0                12   \n",
       "\n",
       "                            document_id        document_hash  ext  \\\n",
       "0  ad108517-c497-43cf-b0a7-6299ff7eb2f8  3463920545297462180  pdf   \n",
       "1  b9843e5e-e1d5-421e-aee4-173fb382d107   582377908831471240  pdf   \n",
       "\n",
       "                                                hash   size  \\\n",
       "0  696963c75a92efe2297cc6d34d2a8078528e2a7d488c24...  39757   \n",
       "1  a3e1b4c0978b2fc9b8f9d215253cc196044e8c046a6bea...   8304   \n",
       "\n",
       "                date_acquired  pdf_convert_time source_filename  \n",
       "0  2025-01-09T16:06:12.636553          1.184641    archive1.zip  \n",
       "1  2025-01-09T16:06:13.154192          0.516265    archive1.zip  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pyarrow.parquet as pq\n",
    "import pandas as pd\n",
    "table = pq.read_table('tmp/output/archive1.parquet')\n",
    "table.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "685229ce-698b-4a97-91f8-b8533337bc0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>contents</th>\n",
       "      <th>num_pages</th>\n",
       "      <th>num_tables</th>\n",
       "      <th>num_doc_elements</th>\n",
       "      <th>document_id</th>\n",
       "      <th>document_hash</th>\n",
       "      <th>ext</th>\n",
       "      <th>hash</th>\n",
       "      <th>size</th>\n",
       "      <th>date_acquired</th>\n",
       "      <th>pdf_convert_time</th>\n",
       "      <th>source_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>redp5110-ch1.pdf</td>\n",
       "      <td>{\"schema_name\":\"DoclingDocument\",\"version\":\"1....</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>3f66ecfd-d644-4f2c-8cc4-e9e111be8383</td>\n",
       "      <td>74198560999363607</td>\n",
       "      <td>pdf</td>\n",
       "      <td>a859e968640a98ebaffd40957d79820e75733b5f1098d0...</td>\n",
       "      <td>35177</td>\n",
       "      <td>2025-01-09T16:06:16.624814</td>\n",
       "      <td>3.465034</td>\n",
       "      <td>redp5110-ch1.pdf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           filename                                           contents  \\\n",
       "0  redp5110-ch1.pdf  {\"schema_name\":\"DoclingDocument\",\"version\":\"1....   \n",
       "\n",
       "   num_pages  num_tables  num_doc_elements  \\\n",
       "0          5           0                48   \n",
       "\n",
       "                            document_id      document_hash  ext  \\\n",
       "0  3f66ecfd-d644-4f2c-8cc4-e9e111be8383  74198560999363607  pdf   \n",
       "\n",
       "                                                hash   size  \\\n",
       "0  a859e968640a98ebaffd40957d79820e75733b5f1098d0...  35177   \n",
       "\n",
       "                date_acquired  pdf_convert_time   source_filename  \n",
       "0  2025-01-09T16:06:16.624814          3.465034  redp5110-ch1.pdf  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = pq.read_table('tmp/output/redp5110-ch1.parquet')\n",
    "table.to_pandas()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}