From 940e2afa230c32a0998cf5ea631473dbd2fe065e Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2024 14:01:02 +0000 Subject: [PATCH 01/18] Added automated infrastructure --- .github/workflows/main.yml | 24 ++ automated/convert.py | 101 ++++++ automated/imviz-profile.ipynb | 636 ++++++++++++++++++++++++++++++++++ 3 files changed, 761 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100644 automated/convert.py create mode 100644 automated/imviz-profile.ipynb diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..8d9a91b --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,24 @@ +name: Automated test + +on: + workflow_dispatch: + pull_request: + push: + +jobs: + test: + runs-on: ubuntu-latest + name: Python 3.12 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Set up dependencies + run: pip install playwright pytest-playwright solara[pytest] nbformat click pytest + - name: Convert notebook to profiling script + run: python convert.py imviz-profile.pynb imviz-profile.py + working-directory: automated + - name: Run profiling script + run: pytest imviz-profile.py + working-directory: automated diff --git a/automated/convert.py b/automated/convert.py new file mode 100644 index 0000000..9919fc9 --- /dev/null +++ b/automated/convert.py @@ -0,0 +1,101 @@ +import os +import click +import nbformat +from textwrap import indent + +def remove_magics(source): + lines = [line for line in source.splitlines() if not line.startswith('%')] + return os.linesep.join(lines) + + +def remove_excludes(source): + lines = [line for line in source.splitlines() if not line.strip().endswith('# EXCLUDE')] + return os.linesep.join(lines) + + + +HEADER = """ +import time +import solara +import playwright +import pytest_playwright +from IPython.display import display + +def watch_screenshot(widget): + display_start = time.time() + last_change_time = display_start + last_screenshot = None + while time.time() - display_start < 5: + screenshot_bytes = widget.screenshot() + if screenshot_bytes != last_screenshot: + last_screenshot = screenshot_bytes + last_change_time = time.time() + return last_screenshot, last_change_time - display_start + +def test_main(page_session, solara_test): + +""" + +DISPLAY_CODE = """ +object_to_capture.add_class("test-object") +display(object_to_capture) +captured_object = page_session.locator(".test-object") +captured_object.wait_for() +""" + +PROFILING_CODE = """ +last_screenshot, time_elapsed = watch_screenshot(captured_object) +print(f"Extra time waiting for display to update: {time_elapsed:.2f}s") +""" + + +@click.command() +@click.argument('input_notebook') +@click.argument('output_script') +def convert(input_notebook, output_script): + + nb = nbformat.read('imviz-profile2.ipynb', as_version=4) + + with open(output_script, 'w') as f: + + f.write(HEADER) + + captured = False + + for icell, cell in enumerate(nb['cells']): + + if cell.cell_type == 'markdown': + f.write(indent(cell.source, ' # ') + '\n\n') + elif cell.cell_type == 'code': + + if cell.source.strip() == '': + continue + + lines = cell.source.splitlines() + + new_lines = [] + + new_lines.append('cell_start = time.time()\n\n') + + for line in lines: + if line.startswith('%') or line.strip().endswith('# EXCLUDE'): + continue + elif line.endswith('# SCREENSHOT'): + new_lines.append('object_to_capture = ' + line) + new_lines.extend(DISPLAY_CODE.splitlines()) + captured = True + else: + new_lines.append(line) + + new_lines.append('cell_end = time.time()\n') + new_lines.append(f'print(f"Cell {icell:2d} Python code executed in {{cell_end - cell_start:.2f}}s")') + + if captured: + new_lines.extend(PROFILING_CODE.splitlines()) + + source = os.linesep.join(new_lines) + + f.write(indent(source, " ") + '\n\n') + +if __name__ == "__main__": + convert() diff --git a/automated/imviz-profile.ipynb b/automated/imviz-profile.ipynb new file mode 100644 index 0000000..6345762 --- /dev/null +++ b/automated/imviz-profile.ipynb @@ -0,0 +1,636 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "52ef14a2-9f4f-4263-b534-515b1af215ab", + "metadata": {}, + "source": [ + "# Imviz profiling" + ] + }, + { + "cell_type": "markdown", + "id": "ad1dc670-0b48-49f7-86db-0c45037b5a6f", + "metadata": {}, + "source": [ + "Create three channels of smoothly varying images:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f676d76e-c33b-4c76-9907-98e5e4164b93", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-12T12:34:55.373191Z", + "iopub.status.busy": "2024-11-12T12:34:55.372801Z", + "iopub.status.idle": "2024-11-12T12:34:55.773270Z", + "shell.execute_reply": "2024-11-12T12:34:55.772727Z", + "shell.execute_reply.started": "2024-11-12T12:34:55.373158Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 341 ms, sys: 51.9 ms, total: 393 ms\n", + "Wall time: 392 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "from time import sleep\n", + "\n", + "import numpy as np\n", + "from scipy.ndimage import gaussian_filter\n", + "from astropy.nddata import NDData\n", + "from astropy.wcs import WCS\n", + "\n", + "from jdaviz import Imviz\n", + "\n", + "shape = (4000, 4000)\n", + "\n", + "wcs = WCS({\n", + " 'CTYPE1': 'RA---TAN', \n", + " 'CUNIT1': 'deg', \n", + " 'CDELT1': 0.01,\n", + " 'CRPIX1': shape[0] / 2, \n", + " 'CRVAL1': 180,\n", + " 'CTYPE2': 'DEC--TAN', \n", + " 'CUNIT2': 'deg', \n", + " 'CDELT2': 0.01,\n", + " 'CRPIX2': shape[1] / 2, \n", + " 'CRVAL2': 0\n", + "})\n", + "\n", + "random_shape = (6, 6)\n", + "image_c = np.random.uniform(size=random_shape)\n", + "image_y = np.random.uniform(size=random_shape)\n", + "image_m = np.random.uniform(size=random_shape)\n", + "\n", + "colors = ['c', 'y', 'm']\n", + "\n", + "def low_pass_filter(x):\n", + " rfft = np.fft.rfft2(x)\n", + " irfft = np.fft.irfft2(rfft, shape)\n", + " return 2**16 * irfft / irfft.max()\n", + "\n", + "nddata_cym = [\n", + " NDData(low_pass_filter(image), wcs=wcs) \n", + " for image in [image_c, image_y, image_m]\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "b86e7b76-d38e-4c62-9c1c-eec252a7d1df", + "metadata": {}, + "source": [ + "Initialize and show Imviz:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ca625d38-000f-4bcb-9df4-bf40b2dfbc57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-12T12:35:33.416916Z", + "iopub.status.busy": "2024-11-12T12:35:33.416709Z", + "iopub.status.idle": "2024-11-12T12:35:34.134737Z", + "shell.execute_reply": "2024-11-12T12:35:34.134237Z", + "shell.execute_reply.started": "2024-11-12T12:35:33.416900Z" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f881f9478b7f4101bba0df444ba7ceb2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Application(config='imviz', docs_link='https://jdaviz.readthedocs.io/en/v4.0.0/imviz/index.html', events=['cal…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 637 ms, sys: 110 ms, total: 747 ms\n", + "Wall time: 714 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "imviz = Imviz()\n", + "label_mouseover = imviz.app.session.application._tools['g-coords-info']\n", + "viewer = imviz.default_viewer._obj \n", + "viewer.figure_widget # SCREENSHOT\n", + "imviz.show(height=800) # EXCLUDE\n", + "# imviz.show('sidecar:split-right', height=800)\n", + "imviz.app.layout.border = '2px solid rgb(143, 56, 201)'" + ] + }, + { + "cell_type": "markdown", + "id": "86a709c4-0112-4909-b21d-52a54cb9a207", + "metadata": {}, + "source": [ + "You can optionally set the compression to one of these two settings:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b9254c25-d29e-4712-be2f-b2a389c92161", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-12T12:04:30.359174Z", + "iopub.status.busy": "2024-11-12T12:04:30.358973Z", + "iopub.status.idle": "2024-11-12T12:04:30.363411Z", + "shell.execute_reply": "2024-11-12T12:04:30.362876Z", + "shell.execute_reply.started": "2024-11-12T12:04:30.359156Z" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "446f1c5c2b674461b6b2a0150556aa40", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Application(config='imviz', docs_link='https://jdaviz.readthedocs.io/en/v4.0.0/imviz/index.html', events=['cal…" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "imviz.app" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "703ad320-cf9f-46f2-a793-5a4fea128f96", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-12T12:07:15.809694Z", + "iopub.status.busy": "2024-11-12T12:07:15.809499Z", + "iopub.status.idle": "2024-11-12T12:07:15.814890Z", + "shell.execute_reply": "2024-11-12T12:07:15.814303Z", + "shell.execute_reply.started": "2024-11-12T12:07:15.809677Z" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a5182fae269540459c1b5d7fe8c4884b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Figure(axes=[Axis(grid_lines='none', label='x', scale=LinearScale(allow_padding=False, max=1.0, min=0.0), side…" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "viewer = imviz.app.get_viewer_by_id('imviz-0')\n", + "viewer.figure_widget" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "b5bb227f-0556-4861-8e3d-0373b0a45cf9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-07T13:04:44.848205Z", + "iopub.status.busy": "2024-11-07T13:04:44.847974Z", + "iopub.status.idle": "2024-11-07T13:04:44.850677Z", + "shell.execute_reply": "2024-11-07T13:04:44.850216Z", + "shell.execute_reply.started": "2024-11-07T13:04:44.848189Z" + } + }, + "outputs": [], + "source": [ + "# imviz.app.get_viewer('imviz-0')._composite_image.compression = 'png'\n", + "\n", + "# imviz.app.get_viewer('imviz-0')._composite_image.compression = 'none'" + ] + }, + { + "cell_type": "markdown", + "id": "b016c45f-3d33-40e7-888d-92ce25a830fc", + "metadata": {}, + "source": [ + "Load the images into Imviz:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "133183f4-b389-4cae-900d-7010c05dae4a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-07T13:04:56.657562Z", + "iopub.status.busy": "2024-11-07T13:04:56.657273Z", + "iopub.status.idle": "2024-11-07T13:04:57.478473Z", + "shell.execute_reply": "2024-11-07T13:04:57.477900Z", + "shell.execute_reply.started": "2024-11-07T13:04:56.657539Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "glue-jupyter compression configuration: 'png'\n", + "CPU times: user 827 ms, sys: 11.8 ms, total: 838 ms\n", + "Wall time: 817 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "with imviz.batch_load():\n", + " for nddata, data_label in zip(nddata_cym, colors):\n", + " imviz.load_data(nddata, data_label=data_label)\n", + "\n", + "compression = imviz.app.get_viewer('imviz-0')._composite_image.compression\n", + "print(f\"glue-jupyter compression configuration: '{compression}'\")" + ] + }, + { + "cell_type": "markdown", + "id": "52615af9-bd32-4d76-90f6-7677d8287198", + "metadata": {}, + "source": [ + "Use WCS to align the images, zoom to fit:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "508fa0c3-0f04-4ffb-a83f-955768070a20", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-07T13:05:07.927083Z", + "iopub.status.busy": "2024-11-07T13:05:07.926542Z", + "iopub.status.idle": "2024-11-07T13:05:10.473757Z", + "shell.execute_reply": "2024-11-07T13:05:10.473231Z", + "shell.execute_reply.started": "2024-11-07T13:05:07.927056Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 3.09 s, sys: 612 ms, total: 3.7 s\n", + "Wall time: 2.54 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "orientation = imviz.plugins['Orientation']\n", + "orientation.align_by = 'WCS'\n", + "viewer.zoom_level = 'fit'" + ] + }, + { + "cell_type": "markdown", + "id": "198cf199-d330-4227-9489-6ad5f1e17afe", + "metadata": {}, + "source": [ + "Rotate the image orientation by 180 deg counter-clockwise:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "8fb46df8-e412-453e-93a0-d2660861942b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-07T13:05:19.008686Z", + "iopub.status.busy": "2024-11-07T13:05:19.008487Z", + "iopub.status.idle": "2024-11-07T13:05:20.754515Z", + "shell.execute_reply": "2024-11-07T13:05:20.753718Z", + "shell.execute_reply.started": "2024-11-07T13:05:19.008669Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 2.04 s, sys: 367 ms, total: 2.41 s\n", + "Wall time: 1.74 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "# rotate the image by 180 deg (CCW):\n", + "orientation.rotation_angle = 180\n", + "orientation.add_orientation()" + ] + }, + { + "cell_type": "markdown", + "id": "9e8d9faf-7fb7-429a-bc70-0019249f1485", + "metadata": {}, + "source": [ + "Assign colors to each of the layers:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "216ea01c-ab6d-4a1a-af63-9ab1070f4a91", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-07T13:05:30.557110Z", + "iopub.status.busy": "2024-11-07T13:05:30.556905Z", + "iopub.status.idle": "2024-11-07T13:05:32.015177Z", + "shell.execute_reply": "2024-11-07T13:05:32.014453Z", + "shell.execute_reply.started": "2024-11-07T13:05:30.557094Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 1.45 s, sys: 16.2 ms, total: 1.47 s\n", + "Wall time: 1.45 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "plot_options = imviz.plugins[\"Plot Options\"]\n", + "for layer, color in zip(plot_options.layer.choices, colors):\n", + " plot_options.layer = layer\n", + " plot_options.image_color_mode = 'Color'\n", + " plot_options.image_color = color\n", + " plot_options.image_opacity = 0.7" + ] + }, + { + "cell_type": "markdown", + "id": "12be3766-0de1-4c29-b8e3-8c7db14d5fde", + "metadata": {}, + "source": [ + "Return to the default orientation:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "dc9b71b1-f201-41bb-a5c2-426701aab578", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-07T13:05:41.715059Z", + "iopub.status.busy": "2024-11-07T13:05:41.714808Z", + "iopub.status.idle": "2024-11-07T13:05:43.267211Z", + "shell.execute_reply": "2024-11-07T13:05:43.266617Z", + "shell.execute_reply.started": "2024-11-07T13:05:41.715042Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 1.81 s, sys: 292 ms, total: 2.1 s\n", + "Wall time: 1.55 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "orientation.orientation = 'Default orientation'" + ] + }, + { + "cell_type": "markdown", + "id": "99a2693f-473b-4dcc-86cd-3d7e395d2563", + "metadata": {}, + "source": [ + "Blink through each layer (one visible at a time):" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "13a2aff2-ac88-4fcb-801c-eec8e7b1b730", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-07T13:05:52.790115Z", + "iopub.status.busy": "2024-11-07T13:05:52.789447Z", + "iopub.status.idle": "2024-11-07T13:05:53.570934Z", + "shell.execute_reply": "2024-11-07T13:05:53.570289Z", + "shell.execute_reply.started": "2024-11-07T13:05:52.790097Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 781 ms, sys: 4.36 ms, total: 786 ms\n", + "Wall time: 777 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "for i in range(len(plot_options.layer.choices)):\n", + " viewer.blink_once()" + ] + }, + { + "cell_type": "markdown", + "id": "c23c7704-47d7-42a2-8d3a-82134d390809", + "metadata": {}, + "source": [ + "Make all layers visible again:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "963c94d3-6956-4b82-8dc0-4d29daf602ea", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-07T13:06:03.359546Z", + "iopub.status.busy": "2024-11-07T13:06:03.359247Z", + "iopub.status.idle": "2024-11-07T13:06:03.779840Z", + "shell.execute_reply": "2024-11-07T13:06:03.779265Z", + "shell.execute_reply.started": "2024-11-07T13:06:03.359528Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 414 ms, sys: 7.44 ms, total: 422 ms\n", + "Wall time: 415 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "for layer in plot_options.layer.choices:\n", + " plot_options.layer = layer\n", + " plot_options.image_visible = True" + ] + }, + { + "cell_type": "markdown", + "id": "26892592-f701-4074-8dab-adb34771b1c8", + "metadata": {}, + "source": [ + "Increment through colormap upper limits for each layer:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "63377d12-4f15-4549-b756-4a8168273499", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-07T13:06:14.512882Z", + "iopub.status.busy": "2024-11-07T13:06:14.512685Z", + "iopub.status.idle": "2024-11-07T13:06:19.962154Z", + "shell.execute_reply": "2024-11-07T13:06:19.961644Z", + "shell.execute_reply.started": "2024-11-07T13:06:14.512865Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 5.5 s, sys: 42.4 ms, total: 5.54 s\n", + "Wall time: 5.45 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "for vmax in np.linspace(0, 2**16, 10):\n", + " for layer in plot_options.layer.choices:\n", + " plot_options.layer = layer\n", + " plot_options.stretch_vmin = 0\n", + " plot_options.stretch_vmax = vmax" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "001420f2-6a73-469a-84a2-669cdaa53101", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1ad34fd-0baf-4698-a2e4-7999e18e9528", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "563897f5-c664-4ae4-a5d0-5d21ab4479fa", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06d6a17d-4ec2-4071-8c17-41587904be41", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "122448e0-3c77-444c-a8bc-950b5081518d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5983836d-4c3b-43d1-af87-2256d46f6878", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c789009-7f2e-4f3e-86b0-2b693ac3b731", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 6604e41ab449ff9fcc99752b784acd703a802d0e Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2024 14:03:44 +0000 Subject: [PATCH 02/18] Fix filename --- automated/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated/convert.py b/automated/convert.py index 9919fc9..d339ab6 100644 --- a/automated/convert.py +++ b/automated/convert.py @@ -54,7 +54,7 @@ def test_main(page_session, solara_test): @click.argument('output_script') def convert(input_notebook, output_script): - nb = nbformat.read('imviz-profile2.ipynb', as_version=4) + nb = nbformat.read(input_notebook, as_version=4) with open(output_script, 'w') as f: From 58823c1c7f9f9b8e63889b50d5b1415842b98f12 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2024 14:06:36 +0000 Subject: [PATCH 03/18] Try with ./ --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8d9a91b..84b5fdf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ jobs: run: pip install playwright pytest-playwright solara[pytest] nbformat click pytest - name: Convert notebook to profiling script run: python convert.py imviz-profile.pynb imviz-profile.py - working-directory: automated + working-directory: ./automated - name: Run profiling script run: pytest imviz-profile.py - working-directory: automated + working-directory: ./automated From c1ac5294d4751200189229158d7c01eaa40c5606 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2024 14:08:19 +0000 Subject: [PATCH 04/18] Fix typo --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 84b5fdf..481d7b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: - name: Set up dependencies run: pip install playwright pytest-playwright solara[pytest] nbformat click pytest - name: Convert notebook to profiling script - run: python convert.py imviz-profile.pynb imviz-profile.py + run: python convert.py imviz-profile.ipynb imviz-profile.py working-directory: ./automated - name: Run profiling script run: pytest imviz-profile.py From 0b4923963de01da2c91c5cde95a97c997204eb6e Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2024 14:10:05 +0000 Subject: [PATCH 05/18] Playwright install --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 481d7b6..03bcef4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,6 +16,8 @@ jobs: python-version: '3.12' - name: Set up dependencies run: pip install playwright pytest-playwright solara[pytest] nbformat click pytest + - name: Initialize playwright + run: playwright install - name: Convert notebook to profiling script run: python convert.py imviz-profile.ipynb imviz-profile.py working-directory: ./automated From 3e5c2f4a13c1106678b401ccbadcd6c16ec7d0ff Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2024 14:12:19 +0000 Subject: [PATCH 06/18] Install scipy --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 03bcef4..d460f4d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: with: python-version: '3.12' - name: Set up dependencies - run: pip install playwright pytest-playwright solara[pytest] nbformat click pytest + run: pip install playwright pytest-playwright solara[pytest] nbformat click pytest scipy - name: Initialize playwright run: playwright install - name: Convert notebook to profiling script From 2321831773b717f81db54f9676dc13ea82041e3c Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2024 14:15:03 +0000 Subject: [PATCH 07/18] More dependencies --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d460f4d..b09f6c8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,8 +14,9 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' + cache: 'pip' - name: Set up dependencies - run: pip install playwright pytest-playwright solara[pytest] nbformat click pytest scipy + run: pip install playwright pytest-playwright solara[pytest] nbformat click pytest scipy astropy jdaviz - name: Initialize playwright run: playwright install - name: Convert notebook to profiling script From 995c073f1e30515a14649a3bddff8ffa0051f706 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2024 14:17:18 +0000 Subject: [PATCH 08/18] Verbose mode --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b09f6c8..f7db9d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,5 +23,5 @@ jobs: run: python convert.py imviz-profile.ipynb imviz-profile.py working-directory: ./automated - name: Run profiling script - run: pytest imviz-profile.py + run: pytest imviz-profile.py -vs working-directory: ./automated From 404fe60e7b20a0bec4f7e550ca496f63da520d46 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2024 14:20:40 +0000 Subject: [PATCH 09/18] Fix indentation --- automated/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated/convert.py b/automated/convert.py index d339ab6..0bec061 100644 --- a/automated/convert.py +++ b/automated/convert.py @@ -30,7 +30,7 @@ def watch_screenshot(widget): if screenshot_bytes != last_screenshot: last_screenshot = screenshot_bytes last_change_time = time.time() - return last_screenshot, last_change_time - display_start + return last_screenshot, last_change_time - display_start def test_main(page_session, solara_test): From 7aed9be8bd1837eb60878e458b23c0baa4fd9f6e Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 Nov 2024 11:15:04 +0000 Subject: [PATCH 10/18] Added infrastructure to run local Jupyter Lab instance and also added pre-commit configuration --- .github/dependabot.yml | 15 +++ .gitignore | 1 + .pre-commit-config.yaml | 29 ++++++ jupyter_output_monitor/__init__.py | 2 + jupyter_output_monitor/_monitor.py | 145 +++++++++++++++++++---------- jupyter_output_monitor/_server.py | 20 ++++ jupyter_output_monitor/_utils.py | 33 +++++++ pyproject.toml | 17 +++- 8 files changed, 208 insertions(+), 54 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .pre-commit-config.yaml create mode 100644 jupyter_output_monitor/_server.py create mode 100644 jupyter_output_monitor/_utils.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..6b0c235 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" # See documentation for possible values + directory: ".github/workflows" # Location of package manifests + schedule: + interval: "weekly" + groups: + actions: + patterns: + - "*" diff --git a/.gitignore b/.gitignore index 0ff5bf4..73489df 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__ dist build .ipynb_checkpoints +__pycache__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..45988c9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +ci: + autofix_prs: false + autoupdate_schedule: 'monthly' + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-added-large-files + args: ["--enforce-all", "--maxkb=300"] + - id: check-case-conflict + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: check-yaml + exclude: ".*(.github.*)$" + - id: detect-private-key + - id: end-of-file-fixer + exclude: ".*(data.*|extern.*|licenses.*|_static.*|_parsetab.py)$" + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.3.4" + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format diff --git a/jupyter_output_monitor/__init__.py b/jupyter_output_monitor/__init__.py index 0069135..598ec7d 100644 --- a/jupyter_output_monitor/__init__.py +++ b/jupyter_output_monitor/__init__.py @@ -1,2 +1,4 @@ from ._monitor import monitor from ._version import __version__ + +__all__ = ["monitor", "__version__"] diff --git a/jupyter_output_monitor/_monitor.py b/jupyter_output_monitor/_monitor.py index 5a08779..09edacc 100644 --- a/jupyter_output_monitor/_monitor.py +++ b/jupyter_output_monitor/_monitor.py @@ -4,57 +4,91 @@ import os import sys +import tempfile import time -import click -import datetime +from io import BytesIO -import numpy as np +import click from PIL import Image from playwright.sync_api import sync_playwright -from io import BytesIO +from ._server import jupyter_server +from ._utils import clear_notebook, isotime RG_SPECIAL = (143, 56) -def isotime(): - return datetime.datetime.now().isoformat() @click.command() -@click.argument('url') -@click.option('--output', default=None, help='Output directory - if not specified, this defaults to output_') -@click.option('--wait-after-execute', default=10, help='Time in s to wait after executing each cell') -@click.option('--headless', is_flag=True, help='Whether to run in headless mode') -def monitor(url, output, wait_after_execute, headless): - +@click.option( + "--notebook", + default=None, + help="The notebook to profile. If specified a local Jupyter Lab instance will be run", +) +@click.option( + "--url", + default=None, + help="The URL hosting the notebook to profile, including any token and notebook path.", +) +@click.option( + "--output", + default=None, + help="Output directory - if not specified, this defaults to output_", +) +@click.option( + "--wait-after-execute", + default=10, + help="Time in s to wait after executing each cell", +) +@click.option("--headless", is_flag=True, help="Whether to run in headless mode") +def monitor(notebook, url, output, wait_after_execute, headless): if output is None: - output = f'output-{isotime()}' + output = f"output-{isotime()}" if os.path.exists(output): - print('Output directory {output} already exists') + print("Output directory {output} already exists") sys.exit(1) os.makedirs(output) + if notebook is None and url is None: + print("Either --notebook or --url should be specified") + sys.exit(1) + elif notebook is not None and url is not None: + print("Only one of --notebook or --url should be specified") + sys.exit(1) + elif notebook is not None: + # Create a temporary directory with a clean version of the notebook + notebook_dir = tempfile.mkdtemp() + clear_notebook(notebook, os.path.join(notebook_dir, "notebook.ipynb")) + with jupyter_server(notebook_dir) as server: + url = server.base_url + "/lab/tree/notebook.ipynb" + _monitor_output(url, output, wait_after_execute, headless) + else: + _monitor_output(url, output, wait_after_execute, headless) + + +def _monitor_output(url, output, wait_after_execute, headless): # Index of the current last screenshot, by output index last_screenshot = {} - with sync_playwright() as p, open(os.path.join(output, 'event_log.csv'), 'w') as log: - - log.write('time,event,index,screenshot\n') + with ( + sync_playwright() as p, + open(os.path.join(output, "event_log.csv"), "w") as log, + ): + log.write("time,event,index,screenshot\n") log.flush() # Launch browser and open URL browser = p.firefox.launch(headless=headless) - page = browser.new_page(viewport={'width':2000, 'height':10000}) + page = browser.new_page(viewport={"width": 2000, "height": 10000}) page.goto(url) while True: - - print('Checking for input cells') + print("Checking for input cells") # Construct list of input and output cells in the notebook - input_cells = list(page.query_selector_all('.jp-InputArea-editor')) + input_cells = list(page.query_selector_all(".jp-InputArea-editor")) # Keep only input cells that are visible input_cells = [cell for cell in input_cells if cell.is_visible()] @@ -62,23 +96,22 @@ def monitor(url, output, wait_after_execute, headless): if len(input_cells) > 0: break - print('-> No input cells found, waiting before checking again') + print("-> No input cells found, waiting before checking again") # If no visible input cells, wait and try again page.wait_for_timeout(1000) - print(f'{len(input_cells)} input cells found') + print(f"{len(input_cells)} input cells found") last_screenshot = {} # Now loop over each input cell and execute for input_index, input_cell in enumerate(input_cells): - - if input_cell.text_content().strip() == '': - print(f'Skipping empty input cell {input_index}') + if input_cell.text_content().strip() == "": + print(f"Skipping empty input cell {input_index}") continue - print(f'Execute input cell {input_index}') + print(f"Execute input cell {input_index}") # Take screenshot before we start executing cell but save it after screenshot_bytes = input_cell.screenshot() @@ -87,48 +120,51 @@ def monitor(url, output, wait_after_execute, headless): input_cell.click() # Execute it - page.keyboard.press('Shift+Enter') + page.keyboard.press("Shift+Enter") timestamp = isotime() - screenshot_filename = os.path.join(output, f'input-{input_index:03d}-{timestamp}.png') + screenshot_filename = os.path.join( + output, + f"input-{input_index:03d}-{timestamp}.png", + ) image = Image.open(BytesIO(screenshot_bytes)) image.save(screenshot_filename) - log.write(f'{timestamp},execute-input,{input_index},{screenshot_filename}\n') + log.write( + f"{timestamp},execute-input,{input_index},{screenshot_filename}\n", + ) # Now loop and check for changes in any of the output cells - if a cell # output changes, save a screenshot - print('Watching for changes in output cells') + print("Watching for changes in output cells") start = time.time() while time.time() - start < wait_after_execute: - - output_cells = list(page.query_selector_all('.jp-OutputArea-output')) + output_cells = list(page.query_selector_all(".jp-OutputArea-output")) for output_cell in output_cells: - if not output_cell.is_visible(): continue # The element we are interested in is one level down - div = output_cell.query_selector('div') + div = output_cell.query_selector("div") if div is None: continue - style = div.get_attribute('style') + style = div.get_attribute("style") - if style is None or 'border-color: rgb(' not in style: + if style is None or "border-color: rgb(" not in style: continue # Parse rgb values for border - start_pos = style.index('border-color:') - start_pos = style.index('(', start_pos) + 1 - end_pos = style.index(')', start_pos) - r, g, b = [int(x) for x in style[start_pos:end_pos].split(',')] + start_pos = style.index("border-color:") + start_pos = style.index("(", start_pos) + 1 + end_pos = style.index(")", start_pos) + r, g, b = (int(x) for x in style[start_pos:end_pos].split(",")) # The (r,g) pair is chosen to be random and unlikely to # happen by chance on the page. If this values don't match, we @@ -142,30 +178,39 @@ def monitor(url, output, wait_after_execute, headless): # which should be sufficient output_index = b - print(f'- taking screenshot of output cell {output_index}') + print(f"- taking screenshot of output cell {output_index}") screenshot_bytes = div.screenshot() # If screenshot didn't exist before for this cell or if it has # changed, we save it to a file and keep track of it. - if output_index not in last_screenshot or last_screenshot[output_index] != screenshot_bytes: - - print(f' -> change detected!') + if ( + output_index not in last_screenshot + or last_screenshot[output_index] != screenshot_bytes + ): + print(" -> change detected!") timestamp = isotime() - screenshot_filename = os.path.join(output, f'output-{output_index:03d}-{timestamp}.png') + screenshot_filename = os.path.join( + output, + f"output-{output_index:03d}-{timestamp}.png", + ) image = Image.open(BytesIO(screenshot_bytes)) image.save(screenshot_filename) - log.write(f'{timestamp},output-changed,{output_index},{screenshot_filename}\n') + log.write( + f"{timestamp},output-changed,{output_index},{screenshot_filename}\n", + ) log.flush() - print(f"Saving screenshot of output {output_index} at {timestamp}") + print( + f"Saving screenshot of output {output_index} at {timestamp}", + ) last_screenshot[output_index] = screenshot_bytes - print('Stopping monitoring output and moving on to next input cell') + print("Stopping monitoring output and moving on to next input cell") -if __name__ == '__main__': +if __name__ == "__main__": monitor() diff --git a/jupyter_output_monitor/_server.py b/jupyter_output_monitor/_server.py new file mode 100644 index 0000000..4881aea --- /dev/null +++ b/jupyter_output_monitor/_server.py @@ -0,0 +1,20 @@ +from contextlib import contextmanager + +from solara.test.pytest_plugin import ( + ServerJupyter, +) + +from ._utils import get_free_port + +__all__ = ["jupyter_server"] + + +@contextmanager +def jupyter_server(notebook_path): + server = ServerJupyter(notebook_path, get_free_port(), "localhost") + try: + server.serve_threaded() + server.wait_until_serving() + yield server + finally: + server.stop_serving() diff --git a/jupyter_output_monitor/_utils.py b/jupyter_output_monitor/_utils.py new file mode 100644 index 0000000..bc2888a --- /dev/null +++ b/jupyter_output_monitor/_utils.py @@ -0,0 +1,33 @@ +import datetime +import socket + +from nbconvert import NotebookExporter +from traitlets.config import Config + +__all__ = ["get_free_port", "clear_notebook", "isotime"] + + +def get_free_port(): + """Return a free port number.""" + sock = socket.socket() + sock.bind(("", 0)) + return sock.getsockname()[1] + + +def clear_notebook(input_notebook, output_notebook): + """Write out a copy of the notebook with output and metadata removed.""" + c = Config() + c.NotebookExporter.preprocessors = [ + "nbconvert.preprocessors.ClearOutputPreprocessor", + "nbconvert.preprocessors.ClearMetadataPreprocessor", + ] + + exporter = NotebookExporter(config=c) + body, resources = exporter.from_filename(input_notebook) + + with open(output_notebook, "w") as f: + f.write(body) + + +def isotime(): + return datetime.datetime.now().isoformat() diff --git a/pyproject.toml b/pyproject.toml index 9a42fe5..244a28f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,8 +38,17 @@ find = {namespaces = false} write_to = "jupyter_output_monitor/_version.py" [tool.ruff] -lint.select = [ - "B", # flake8-bugbear - "I", # isort - "UP", # pyupgrade +lint.select = ["ALL"] +lint.ignore = [ + "A00", + "ANN", + "T201", + "PTH", + "D100", + "D103", + "D104", + "C901", + "PLR0915", + "DTZ", + "E501" ] From 720b34385cc0578b8eebff7a9f3d51b04e3b2558 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 Nov 2024 11:19:04 +0000 Subject: [PATCH 11/18] Updated CI to test jupyter-output-monitor --- .github/workflows/main.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f7db9d8..dfe9d53 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ on: push: jobs: - test: + test_convert: runs-on: ubuntu-latest name: Python 3.12 steps: @@ -25,3 +25,24 @@ jobs: - name: Run profiling script run: pytest imviz-profile.py -vs working-directory: ./automated + test_main: + runs-on: ubuntu-latest + name: Python 3.12 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - name: Set up dependencies + run: pip install . + - name: Initialize playwright + run: playwright install + - name: Run monitoring command + run: jupyter-output-monitor --notebook automated/imviz-profile.ipynb --headless + - name: Convert notebook to profiling script + run: python convert.py imviz-profile.ipynb imviz-profile.py + working-directory: ./automated + - name: Run profiling script + run: pytest imviz-profile.py -vs + working-directory: ./automated From 8e2a3dc0824efd7e501fc42b935c8a0f5e078dfa Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 Nov 2024 11:20:35 +0000 Subject: [PATCH 12/18] Added dependency --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 244a28f..acf5543 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,8 @@ dependencies = [ "numpy>=1.23", "click", "pillow", - "playwright" + "playwright", + "solara[pytest]" ] dynamic = ["version"] From b17d0b38d28ab463f63d6d5c124a3534f28167b9 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 Nov 2024 13:09:39 +0000 Subject: [PATCH 13/18] Added proper testing infrastructure and first test --- .github/workflows/main.yml | 29 +- automated/imviz-profile.ipynb | 636 ------------------ jupyter_output_monitor/__main__.py | 4 + .../__pycache__/__init__.cpython-311.pyc | Bin 303 -> 0 bytes .../__pycache__/_monitor.cpython-311.pyc | Bin 7803 -> 0 bytes .../__pycache__/_version.cpython-311.pyc | Bin 690 -> 0 bytes .../_convert.py | 52 +- jupyter_output_monitor/_monitor.py | 17 +- jupyter_output_monitor/_version.py | 15 + jupyter_output_monitor/tests/__init__.py | 0 .../tests/data/simple.ipynb | 97 +++ jupyter_output_monitor/tests/test_monitor.py | 42 ++ pyproject.toml | 14 +- tox.ini | 14 + 14 files changed, 223 insertions(+), 697 deletions(-) delete mode 100644 automated/imviz-profile.ipynb create mode 100644 jupyter_output_monitor/__main__.py delete mode 100644 jupyter_output_monitor/__pycache__/__init__.cpython-311.pyc delete mode 100644 jupyter_output_monitor/__pycache__/_monitor.cpython-311.pyc delete mode 100644 jupyter_output_monitor/__pycache__/_version.cpython-311.pyc rename automated/convert.py => jupyter_output_monitor/_convert.py (61%) create mode 100644 jupyter_output_monitor/_version.py create mode 100644 jupyter_output_monitor/tests/__init__.py create mode 100644 jupyter_output_monitor/tests/data/simple.ipynb create mode 100644 jupyter_output_monitor/tests/test_monitor.py create mode 100644 tox.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dfe9d53..d2a3162 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,28 +6,9 @@ on: push: jobs: - test_convert: + test: runs-on: ubuntu-latest - name: Python 3.12 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - cache: 'pip' - - name: Set up dependencies - run: pip install playwright pytest-playwright solara[pytest] nbformat click pytest scipy astropy jdaviz - - name: Initialize playwright - run: playwright install - - name: Convert notebook to profiling script - run: python convert.py imviz-profile.ipynb imviz-profile.py - working-directory: ./automated - - name: Run profiling script - run: pytest imviz-profile.py -vs - working-directory: ./automated - test_main: - runs-on: ubuntu-latest - name: Python 3.12 + name: jupyter-output-monitor steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -40,9 +21,3 @@ jobs: run: playwright install - name: Run monitoring command run: jupyter-output-monitor --notebook automated/imviz-profile.ipynb --headless - - name: Convert notebook to profiling script - run: python convert.py imviz-profile.ipynb imviz-profile.py - working-directory: ./automated - - name: Run profiling script - run: pytest imviz-profile.py -vs - working-directory: ./automated diff --git a/automated/imviz-profile.ipynb b/automated/imviz-profile.ipynb deleted file mode 100644 index 6345762..0000000 --- a/automated/imviz-profile.ipynb +++ /dev/null @@ -1,636 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "52ef14a2-9f4f-4263-b534-515b1af215ab", - "metadata": {}, - "source": [ - "# Imviz profiling" - ] - }, - { - "cell_type": "markdown", - "id": "ad1dc670-0b48-49f7-86db-0c45037b5a6f", - "metadata": {}, - "source": [ - "Create three channels of smoothly varying images:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "f676d76e-c33b-4c76-9907-98e5e4164b93", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-12T12:34:55.373191Z", - "iopub.status.busy": "2024-11-12T12:34:55.372801Z", - "iopub.status.idle": "2024-11-12T12:34:55.773270Z", - "shell.execute_reply": "2024-11-12T12:34:55.772727Z", - "shell.execute_reply.started": "2024-11-12T12:34:55.373158Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 341 ms, sys: 51.9 ms, total: 393 ms\n", - "Wall time: 392 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "from time import sleep\n", - "\n", - "import numpy as np\n", - "from scipy.ndimage import gaussian_filter\n", - "from astropy.nddata import NDData\n", - "from astropy.wcs import WCS\n", - "\n", - "from jdaviz import Imviz\n", - "\n", - "shape = (4000, 4000)\n", - "\n", - "wcs = WCS({\n", - " 'CTYPE1': 'RA---TAN', \n", - " 'CUNIT1': 'deg', \n", - " 'CDELT1': 0.01,\n", - " 'CRPIX1': shape[0] / 2, \n", - " 'CRVAL1': 180,\n", - " 'CTYPE2': 'DEC--TAN', \n", - " 'CUNIT2': 'deg', \n", - " 'CDELT2': 0.01,\n", - " 'CRPIX2': shape[1] / 2, \n", - " 'CRVAL2': 0\n", - "})\n", - "\n", - "random_shape = (6, 6)\n", - "image_c = np.random.uniform(size=random_shape)\n", - "image_y = np.random.uniform(size=random_shape)\n", - "image_m = np.random.uniform(size=random_shape)\n", - "\n", - "colors = ['c', 'y', 'm']\n", - "\n", - "def low_pass_filter(x):\n", - " rfft = np.fft.rfft2(x)\n", - " irfft = np.fft.irfft2(rfft, shape)\n", - " return 2**16 * irfft / irfft.max()\n", - "\n", - "nddata_cym = [\n", - " NDData(low_pass_filter(image), wcs=wcs) \n", - " for image in [image_c, image_y, image_m]\n", - "]" - ] - }, - { - "cell_type": "markdown", - "id": "b86e7b76-d38e-4c62-9c1c-eec252a7d1df", - "metadata": {}, - "source": [ - "Initialize and show Imviz:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "ca625d38-000f-4bcb-9df4-bf40b2dfbc57", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-12T12:35:33.416916Z", - "iopub.status.busy": "2024-11-12T12:35:33.416709Z", - "iopub.status.idle": "2024-11-12T12:35:34.134737Z", - "shell.execute_reply": "2024-11-12T12:35:34.134237Z", - "shell.execute_reply.started": "2024-11-12T12:35:33.416900Z" - } - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f881f9478b7f4101bba0df444ba7ceb2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Application(config='imviz', docs_link='https://jdaviz.readthedocs.io/en/v4.0.0/imviz/index.html', events=['cal…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 637 ms, sys: 110 ms, total: 747 ms\n", - "Wall time: 714 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "imviz = Imviz()\n", - "label_mouseover = imviz.app.session.application._tools['g-coords-info']\n", - "viewer = imviz.default_viewer._obj \n", - "viewer.figure_widget # SCREENSHOT\n", - "imviz.show(height=800) # EXCLUDE\n", - "# imviz.show('sidecar:split-right', height=800)\n", - "imviz.app.layout.border = '2px solid rgb(143, 56, 201)'" - ] - }, - { - "cell_type": "markdown", - "id": "86a709c4-0112-4909-b21d-52a54cb9a207", - "metadata": {}, - "source": [ - "You can optionally set the compression to one of these two settings:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "b9254c25-d29e-4712-be2f-b2a389c92161", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-12T12:04:30.359174Z", - "iopub.status.busy": "2024-11-12T12:04:30.358973Z", - "iopub.status.idle": "2024-11-12T12:04:30.363411Z", - "shell.execute_reply": "2024-11-12T12:04:30.362876Z", - "shell.execute_reply.started": "2024-11-12T12:04:30.359156Z" - } - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "446f1c5c2b674461b6b2a0150556aa40", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Application(config='imviz', docs_link='https://jdaviz.readthedocs.io/en/v4.0.0/imviz/index.html', events=['cal…" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "imviz.app" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "703ad320-cf9f-46f2-a793-5a4fea128f96", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-12T12:07:15.809694Z", - "iopub.status.busy": "2024-11-12T12:07:15.809499Z", - "iopub.status.idle": "2024-11-12T12:07:15.814890Z", - "shell.execute_reply": "2024-11-12T12:07:15.814303Z", - "shell.execute_reply.started": "2024-11-12T12:07:15.809677Z" - } - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a5182fae269540459c1b5d7fe8c4884b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Figure(axes=[Axis(grid_lines='none', label='x', scale=LinearScale(allow_padding=False, max=1.0, min=0.0), side…" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "viewer = imviz.app.get_viewer_by_id('imviz-0')\n", - "viewer.figure_widget" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "b5bb227f-0556-4861-8e3d-0373b0a45cf9", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-07T13:04:44.848205Z", - "iopub.status.busy": "2024-11-07T13:04:44.847974Z", - "iopub.status.idle": "2024-11-07T13:04:44.850677Z", - "shell.execute_reply": "2024-11-07T13:04:44.850216Z", - "shell.execute_reply.started": "2024-11-07T13:04:44.848189Z" - } - }, - "outputs": [], - "source": [ - "# imviz.app.get_viewer('imviz-0')._composite_image.compression = 'png'\n", - "\n", - "# imviz.app.get_viewer('imviz-0')._composite_image.compression = 'none'" - ] - }, - { - "cell_type": "markdown", - "id": "b016c45f-3d33-40e7-888d-92ce25a830fc", - "metadata": {}, - "source": [ - "Load the images into Imviz:" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "133183f4-b389-4cae-900d-7010c05dae4a", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-07T13:04:56.657562Z", - "iopub.status.busy": "2024-11-07T13:04:56.657273Z", - "iopub.status.idle": "2024-11-07T13:04:57.478473Z", - "shell.execute_reply": "2024-11-07T13:04:57.477900Z", - "shell.execute_reply.started": "2024-11-07T13:04:56.657539Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "glue-jupyter compression configuration: 'png'\n", - "CPU times: user 827 ms, sys: 11.8 ms, total: 838 ms\n", - "Wall time: 817 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "with imviz.batch_load():\n", - " for nddata, data_label in zip(nddata_cym, colors):\n", - " imviz.load_data(nddata, data_label=data_label)\n", - "\n", - "compression = imviz.app.get_viewer('imviz-0')._composite_image.compression\n", - "print(f\"glue-jupyter compression configuration: '{compression}'\")" - ] - }, - { - "cell_type": "markdown", - "id": "52615af9-bd32-4d76-90f6-7677d8287198", - "metadata": {}, - "source": [ - "Use WCS to align the images, zoom to fit:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "508fa0c3-0f04-4ffb-a83f-955768070a20", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-07T13:05:07.927083Z", - "iopub.status.busy": "2024-11-07T13:05:07.926542Z", - "iopub.status.idle": "2024-11-07T13:05:10.473757Z", - "shell.execute_reply": "2024-11-07T13:05:10.473231Z", - "shell.execute_reply.started": "2024-11-07T13:05:07.927056Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3.09 s, sys: 612 ms, total: 3.7 s\n", - "Wall time: 2.54 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "orientation = imviz.plugins['Orientation']\n", - "orientation.align_by = 'WCS'\n", - "viewer.zoom_level = 'fit'" - ] - }, - { - "cell_type": "markdown", - "id": "198cf199-d330-4227-9489-6ad5f1e17afe", - "metadata": {}, - "source": [ - "Rotate the image orientation by 180 deg counter-clockwise:" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "8fb46df8-e412-453e-93a0-d2660861942b", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-07T13:05:19.008686Z", - "iopub.status.busy": "2024-11-07T13:05:19.008487Z", - "iopub.status.idle": "2024-11-07T13:05:20.754515Z", - "shell.execute_reply": "2024-11-07T13:05:20.753718Z", - "shell.execute_reply.started": "2024-11-07T13:05:19.008669Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 2.04 s, sys: 367 ms, total: 2.41 s\n", - "Wall time: 1.74 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "# rotate the image by 180 deg (CCW):\n", - "orientation.rotation_angle = 180\n", - "orientation.add_orientation()" - ] - }, - { - "cell_type": "markdown", - "id": "9e8d9faf-7fb7-429a-bc70-0019249f1485", - "metadata": {}, - "source": [ - "Assign colors to each of the layers:" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "216ea01c-ab6d-4a1a-af63-9ab1070f4a91", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-07T13:05:30.557110Z", - "iopub.status.busy": "2024-11-07T13:05:30.556905Z", - "iopub.status.idle": "2024-11-07T13:05:32.015177Z", - "shell.execute_reply": "2024-11-07T13:05:32.014453Z", - "shell.execute_reply.started": "2024-11-07T13:05:30.557094Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.45 s, sys: 16.2 ms, total: 1.47 s\n", - "Wall time: 1.45 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "plot_options = imviz.plugins[\"Plot Options\"]\n", - "for layer, color in zip(plot_options.layer.choices, colors):\n", - " plot_options.layer = layer\n", - " plot_options.image_color_mode = 'Color'\n", - " plot_options.image_color = color\n", - " plot_options.image_opacity = 0.7" - ] - }, - { - "cell_type": "markdown", - "id": "12be3766-0de1-4c29-b8e3-8c7db14d5fde", - "metadata": {}, - "source": [ - "Return to the default orientation:" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "dc9b71b1-f201-41bb-a5c2-426701aab578", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-07T13:05:41.715059Z", - "iopub.status.busy": "2024-11-07T13:05:41.714808Z", - "iopub.status.idle": "2024-11-07T13:05:43.267211Z", - "shell.execute_reply": "2024-11-07T13:05:43.266617Z", - "shell.execute_reply.started": "2024-11-07T13:05:41.715042Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.81 s, sys: 292 ms, total: 2.1 s\n", - "Wall time: 1.55 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "orientation.orientation = 'Default orientation'" - ] - }, - { - "cell_type": "markdown", - "id": "99a2693f-473b-4dcc-86cd-3d7e395d2563", - "metadata": {}, - "source": [ - "Blink through each layer (one visible at a time):" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "13a2aff2-ac88-4fcb-801c-eec8e7b1b730", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-07T13:05:52.790115Z", - "iopub.status.busy": "2024-11-07T13:05:52.789447Z", - "iopub.status.idle": "2024-11-07T13:05:53.570934Z", - "shell.execute_reply": "2024-11-07T13:05:53.570289Z", - "shell.execute_reply.started": "2024-11-07T13:05:52.790097Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 781 ms, sys: 4.36 ms, total: 786 ms\n", - "Wall time: 777 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "for i in range(len(plot_options.layer.choices)):\n", - " viewer.blink_once()" - ] - }, - { - "cell_type": "markdown", - "id": "c23c7704-47d7-42a2-8d3a-82134d390809", - "metadata": {}, - "source": [ - "Make all layers visible again:" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "963c94d3-6956-4b82-8dc0-4d29daf602ea", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-07T13:06:03.359546Z", - "iopub.status.busy": "2024-11-07T13:06:03.359247Z", - "iopub.status.idle": "2024-11-07T13:06:03.779840Z", - "shell.execute_reply": "2024-11-07T13:06:03.779265Z", - "shell.execute_reply.started": "2024-11-07T13:06:03.359528Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 414 ms, sys: 7.44 ms, total: 422 ms\n", - "Wall time: 415 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "for layer in plot_options.layer.choices:\n", - " plot_options.layer = layer\n", - " plot_options.image_visible = True" - ] - }, - { - "cell_type": "markdown", - "id": "26892592-f701-4074-8dab-adb34771b1c8", - "metadata": {}, - "source": [ - "Increment through colormap upper limits for each layer:" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "63377d12-4f15-4549-b756-4a8168273499", - "metadata": { - "execution": { - "iopub.execute_input": "2024-11-07T13:06:14.512882Z", - "iopub.status.busy": "2024-11-07T13:06:14.512685Z", - "iopub.status.idle": "2024-11-07T13:06:19.962154Z", - "shell.execute_reply": "2024-11-07T13:06:19.961644Z", - "shell.execute_reply.started": "2024-11-07T13:06:14.512865Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 5.5 s, sys: 42.4 ms, total: 5.54 s\n", - "Wall time: 5.45 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "for vmax in np.linspace(0, 2**16, 10):\n", - " for layer in plot_options.layer.choices:\n", - " plot_options.layer = layer\n", - " plot_options.stretch_vmin = 0\n", - " plot_options.stretch_vmax = vmax" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "001420f2-6a73-469a-84a2-669cdaa53101", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e1ad34fd-0baf-4698-a2e4-7999e18e9528", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "563897f5-c664-4ae4-a5d0-5d21ab4479fa", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "06d6a17d-4ec2-4071-8c17-41587904be41", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "122448e0-3c77-444c-a8bc-950b5081518d", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5983836d-4c3b-43d1-af87-2256d46f6878", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4c789009-7f2e-4f3e-86b0-2b693ac3b731", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/jupyter_output_monitor/__main__.py b/jupyter_output_monitor/__main__.py new file mode 100644 index 0000000..12c22cd --- /dev/null +++ b/jupyter_output_monitor/__main__.py @@ -0,0 +1,4 @@ +from ._monitor import monitor + +if __name__ == "__main__": + monitor() diff --git a/jupyter_output_monitor/__pycache__/__init__.cpython-311.pyc b/jupyter_output_monitor/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 49610c79e1b274a53b9e54cb1e05ceaf30a18071..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 303 zcmZvX!Ab)`42Cm13oeyDf`Z2d>%2e_pFj%UdI?K6>MlE*B|B5752BACzKO>kORt{1 z3BC1X7Pbe$5R&gF2}$0k(+Q}3zFt4wY5p3-$&wKcmkMuCLTMWCW@*g8Nt1ZHw957Y z^7PwiQ*QQWp4%$0D4rv(N&1Rg#QQwmXE@fxYOl(%Dyhm3IB109!acJm7udUna80Yw zySgma)yu3Sd1UcLyP}z=qK`tZj@>4ZAji2orrR8|x#u!AbSuNF!)IJDfQ)bLPyMGiT18GkjE4Wy9dfVEvK*Y{antLY2Z}%>&;ha18qpqc94O zVpF*ipTf1eValMD#wnv#nx;%zX`ZTp(h#j&uuNHSG{zXUF4(4Q8f=Q%7lan4bg`bz-5DpWA7uclC?D!&ANzJg!w(RynOHOfNal=ZRgOC#XG@0B=q zilD0g1Wq;OplTl1>giCD zq0kQH+&-M%7wdi#0D9Ka8q=ly2-f3FV!im4UZZNELIN!?3$$vCvCFEJ;n^9MTL=k% zk9~(my}UXw%P!CZ0=qD9m1F1Vu)q(@QK2O!;fvFph8qjhz6CnOFLLz2++rM-$pzU( zA-*UC7uXmhu-rfa@W)rxN|*!9$RTa2gCE}k*&6oTUZ>bww(VrbPRjOn#ooSc@6Xu# zWxHRo`z4c~s{%aLxX48l7N0LSL*l}OHWNuP9Pq|+tE7))X2=*TkbImDGcycL^^?LZ z!;=&}6IzT4JSnhbp1%>q2rq;d;-`8IYGocnHOSz4H-Im86F z7(+A)o#X;b(TEL(56(nG5zYxyg-wtZ9s&SOMO5J8LrlciYAIYOQ3?Gfe}X+UBypic z9p?7oG@^6h5pn(5uDJpzp;o!CoXWS5bf|;{m*q+2$$>c;&O<7=tBE!oIC2 zSK8xP(kvj2$v>}InPxEyIE7NgDbts*`mgpSK$-P*pk5(Xq)S#*7z-<}C|?m^qTM*~ z=yw3eMDrX{TkT0#7ogHoG(Ik~n8`}9GQDqWOL4(^S}EHD!(+RipQJ^!r1vdw=?D?i zW8#<<)9q;WBviENC^)-RJDGTzLvC7j(eDBJy&LQl2)O+NJ_NoM^QENnyzzr z?68i*T0I`f)RJ};=@cQ#S&q4}R7&Z`X8#1E>K-1@Z%rTjiZhh+d=5()-&wi?I+e5~ z?V>&1P{b`ldYwqacl_X^$Dv&LFHc=5|5UwbLudNHD$nY$8eWYB+_?|VTYkop1mHFO zX*{$GbDjcTy14+AmQ-Uor?0nF$*P}un%$y}YWl=PdBiH%yXJ>wr#V?I3>A5q+qYuE zaIsY+^!({-Nr&i2w-&LBkXWYuAdA=74UM-Jv>((Vu^RH$hX*W)ZtuK$9>Bt$F3vmW zEx>whN!I)nEU_j5mQgg!YeZ58&ia#1I;rPKUrW}CwQvUTcb+AkqBBhv_$w{-yz=;l z8!+2SF$yr%Rz|%}tXso3j)-;X_5x06Nwt^ZyF}N!CaObpE#Vx#aY=NgOKe$jB-L3q zuKu-{hsApRSAsGofKc-FBf82*Bpax1u!b(NUTpYlL-A;(JBu6@A?lExw{jbL2fn1q z#-uyhBsPlf8G~@KICft9%EHS3U2IBvbb8ezdR}EYK(as0CU`{C63!E%N%Ry~AQJf* z|4lbnuYLuf!>{#Fe)g%yMbH0}Q~w5%`0G#oPxT*klI_!I_9LtE&{QsEQQMsnP=^%0 z@gKj$B0RC`X7iQLkTPzpSW&D9ul`ivSj^sm9mROlvSGz|3tPr<>=uT@`)X_oo5Ws+ z521F%76mY`L%M2yVsui!_85Rw8eFmHocVTs|H=_raE4E`gk3B?>d@^VR% zLk{Ol+ej4T6J{aENi^ppGXF@F4TqxqDF6zcNOgGRpsit&=W;Hk&pkc%&!_+Cv^;!C z8NLK(Soklu_H;K+`_VoBD?s4UNZ9k%$z0lMd&sAcezf#pX}?L^>O_Z+ z6hfNb7f%q$&XkIYOt?tUg`5l_;uat%r>V9?O@B^YzFZI0pJm9bQJ`6ajzbVmLJ%1xScz(- zV~cPX0=Fy%11SaqfB^SwanA2(`<;N&`RJ^fkU5t4ylo9 z;zLWo3L?%OLJie=PSmOhEd)aXke-Ii4Ara|M3^fc1u@vJpASx69X&gC=2CAnhr&s< zBCq96jdAUcBbYx!9P%7gM>NC>!4f9%I4qTosFl+kyUf!Z5uAR7`W}FI@+~l@m zk6KeC6r4u)WvUgeMZ2L<>qB0|Q_y(hQCLI1{?Y8KTA5c)js|QL0P>3!=s@jNW0vZ*yV=^V zhl3vtJs6T|yCAJs>{eRuE&t8k-``Em$d!kb%0qjY!R&f&sgkNs$d)0+G9+1so?Gl2 zmQ?K@>>KuV`>v_t-gv4)HZ>}yM#`eOR? z$fM|I(I!2@7;E`XPoU@7bIu9>^!bGkFQr|9Sz$KZ^q$$()o40)O#K!hgWu7P#hQ5O<4

d~RVcFqT9Nu(W#?c`;IS4rM69~KAuamC7@UWk|nU?X<2sw zXFW%sG|Sz;Qo4V&huy)g*YIz1g-mp;k8gT*98KGfu8gBgcJwHYo^6Lem2x_)~fmfNS5_GyXmK6f?UzxZ%j>K>H6Clv1q*)^oN zhSo>3MD6<*-@7O^z9AER3ehJKeLIBX{jvAPHk+h|w;`dmp;9_?BM&sZEfcpC;+90* z`rd|BIVI-_nHW-tA&D5;b-C}4eK3B1e0}t}v*CX42YvVZ(q`FtSn4^Mah{Z&CwILa z>2uqi!b-F!6Dd{l0JOKE;< z55w!*vThRe6*@e}GM-~kZpxlv#WS2byVK}N{SG>JTD;pWzD$d6DW; zm6Uo$z8;l&#vmaQ6Bl~CetqGYNQOJ?hSe87jHJp9YB$LAmIs7yvlc%A*wd2$I zkLR~KwpQiVA*FRFbv~|+;QaQ$<;=k4XJL8Zx-xJbzN2#Mq|!S1 z!THpg)NeQ2_6V%Wvw1YN^o1wW*e5miAr@>{=i$_)U00Ljx$-pu86?>?p|~a_*Thcu z8;|aOc318mRJsQxS0}95-MMvIYCH+)Df=v{oVqOq*~}?cI>qj~4@s@ZWp_Yv2PAi3 zx3ll>#y%h0v_AJAc^vs;^dF*61JYUc*&WF8=`WdmSoX&ie_V3+Jtq%;Zu`eU>8-PJ z-#Ml4oJ@`?pt|U?PHtN5s=-3ihEFU5AL{|Kec{rP2ZB*XCMLhS}9FT=YjSa z*&R{b5y>6N=5RlKb3ALf^yxUr6mHWa6qqT$PBc0PL+Av?IJXifgtpZ~gMcw>g9PKegV~D|L{_5Bz&G%62ivM)^ zwST@saVFjnb+vRH$pPhc>r8`o@mCL9MNwVzdB2FF_cy+iMM?|HL00$WpN z8E;VVhAdtM)t2?Sbf@H7mhcrBUs3Rte6v*Dxiu{fQW8$fIIZAx0X>>NCk*{Yxo|f72g^$!*v{*5ikP)jeu1ERuIq#)1tEWo_Eid8}}C`%tWjwGR7v! a*pxNd*UrB?c5iG=d-sf($pg)r%>NH^*mLNZdi;^wLrSkm8~sX8KxnJQ9H+>-ZL z)>0`;rLvg%TgAAt;%@{qVJ58v+f!!RN;W}0_2`UWpBJgp3cMkrO55qXj!t7Egng!H z3c^bXAzlCD5T4FL+Uj%}nB)q2QMb_O-tFFlQn6C*mUXLCD3y!F!u`CAMfTVk8w#K@ zPnul$H>;l4sMdPfsKWh&>i(0%`eUk+vDbHcSYm~kkB~}zhfv8Um})Po&zpzO>af*r zJgrjIeKkNwgswr@aU7R)2y4QJ5Uada;S}Q(XY?JMGX@t8*vDyZQOnOqw-k@h`@=yWy68k~OBaY3($- z#VC=Uvh_9{X_*jk%t0pF*a-p09PC7E+achXgYDDnl@L^>srS9;=0OM;){kp^;SKh! EU-v`6ng9R* diff --git a/automated/convert.py b/jupyter_output_monitor/_convert.py similarity index 61% rename from automated/convert.py rename to jupyter_output_monitor/_convert.py index 0bec061..92c3bc0 100644 --- a/automated/convert.py +++ b/jupyter_output_monitor/_convert.py @@ -1,19 +1,26 @@ +# Experimental command to convert a notebook to a script that tests widget +# output with solara but without launching a Jupyter Lab instance. Not yet +# exposed via the public API. + import os +from textwrap import indent + import click import nbformat -from textwrap import indent + def remove_magics(source): - lines = [line for line in source.splitlines() if not line.startswith('%')] + lines = [line for line in source.splitlines() if not line.startswith("%")] return os.linesep.join(lines) def remove_excludes(source): - lines = [line for line in source.splitlines() if not line.strip().endswith('# EXCLUDE')] + lines = [ + line for line in source.splitlines() if not line.strip().endswith("# EXCLUDE") + ] return os.linesep.join(lines) - HEADER = """ import time import solara @@ -50,52 +57,51 @@ def test_main(page_session, solara_test): @click.command() -@click.argument('input_notebook') -@click.argument('output_script') +@click.argument("input_notebook") +@click.argument("output_script") def convert(input_notebook, output_script): - nb = nbformat.read(input_notebook, as_version=4) - with open(output_script, 'w') as f: - + with open(output_script, "w") as f: f.write(HEADER) captured = False - for icell, cell in enumerate(nb['cells']): - - if cell.cell_type == 'markdown': - f.write(indent(cell.source, ' # ') + '\n\n') - elif cell.cell_type == 'code': - - if cell.source.strip() == '': + for icell, cell in enumerate(nb["cells"]): + if cell.cell_type == "markdown": + f.write(indent(cell.source, " # ") + "\n\n") + elif cell.cell_type == "code": + if cell.source.strip() == "": continue lines = cell.source.splitlines() new_lines = [] - new_lines.append('cell_start = time.time()\n\n') + new_lines.append("cell_start = time.time()\n\n") for line in lines: - if line.startswith('%') or line.strip().endswith('# EXCLUDE'): + if line.startswith("%") or line.strip().endswith("# EXCLUDE"): continue - elif line.endswith('# SCREENSHOT'): - new_lines.append('object_to_capture = ' + line) + elif line.endswith("# SCREENSHOT"): + new_lines.append("object_to_capture = " + line) new_lines.extend(DISPLAY_CODE.splitlines()) captured = True else: new_lines.append(line) - new_lines.append('cell_end = time.time()\n') - new_lines.append(f'print(f"Cell {icell:2d} Python code executed in {{cell_end - cell_start:.2f}}s")') + new_lines.append("cell_end = time.time()\n") + new_lines.append( + f'print(f"Cell {icell:2d} Python code executed in {{cell_end - cell_start:.2f}}s")', + ) if captured: new_lines.extend(PROFILING_CODE.splitlines()) source = os.linesep.join(new_lines) - f.write(indent(source, " ") + '\n\n') + f.write(indent(source, " ") + "\n\n") + if __name__ == "__main__": convert() diff --git a/jupyter_output_monitor/_monitor.py b/jupyter_output_monitor/_monitor.py index 09edacc..1024a5c 100644 --- a/jupyter_output_monitor/_monitor.py +++ b/jupyter_output_monitor/_monitor.py @@ -45,7 +45,7 @@ def monitor(notebook, url, output, wait_after_execute, headless): output = f"output-{isotime()}" if os.path.exists(output): - print("Output directory {output} already exists") + print(f"Output directory {output} already exists") sys.exit(1) os.makedirs(output) @@ -150,14 +150,11 @@ def _monitor_output(url, output, wait_after_execute, headless): # The element we are interested in is one level down - div = output_cell.query_selector("div") - - if div is None: - continue - - style = div.get_attribute("style") - - if style is None or "border-color: rgb(" not in style: + for child in output_cell.query_selector_all("*"): + style = child.get_attribute("style") + if style is not None and "border-color: rgb(" in style: + break + else: continue # Parse rgb values for border @@ -180,7 +177,7 @@ def _monitor_output(url, output, wait_after_execute, headless): print(f"- taking screenshot of output cell {output_index}") - screenshot_bytes = div.screenshot() + screenshot_bytes = child.screenshot() # If screenshot didn't exist before for this cell or if it has # changed, we save it to a file and keep track of it. diff --git a/jupyter_output_monitor/_version.py b/jupyter_output_monitor/_version.py new file mode 100644 index 0000000..e1e7039 --- /dev/null +++ b/jupyter_output_monitor/_version.py @@ -0,0 +1,15 @@ +# file generated by setuptools_scm +# don't change, don't track in version control +TYPE_CHECKING = False +if TYPE_CHECKING: + VERSION_TUPLE = tuple[int | str, ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = "0.1.dev20+g8e2a3dc.d20241113" +__version_tuple__ = version_tuple = (0, 1, "dev20", "g8e2a3dc.d20241113") diff --git a/jupyter_output_monitor/tests/__init__.py b/jupyter_output_monitor/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jupyter_output_monitor/tests/data/simple.ipynb b/jupyter_output_monitor/tests/data/simple.ipynb new file mode 100644 index 0000000..c1e1d70 --- /dev/null +++ b/jupyter_output_monitor/tests/data/simple.ipynb @@ -0,0 +1,97 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b26895d5-f326-446e-b847-a4bd71c30b33", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-13T11:58:55.701204Z", + "iopub.status.busy": "2024-11-13T11:58:55.701016Z", + "iopub.status.idle": "2024-11-13T11:58:55.706466Z", + "shell.execute_reply": "2024-11-13T11:58:55.705916Z", + "shell.execute_reply.started": "2024-11-13T11:58:55.701186Z" + } + }, + "source": [ + "Test a few widgets!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c53acb0f-394f-45a1-99ef-dbbd0bb16afd", + "metadata": {}, + "outputs": [], + "source": [ + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd9213d9-cbac-4dcc-898e-5364b038b9bc", + "metadata": {}, + "outputs": [], + "source": [ + "from ipywidgets import Button, Textarea" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e4d91f0-19b8-4603-8077-b30ad6ccb19f", + "metadata": {}, + "outputs": [], + "source": [ + "button = Button(description='Test')\n", + "button.layout.border = '1px solid rgb(143, 56, 3)'\n", + "button" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74a37995-adf9-4b0c-8211-5611e0974a2c", + "metadata": {}, + "outputs": [], + "source": [ + "area = Textarea()\n", + "area.layout.border = '1px solid rgb(143, 56, 33)'\n", + "area" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0a116202-6c97-4ae3-b3b4-3accda88c8e5", + "metadata": {}, + "outputs": [], + "source": [ + "area.value = 'Test1'\n", + "time.sleep(3)\n", + "area.value = 'Test2'" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter_output_monitor/tests/test_monitor.py b/jupyter_output_monitor/tests/test_monitor.py new file mode 100644 index 0000000..51fb922 --- /dev/null +++ b/jupyter_output_monitor/tests/test_monitor.py @@ -0,0 +1,42 @@ +import csv +import subprocess +import sys +from pathlib import Path + +DATA = Path(__file__).parent / "data" + + +def test_simple(tmp_path): + output_path = tmp_path / "output" + subprocess.run( + [ + sys.executable, + "-m", + "jupyter_output_monitor", + "--notebook", + str(DATA / "simple.ipynb"), + "--output", + str(output_path), + "--headless", + ], + check=True, + ) + + # Check that the expected screenshots are there + + # Input cells + assert len(list(output_path.glob("input-*.png"))) == 5 + + # Output screenshots + assert len(list(output_path.glob("output-*.png"))) == 4 + + # Specifically for cell with index 33 + assert len(list(output_path.glob("output-003-*.png"))) == 1 + + # Specifically for cell with index 33 + assert len(list(output_path.glob("output-033-*.png"))) == 3 + + # Check that event log exists and is parsable + with open(output_path / "event_log.csv") as f: + reader = csv.reader(f, delimiter=",") + assert len(list(reader)) == 10 diff --git a/pyproject.toml b/pyproject.toml index acf5543..ee5ff53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,18 @@ lint.ignore = [ "D104", "C901", "PLR0915", + "PLR2004", "DTZ", - "E501" + "E501", + "RET", + "INP", + "S101", + "SIM108", + "S603" +] + +[project.optional-dependencies] +test = [ + "pytest>=7.0", + "ipywidgets" ] diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..8c9e5a6 --- /dev/null +++ b/tox.ini @@ -0,0 +1,14 @@ +[tox] +envlist = py{38,39,310,311,312}-test + +[testenv] +passenv = + DISPLAY +changedir = + test: .tmp/{envname} +extras = + test +commands = + pip freeze + playwright install firefox + pytest --pyargs jupyter_output_monitor {posargs} From 5be96fc52117f0323e87ef09ba0151af669f8d8f Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 Nov 2024 13:11:27 +0000 Subject: [PATCH 14/18] Use OpenAstronomy workflow --- .github/workflows/main.yml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d2a3162..b7932cd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,17 +7,18 @@ on: jobs: test: - runs-on: ubuntu-latest - name: jupyter-output-monitor - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - cache: 'pip' - - name: Set up dependencies - run: pip install . - - name: Initialize playwright - run: playwright install - - name: Run monitoring command - run: jupyter-output-monitor --notebook automated/imviz-profile.ipynb --headless + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + with: + envs: | + - linux: py310-test + - linux: py311-test + - linux: py312-test + - linux: py313-test + - macos: py310-test + - macos: py311-test + - macos: py312-test + - macos: py313-test + - windows: py310-test + - windows: py311-test + - windows: py312-test + - windows: py313-test From 9249ad667033b22df3ffbe6e88874b07c4a2aa9a Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 Nov 2024 13:21:57 +0000 Subject: [PATCH 15/18] Don't use colons in filenames as this doesn't work on Windows --- jupyter_output_monitor/_monitor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/jupyter_output_monitor/_monitor.py b/jupyter_output_monitor/_monitor.py index 1024a5c..6791394 100644 --- a/jupyter_output_monitor/_monitor.py +++ b/jupyter_output_monitor/_monitor.py @@ -124,9 +124,12 @@ def _monitor_output(url, output, wait_after_execute, headless): timestamp = isotime() + # Colons are invalid in filenames on Windows + filename_timestamp = timestamp.replace(":", "-") + screenshot_filename = os.path.join( output, - f"input-{input_index:03d}-{timestamp}.png", + f"input-{input_index:03d}-{filename_timestamp}.png", ) image = Image.open(BytesIO(screenshot_bytes)) image.save(screenshot_filename) @@ -188,9 +191,13 @@ def _monitor_output(url, output, wait_after_execute, headless): print(" -> change detected!") timestamp = isotime() + + # Colons are invalid in filenames on Windows + filename_timestamp = timestamp.replace(":", "-") + screenshot_filename = os.path.join( output, - f"output-{output_index:03d}-{timestamp}.png", + f"output-{output_index:03d}-{filename_timestamp}.png", ) image = Image.open(BytesIO(screenshot_bytes)) image.save(screenshot_filename) From 2743edd75ce30077fe0f9dad2a335d3d48042cf3 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 Nov 2024 13:23:50 +0000 Subject: [PATCH 16/18] Don't test on Python<3.12 on Windows as it doesn't work --- .github/workflows/main.yml | 2 -- jupyter_output_monitor/_version.py | 15 --------------- 2 files changed, 17 deletions(-) delete mode 100644 jupyter_output_monitor/_version.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7932cd..38971cc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,5 @@ jobs: - macos: py311-test - macos: py312-test - macos: py313-test - - windows: py310-test - - windows: py311-test - windows: py312-test - windows: py313-test diff --git a/jupyter_output_monitor/_version.py b/jupyter_output_monitor/_version.py deleted file mode 100644 index e1e7039..0000000 --- a/jupyter_output_monitor/_version.py +++ /dev/null @@ -1,15 +0,0 @@ -# file generated by setuptools_scm -# don't change, don't track in version control -TYPE_CHECKING = False -if TYPE_CHECKING: - VERSION_TUPLE = tuple[int | str, ...] -else: - VERSION_TUPLE = object - -version: str -__version__: str -__version_tuple__: VERSION_TUPLE -version_tuple: VERSION_TUPLE - -__version__ = version = "0.1.dev20+g8e2a3dc.d20241113" -__version_tuple__ = version_tuple = (0, 1, "dev20", "g8e2a3dc.d20241113") From 0a70ce89d3c42b96667b4a85d4c1d708d208f71f Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 Nov 2024 13:33:51 +0000 Subject: [PATCH 17/18] Updated README --- README.md | 87 +++++++++++++++++++++++-------------------------------- 1 file changed, 36 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index adb5155..2b74c0d 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,51 @@ This repository contains an experimental utility to monitor the visual output of cells from Jupyter notebooks. -## Requirements +## Installing -On the machine being used to run the ``monitor_cells.py``: +To install, check out this repository and: -* [numpy](https://numpy.org) -* [click](https://click.palletsprojects.com/en/stable/) -* [pillow](https://python-pillow.org/) -* [playwright](https://pypi.org/project/playwright/) + pip install -e . -On the Jupyter Lab server, optionally (but recommended): +If this is the first time using playwright, you will also need to run:: -* [jupyter-collaboration](https://github.com/jupyterlab/jupyter-collaboration) + playwright install firefox -If this is the first time using playwright, you will need to run:: +## Quick start - playwright install firefox +First, write one or more blocks of code you want to benchmark each in a cell. In +addition, as early as possible in the notebook, make sure you set the border +color on any ipywidget layout you want to record: -## Installing + widget.layout.border = '1px solid rgb(143, 56, 3)' -To install, check out this repository and: +The R and G values should be kept as (143, 56), and the B color should be unique for each widget and be a value between 0 and 255 (inclusive). - pip install -e . +Then, to run the notebook and monitor the changes in widget output, run: + + jupyter-output-monitor --notebook mynotebook.ipynb + +Where ``mynotebook.ipynb`` is the name of your notebook. By default, this will +open a window showing you what is happening, but you can also pass ``--headless`` +to run in headless mode. + +## Using this on a remote Jupyter Lab instance + +If you want to test this on an existing Jupyter Lab instance, including +remote ones, you can use ``--url`` instead of ``--notebook``: + + jupyter-output-monitor http://localhost:8987/lab/tree/notebook.ipynb?token=7bb9a... + +Note that the URL should include the path to the notebook, and will likely +require the token too. + +You should make sure that all output cells in the notebook have been cleared +before running the above command, and that the widget border color has been +set as mention in the **Quick start** guide above. + +If you make use of the [jupyter-collaboration](https://github.com/jupyterlab/jupyter-collaboration) plugin on the Jupyter Lab server, you will be able to +more easily e.g. clear the output between runs and edit the notebook in +between runs of ``jupyter-output-monitor``. ## How this works @@ -83,46 +106,8 @@ and if using jdaviz: To stop recording output for a given cell, you can set the border attribute to ``''``. -## Headless vs non-headless mode - -By default, the script will open up a window and show what it is doing. It will -also wait until it detects any input cells before proceeding. This then gives -you the opportunity to enter any required passwords, and open the correct -notebook. However, note that if Jupyter Lab opens up with a different notebook -to the one you want by default, it will start executing that one! It's also -better if the notebook starts off with output cells cleared, otherwise the script -may start taking screenshots straight away. - -The easiest way to ensure that the correct notebook gets executed and that it -has had its output cells cleared is to make use of the -[jupyter-collaboration](https://github.com/jupyterlab/jupyter-collaboration) -plugin. With this plugin installed, you can open Jupyter Lab in a regular browser window, -and set it up so that the correct notebook is open by default and has its cells cleared, -and you can then launch the monitoring script. In fact, if you do this you can then -also run the script in headless mode since you know it should be doing the right thing. - -One final note is that to avoid any jumping up and down of the notebook during -execution, the window opened by the script has a very large height so that the -full notebook fits inside the window without scrolling. - -## How to use - -* Assuming you have installed - [jupyter-collaboration](https://github.com/jupyterlab/jupyter-collaboration), - start up Jupyter Lab instance on a regular browser and go to the notebook you - want to profile. -* If not already done, write one or more blocks of code you want to benchmark - each in a cell. In addition, as early as possible in the notebook, make sure - you set the border color on any ipywidget layout you want to record. -* Make sure the notebook you want to profile is the main one opened and that - you have cleared any output cells. -* Run the main command in this package, specifying the URL to connect to for Jupyter Lab, e.g.: - - jupyter-output-monitor http://localhost:8987 - ## Settings - ### Headless To run in headless mode, include ``--headless`` From 0e40a53f30582e9ba34e0e9b50e958cc83168153 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 Nov 2024 13:34:58 +0000 Subject: [PATCH 18/18] Updated README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b74c0d..9dd581c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ To install, check out this repository and: pip install -e . -If this is the first time using playwright, you will also need to run:: +Python 3.10 or later is supported (Python 3.12 or later on Windows). + +If this is the first time using playwright, you will also need to run: playwright install firefox