Skip to content

Commit

Permalink
Enhance From File catalog loading to support more columns and improve…
Browse files Browse the repository at this point in the history 
… Clear Table functionality (spacetelescope#3359)

* Add functionality to load catalogs from a file

* Fix table and viewer clearing functionality

* Remove redundant 'Clear' button and combine its functionality with 'Clear Table' for intuitive behavior

* Codestyle

* Add change log

* Update tests

* Fix failing test

* Fix the test, ensuring to first select a row before zooming in.

* Update change log

* Preserve the unit when serializing

* Generate default Object IDs when label column is missing

* Update docs related to label column in catalogs

* Update tests after merge

* Update test

* Move changelog to correct milestone

* Adjust test assertions

* Add test for loading catalogs with additional columns

* Update change log

* Update based on feedback

* Apply suggestions from code review

Co-authored-by: P. L. Lim <[email protected]>

* Update CHANGES.rst

Co-authored-by: P. L. Lim <[email protected]>

* Adjust test tolerance

---------

Co-authored-by: P. L. Lim <[email protected]>
  • Loading branch information
@haticekaratay @pllim @rosteen
2 people authored and rosteen committed Jan 6, 2025
1 parent 7760fa7 commit 84d2823
Show file tree
Hide file tree
Showing 6 changed files with 137 additions and 45 deletions.
4 changes: 4 additions & 0 deletions CHANGES.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ Cubeviz
Imviz
^^^^^

- Enhance the Catalog Search plugin to support additional columns when loading catalog data from files. [#3359]

- Catalog Search ``clear_table`` now removes all associated markers from the viewer. [#3359]

Mosviz
^^^^^^

Expand Down
5 changes: 3 additions & 2 deletions docs/imviz/plugins.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,8 +382,9 @@ To load a catalog from a supported `JWST ECSV catalog file <https://jwst-pipelin
The file must be able to be parsed by `astropy.table.Table.read` and contains the following columns:

* ``'sky_centroid'``: Column with `~astropy.coordinates.SkyCoord` sky coordinates of the sources.
* ``'label'``: Column with string identifiers of the sources. If you have numerical identifiers,
they will be recast as string.
* ``'label'``: (Optional) Column with string identifiers of the sources.
If not provided, unique string identifiers will be generated automatically.
If you have numerical identifiers, they will be recast as strings.

Clicking :guilabel:`SEARCH` will show markers for any entry within the filtered zoom window.

Expand Down
59 changes: 38 additions & 21 deletions jdaviz/configs/imviz/plugins/catalogs/catalogs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@ class Catalogs(PluginTemplateMixin, ViewerSelectMixin, HasFileImportSelect, Tabl
'Object ID': np.nan,
'id': np.nan,
'x_coord': np.nan,
'y_coord': np.nan}
'y_coord': np.nan
}

headers = ['Right Ascension (degrees)', 'Declination (degrees)',
'Object ID', 'x_coord', 'y_coord']

@property
def user_api(self):
Expand All @@ -72,11 +76,8 @@ def __init__(self, *args, **kwargs):
self._marker_name = 'catalog_results'

# initializing the headers in the table that is displayed in the UI
headers = ['Right Ascension (degrees)', 'Declination (degrees)',
'Object ID', 'x_coord', 'y_coord']

self.table.headers_avail = headers
self.table.headers_visible = headers
self.table.headers_avail = self.headers
self.table.headers_visible = self.headers
self.table._default_values_by_colname = self._default_table_values
self.table._selected_rows_changed_callback = lambda msg: self.plot_selected_points()
self.table.item_key = 'id'
Expand All @@ -93,6 +94,9 @@ def _file_parser(path):
except Exception:
return 'Could not parse file with astropy.table.QTable.read', {}

if not table.colnames: # Ensure the file has columns
return "File contains no columns", {}

if 'sky_centroid' not in table.colnames:
return 'Table does not contain required sky_centroid column', {}

Expand All @@ -115,7 +119,7 @@ def search(self, error_on_fail=False):
"""
# calling clear in the case the user forgot after searching
self.clear()
self.clear_table()

# gets the current viewer
viewer = self.viewer.selected_obj
Expand Down Expand Up @@ -204,6 +208,10 @@ def search(self, error_on_fail=False):
# all exceptions when going through the UI should have prevented setting this path
# but this exceptions might be raised here if setting from_file from the UI
table = self.catalog.selected_obj
column_names = table.colnames
self.table.headers_avail = self.headers + [
col for col in column_names if col not in self.headers]
self.table.headers_visible = self.headers
self.app._catalog_source_table = table
if len(table['sky_centroid']) > self.max_sources:
skycoord_table = table['sky_centroid'][:self.max_sources]
Expand Down Expand Up @@ -271,16 +279,22 @@ def search(self, error_on_fail=False):
if len(self.app._catalog_source_table) == 1 or self.max_sources == 1:
x_coordinates = [x_coordinates]
y_coordinates = [y_coordinates]
for idx, (row, x_coord, y_coord) in enumerate(zip(self.app._catalog_source_table, x_coordinates, y_coordinates)): # noqa:E501
row_info = {
'Right Ascension (degrees)': row['sky_centroid'].ra.deg,
'Declination (degrees)': row['sky_centroid'].dec.deg,
'Object ID': str(row.get('label', f"{idx + 1}")),
'id': len(self.table),
'x_coord': x_coord,
'y_coord': y_coord,
}
# Add sky_centroid and label explicitly to row_info
row_info['sky_centroid'] = row['sky_centroid']
row_info['label'] = row.get('label', f"{idx + 1}")
for col in table.colnames:
if col not in self.headers: # Skip already processed columns
row_info[col] = row[col]

for row, x_coord, y_coord in zip(self.app._catalog_source_table,
x_coordinates, y_coordinates):
# Check if the row contains the required keys
row_info = {'Right Ascension (degrees)': row['sky_centroid'].ra.deg,
'Declination (degrees)': row['sky_centroid'].dec.deg,
'Object ID': str(row.get('label', 'N/A')),
'id': len(self.table),
'x_coord': x_coord.item() if x_coord.size == 1 else x_coord,
'y_coord': y_coord.item() if y_coord.size == 1 else y_coord}
self.table.add_item(row_info)

filtered_skycoord_table = viewer.state.reference_data.coords.pixel_to_world(x_coordinates,
Expand Down Expand Up @@ -357,8 +371,7 @@ def zoom_to_selected(self, padding=0.02, return_bounding_box=False):
viewer = self.app._jdaviz_helper._default_viewer

selected_rows = self.table.selected_rows

if not len(selected_rows):
if not selected_rows: # Check if no rows are selected
return

if padding <= 0 or padding > 1:
Expand Down Expand Up @@ -402,9 +415,13 @@ def vue_do_search(self, *args, **kwargs):
# calls self.search() which handles all of the searching logic
self.search()

def clear(self, hide_only=True):
def clear_table(self, hide_only=True):
# gets the current viewer
viewer = self.viewer.selected_obj
# Clear the table before performing a new search
self.table.items = []
self.table.selected_rows = []
self.table.selected_indices = []

if not hide_only and self._marker_name in self.app.data_collection.labels:
# resetting values
Expand All @@ -426,5 +443,5 @@ def clear(self, hide_only=True):
if layer_is_table_data(lyr.layer) and lyr.layer.label == self._marker_name:
lyr.visible = False

def vue_do_clear(self, *args, **kwargs):
self.clear()
def vue_do_clear_table(self, *args, **kwargs):
self.clear_table()
18 changes: 10 additions & 8 deletions jdaviz/configs/imviz/plugins/catalogs/catalogs.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,16 @@
See the <j-external-link link='https://astroquery.readthedocs.io/en/latest/gaia/gaia.html' linktext='astropy.gaia docs'></j-external-link> for details on the query defaults.
</j-docs-link>
</v-row>

<v-row v-if="catalog_selected && catalog_selected.endsWith('.ecsv')">
<v-select
v-model="selected_columns"
:items="column_names"
label="Select Columns"
multiple
hint="Select columns to display in the table."
/>
</v-row>

<v-row>
<v-text-field
Expand All @@ -53,14 +63,6 @@
</v-row>

<v-row class="row-no-outside-padding">
<v-col>
<plugin-action-button
:results_isolated_to_plugin="true"
@click="do_clear"
>
Clear
</plugin-action-button>
</v-col>
<v-col>
<plugin-action-button
:results_isolated_to_plugin="true"
Expand Down
82 changes: 70 additions & 12 deletions jdaviz/configs/imviz/tests/test_catalogs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path):
prev_results = catalogs_plugin.number_of_results

# testing that every variable updates accordingly when markers are cleared
catalogs_plugin.vue_do_clear()
catalogs_plugin.vue_do_clear_table()

assert not catalogs_plugin.results_available

Expand Down Expand Up @@ -158,14 +158,25 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path):
assert imviz_helper.viewers['imviz-0']._obj.state.y_min == -0.5
assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 1488.5

# Re-populate the table with a new search
with pytest.warns(ResourceWarning):
catalogs_plugin.search(error_on_fail=True)
# Ensure at least one row is selected before zooming
catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]]
assert len(catalogs_plugin.table.selected_rows) > 0

# set 'padding' to reproduce original hard-coded 50 pixel window
# so test results don't change
catalogs_plugin.zoom_to_selected(padding=50 / 2048)

assert imviz_helper.viewers['imviz-0']._obj.state.x_min == 858.24969
assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 958.38461
assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 278.86265
assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 378.8691
assert_allclose(
imviz_helper.viewers['imviz-0']._obj.state.x_min, 1022.57570000, atol=0.1)
assert_allclose(
imviz_helper.viewers['imviz-0']._obj.state.x_max, 1122.5757, atol=0.1)
assert_allclose(
imviz_helper.viewers['imviz-0']._obj.state.y_min, 675.29611, atol=0.1)
assert_allclose(
imviz_helper.viewers['imviz-0']._obj.state.y_max, 775.29611, atol=0.1)


def test_from_file_parsing(imviz_helper, tmp_path):
Expand Down Expand Up @@ -200,7 +211,7 @@ def test_from_file_parsing(imviz_helper, tmp_path):
def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path):
sky = SkyCoord(ra=[337.5202807, 337.51909197, 337.51760596],
dec=[-20.83305528, -20.83222194, -20.83083304], unit='deg')
tbl = QTable({'sky_centroid': sky})
tbl = QTable({'sky_centroid': sky}) # Table has no "Label" column
tbl_file = str(tmp_path / 'sky_centroid.ecsv')
tbl.write(tbl_file, overwrite=True)
n_entries = len(tbl)
Expand All @@ -215,6 +226,9 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path):
out_tbl = catalogs_plugin.search(error_on_fail=True)
assert len(out_tbl) == n_entries
assert catalogs_plugin.number_of_results == n_entries
# Assert that Object ID is set to index + 1 when the label column is absent
for idx, item in enumerate(catalogs_plugin.table.items):
assert item['Object ID'] == str(idx + 1)
assert len(imviz_helper.app.data_collection) == 2 # image + markers

catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]]
Expand All @@ -240,27 +254,34 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path):
assert catalogs_plugin.number_of_results == n_entries
assert len(imviz_helper.app.data_collection) == 2 # image + markers

catalogs_plugin.clear()
catalogs_plugin.clear_table()

assert not catalogs_plugin.results_available
assert len(imviz_helper.app.data_collection) == 2 # markers still there, just hidden

catalogs_plugin.clear(hide_only=False)
catalogs_plugin.clear_table(hide_only=False)
assert not catalogs_plugin.results_available
assert len(imviz_helper.app.data_collection) == 1 # markers gone for good

assert imviz_helper.viewers['imviz-0']._obj.state.x_min == -0.5
assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 9.5
assert imviz_helper.viewers['imviz-0']._obj.state.y_min == -0.5
assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 9.5
# Re-populate the table with a new search
out_tbl = catalogs_plugin.search()
assert len(out_tbl) > 0
# Ensure at least one row is selected before zooming
catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]]
assert len(catalogs_plugin.table.selected_rows) > 0

# test the zooming using the default 'padding' of 2% of the viewer size
# around selected points
catalogs_plugin.zoom_to_selected()
assert imviz_helper.viewers['imviz-0']._obj.state.x_min == -0.19966
assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 0.20034000000000002
assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 0.8000100000000001
assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 1.20001
assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_min, -0.19966, rtol=1e-4)
assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_max,
0.20034000000000002, rtol=1e-4)
assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_min, 0.8000100000000001, rtol=1e-4)
assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_max, 1.20001, rtol=1e-4)


def test_zoom_to_selected(imviz_helper, image_2d_wcs, tmp_path):
Expand Down Expand Up @@ -334,3 +355,40 @@ def test_zoom_to_selected(imviz_helper, image_2d_wcs, tmp_path):
# test that appropriate error is raised when padding is not a valud percentage
with pytest.raises(ValueError, match="`padding` must be between 0 and 1."):
catalogs_plugin.zoom_to_selected(padding=5)


def test_offline_ecsv_catalog_with_extra_columns(imviz_helper, image_2d_wcs, tmp_path):
# Create a table with additional columns
sky = SkyCoord(ra=[337.5202807, 337.51909197, 337.51760596],
dec=[-20.83305528, -20.83222194, -20.83083304], unit='deg')
tbl = QTable({
'sky_centroid': sky,
'flux': [1.0, 2.0, 3.0],
'flux_err': [0.1, 0.2, 0.3],
'is_extended': [False, True, False],
'roundness': [0.01, 0.02, 0.03],
'sharpness': [0.1, 0.2, 0.3]
})
tbl_file = str(tmp_path / 'extra_columns.ecsv')
tbl.write(tbl_file, overwrite=True)

ndd = NDData(np.ones((10, 10)), wcs=image_2d_wcs)
imviz_helper.load_data(ndd, data_label='data_with_wcs')
assert len(imviz_helper.app.data_collection) == 1

catalogs_plugin = imviz_helper.plugins['Catalog Search']._obj
catalogs_plugin.from_file = tbl_file
catalogs_plugin.catalog_selected = 'From File...'
catalogs_plugin.search(error_on_fail=True)

extra_columns = ['flux', 'flux_err', 'is_extended', 'roundness', 'sharpness']
for col in extra_columns:
assert col in catalogs_plugin.table.headers_avail

# Check if extra columns are populated correctly
for idx, item in enumerate(catalogs_plugin.table.items):
assert float(item['flux']) == tbl['flux'][idx]
assert float(item['flux_err']) == tbl['flux_err'][idx]
assert item['is_extended'] == tbl['is_extended'][idx]
assert float(item['roundness']) == tbl['roundness'][idx]
assert float(item['sharpness']) == tbl['sharpness'][idx]
14 changes: 12 additions & 2 deletions jdaviz/core/template_mixin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4769,12 +4769,20 @@ def float_precision(column, item):
return ''
elif isinstance(item, tuple) and np.all([np.isnan(i) for i in item]):
return ''

elif isinstance(item, float):
return float_precision(column, item)
elif isinstance(item, (list, tuple)):
return [float_precision(column, i) if isinstance(i, float) else i for i in item]

elif isinstance(item, (np.float32, np.float64)):
return float(item)
elif isinstance(item, u.Quantity):
return {"value": item.value.tolist() if item.size > 1 else item.value, "unit": str(item.unit)} # noqa: E501
elif isinstance(item, np.bool_):
return bool(item)
elif isinstance(item, np.ndarray):
return item.tolist()
elif isinstance(item, tuple):
return tuple(json_safe(v) for v in item)
return item

if isinstance(item, QTable):
Expand Down Expand Up @@ -4816,6 +4824,8 @@ def clear_table(self):
Clear all entries/markers from the current table.
"""
self.items = []
self.selected_rows = []
self.selected_indices = []
self._qtable = None
self._plugin.session.hub.broadcast(PluginTableModifiedMessage(sender=self))

Expand Down

0 comments on commit 84d2823

Please sign in to comment.