From 39a85071e9baf49b7d8b4062332e4eb274dbdfb4 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 20 Oct 2022 11:53:50 -0400 Subject: [PATCH 1/8] Add initial thibaulttabarin code Add function written by thibaulttabarin from the drexel_metadata repo. See https://github.com/hdr-bgnn/drexel_metadata/blob/9d616a6b32a2b2786455e05a09dc799498a38af7/gen_metadata.py#L1002-L1058 Co-authored-by: thibaulttabarin --- dm_formatter.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 dm_formatter.py diff --git a/dm_formatter.py b/dm_formatter.py new file mode 100644 index 0000000..dd0d964 --- /dev/null +++ b/dm_formatter.py @@ -0,0 +1,57 @@ +def reformat_for_bgnn(result): + """ + Reformat and reduce the size of the result dictionary. + Collect only the data necessary for BGNN minnow project. The new format matches the + BGNN_metadata version. Therefore some of the value not calcualted in drexel version are by + defaulset to "None". + + Parameters + ---------- + result : dict + DESCRIPTION. output from gen_metadata() + + Returns + ------- + bgnn_result : dict + DESCRIPTION. {'base_name': xx, 'version':xx, + 'fish': {'fish_num': xx,"bbox":xx, 'pixel_analysis':xx, 'rescale':xx, + 'eye_bbox': xx, 'eye_center':xx , 'angle_degree': xx, + 'eye_direction':xx, 'foreground_mean':xx, 'background_mean':xx}, + 'ruler': {'bbox':xx, 'scale':xx, 'unit':xx}} + + """ + + name_base = list(result.keys())[0] + first_value = list(result.values())[0] + + # Fish metadata + fish_num = first_value['fish_count'] + fish_bbox = first_value['fish'][0]['bbox'] + pixel_analysis = False if first_value['fish'][0]['pixel_analysis_failed'] else True + + if first_value['fish'][0]['has_eye']: + eye_center = first_value['fish'][0]['eye_center'] + else : + eye_center = "None" + + eye_direction = first_value['fish'][0]['side'] + foreground_mean = first_value['fish'][0]['foreground']['mean'] + background_mean = first_value['fish'][0]['background']['mean'] + + dict_fish = {'fish_num': fish_num,"bbox":fish_bbox, + 'pixel_analysis':pixel_analysis, 'rescale':"None", + 'eye_bbox': "None", 'eye_center':eye_center , 'angle_degree': "None", + 'eye_direction':eye_direction, 'foreground_mean':round(foreground_mean,2), + 'background_mean':round(background_mean,2)} + + # Ruler metadata + ruler_bbox = first_value['ruler_bbox'] if first_value['has_ruler'] else "None" + scale = round(first_value['scale'],2) if "scale" in first_value.keys() else "None" + unit = first_value['unit'] if "unit" in first_value.keys() else "None" + + dict_ruler = {'bbox':ruler_bbox, 'scale':scale, 'unit':unit} + + bgnn_result = {'base_name': name_base, 'version':"from drexel", + 'fish': dict_fish, 'ruler': dict_ruler} + + return bgnn_result From 7ed696fd54cb827d302a741469592fce37cef382 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 20 Oct 2022 13:18:13 -0400 Subject: [PATCH 2/8] Add command line options to dm_formatter.py --- dm_formatter.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/dm_formatter.py b/dm_formatter.py index dd0d964..333cb51 100644 --- a/dm_formatter.py +++ b/dm_formatter.py @@ -1,3 +1,8 @@ +# Convert drexel_metadata JSON file to BGNN minnow metadata JSON file +import json +import argparse + + def reformat_for_bgnn(result): """ Reformat and reduce the size of the result dictionary. @@ -55,3 +60,25 @@ def reformat_for_bgnn(result): 'fish': dict_fish, 'ruler': dict_ruler} return bgnn_result + + +def argument_parser(): + parser = argparse.ArgumentParser(description='Convert metadata json file to BGNN minnow project format.') + parser.add_argument('input', help='Path of input drexel_metadata format JSON metadata file.') + parser.add_argument('output', help='Path of output BGNN minnow format JSON metadata file.') + return parser + + +def main(): + parser = argument_parser() + args = parser.parse_args() + print(f"Converting {args.input} to BGNN minnows format {args.output}") + with open(args.input, 'r') as infile: + data = json.load(infile) + result = reformat_for_bgnn(data) + with open(args.output, 'w') as outfile: + outfile.write(json.dumps(result)) + + +if __name__ == '__main__': + main() From e01d4fff24c8d10bb7dd6cfbabb1dad0cd2601b5 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 20 Oct 2022 13:18:36 -0400 Subject: [PATCH 3/8] Add Dockerfile --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..66109f0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.8.10-slim-buster + +WORKDIR /pipeline + +COPY dm_formatter.py /pipeline +CMD echo "python dm_formatter.py [input] [output]" From 4464810b64f5bbe601eda33433c70b8840d4ebe5 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Wed, 26 Oct 2022 14:34:23 -0400 Subject: [PATCH 4/8] Update README.md --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 19c319a..215246e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,20 @@ # drexel_metadata_formatter -Reformats the metadata output file from drexel_metadata repo +Reformats the metadata output file from the [drexel_metadata repo](https://github.com/hdr-bgnn/drexel_metadata/). + +## Requirements +- [Python 3](https://www.python.org/) + + +## Usage +``` +usage: dm_formatter.py [-h] input output + +Convert metadata json file to BGNN minnow project format. + +positional arguments: + input Path of input drexel_metadata format JSON metadata file. + output Path of output BGNN minnow format JSON metadata file. + +optional arguments: + -h, --help show this help message and exit + ``` From c0a85f23c88e5c7568c75049c46d51b3f34f0132 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Wed, 26 Oct 2022 14:36:49 -0400 Subject: [PATCH 5/8] Add logic to build a ghcr.io container on release Based on https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#publishing-a-package-using-an-action --- .github/workflows/deploy-image.yml | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/deploy-image.yml diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml new file mode 100644 index 0000000..8417847 --- /dev/null +++ b/.github/workflows/deploy-image.yml @@ -0,0 +1,41 @@ +name: Create and publish a Docker image + +on: + release: + types: [published] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From aece40b338d68e06575a23dcb8fb7c8052ba920c Mon Sep 17 00:00:00 2001 From: John Bradley Date: Wed, 26 Oct 2022 15:15:55 -0400 Subject: [PATCH 6/8] Add unit tests Also adds a github action to run tests based on: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#using-the-python-starter-workflow --- .github/workflows/run-tests.yml | 21 ++++++++++++ README.md | 6 ++++ test_dm_formatter.py | 57 +++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 .github/workflows/run-tests.yml create mode 100644 test_dm_formatter.py diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..8d0ef3a --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,21 @@ +name: Run python tests + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Run python unit tests + run: | + python3 -m unittest discover . diff --git a/README.md b/README.md index 215246e..5b48e77 100644 --- a/README.md +++ b/README.md @@ -18,3 +18,9 @@ positional arguments: optional arguments: -h, --help show this help message and exit ``` + +## Testing +The unit tests can be run with the following command: +``` +python3 -m unittest discover . +``` diff --git a/test_dm_formatter.py b/test_dm_formatter.py new file mode 100644 index 0000000..a69c5e9 --- /dev/null +++ b/test_dm_formatter.py @@ -0,0 +1,57 @@ +import unittest +from dm_formatter import reformat_for_bgnn + + +DM_INPUT_METADATA = { + "SOME-FISH-1234": { + "fish_count": 1, + "fish": [ { + "pixel_analysis_failed": False, + "bbox": [1,2,3,4], + "has_eye": True, + "eye_center": [50, 50], + "side": "left", + "foreground": { + "mean": 100.1111111, + }, + "background": { + "mean": 200.1111111, + } + } + ], + "has_ruler": True, + "ruler_bbox": [ + 5, + 6, + 7, + 8 + ], + } +} + +BGNN_EXPECTED_METADATA = { + 'base_name': 'SOME-FISH-1234', + 'fish': {'angle_degree': 'None', + 'background_mean': 200.11, + 'bbox': [1, 2, 3, 4], + 'eye_bbox': 'None', + 'eye_center': [50, 50], + 'eye_direction': 'left', + 'fish_num': 1, + 'foreground_mean': 100.11, + 'pixel_analysis': True, + 'rescale': 'None'}, + 'ruler': {'bbox': [5, 6, 7, 8], 'scale': 'None', 'unit': 'None'}, + 'version': 'from drexel' +} + + +class TestFormatter(unittest.TestCase): + def test_reformat_for_bgnn(self): + result = reformat_for_bgnn(DM_INPUT_METADATA) + self.assertEqual(result, BGNN_EXPECTED_METADATA) + + + +if __name__ == '__main__': + unittest.main() From 18e47100e39473970180fd208ab87ca508f20365 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Tue, 1 Nov 2022 09:25:40 -0400 Subject: [PATCH 7/8] Add minor fixes Adds various fixes suggested in PR #1. Co-authored-by: Hilmar Lapp --- Dockerfile | 2 +- dm_formatter.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 66109f0..20f3d38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,4 +3,4 @@ FROM python:3.8.10-slim-buster WORKDIR /pipeline COPY dm_formatter.py /pipeline -CMD echo "python dm_formatter.py [input] [output]" +CMD echo "python dm_formatter.py input output" diff --git a/dm_formatter.py b/dm_formatter.py index 333cb51..7b46628 100644 --- a/dm_formatter.py +++ b/dm_formatter.py @@ -7,8 +7,8 @@ def reformat_for_bgnn(result): """ Reformat and reduce the size of the result dictionary. Collect only the data necessary for BGNN minnow project. The new format matches the - BGNN_metadata version. Therefore some of the value not calcualted in drexel version are by - defaulset to "None". + BGNN_metadata version. Therefore some of the values not calculated in drexel version are by + default set to "None". Parameters ---------- @@ -32,7 +32,7 @@ def reformat_for_bgnn(result): # Fish metadata fish_num = first_value['fish_count'] fish_bbox = first_value['fish'][0]['bbox'] - pixel_analysis = False if first_value['fish'][0]['pixel_analysis_failed'] else True + pixel_analysis = not first_value['fish'][0]['pixel_analysis_failed'] if first_value['fish'][0]['has_eye']: eye_center = first_value['fish'][0]['eye_center'] @@ -43,11 +43,11 @@ def reformat_for_bgnn(result): foreground_mean = first_value['fish'][0]['foreground']['mean'] background_mean = first_value['fish'][0]['background']['mean'] - dict_fish = {'fish_num': fish_num,"bbox":fish_bbox, - 'pixel_analysis':pixel_analysis, 'rescale':"None", - 'eye_bbox': "None", 'eye_center':eye_center , 'angle_degree': "None", - 'eye_direction':eye_direction, 'foreground_mean':round(foreground_mean,2), - 'background_mean':round(background_mean,2)} + dict_fish = {'fish_num': fish_num,"bbox": fish_bbox, + 'pixel_analysis': pixel_analysis, 'rescale': "None", + 'eye_bbox': "None", 'eye_center': eye_center , 'angle_degree': "None", + 'eye_direction': eye_direction, 'foreground_mean': round(foreground_mean,2), + 'background_mean': round(background_mean,2)} # Ruler metadata ruler_bbox = first_value['ruler_bbox'] if first_value['has_ruler'] else "None" From 70e9fe7cc38e58ab7d4ee31853c0f167e2388cf2 Mon Sep 17 00:00:00 2001 From: John Bradley Date: Thu, 3 Nov 2022 09:31:22 -0400 Subject: [PATCH 8/8] Simplify Dockerfile usage Adds dm_formatter.py to PATH so users can run it directly. Also adds metadata to the Docker container. --- Dockerfile | 7 ++++++- dm_formatter.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) mode change 100644 => 100755 dm_formatter.py diff --git a/Dockerfile b/Dockerfile index 20f3d38..128af8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,11 @@ FROM python:3.8.10-slim-buster +LABEL "org.opencontainers.image.authors"="John Bradley " +LABEL "org.opencontainers.image.description"="Tool to reformat drexel metadata JSON files" WORKDIR /pipeline +# ADD scripts in /pipeline to the PATH +ENV PATH="/pipeline:${PATH}" + COPY dm_formatter.py /pipeline -CMD echo "python dm_formatter.py input output" +CMD echo "dm_formatter.py input output" diff --git a/dm_formatter.py b/dm_formatter.py old mode 100644 new mode 100755 index 7b46628..316e546 --- a/dm_formatter.py +++ b/dm_formatter.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Convert drexel_metadata JSON file to BGNN minnow metadata JSON file import json import argparse