feat: GeoContext harvester from admin page, with the log view (#3931)

* Add button to harvest geocontext in admin page

* Initiate geocontext harvester from admin, save the log to file

* Add api to show the logs

bims/admin.py

@@ -183,6 +183,7 @@ class LocationContextAdmin(admin.ModelAdmin):
 
 
 class LocationSiteAdmin(admin.GeoModelAdmin):
+    change_list_template = 'admin/location_site_changelist.html'
     form = LocationSiteForm
     default_zoom = 5
     default_lat = -30

bims/api_urls.py

@@ -1,5 +1,9 @@
 from django.urls import re_path, path
 
+from bims.api_views.geocontext import (
+    IsHarvestingGeocontext, HarvestGeocontextView, ClearHarvestingGeocontextCache,
+    GetGeocontextLogLinesView
+)
 from bims.api_views.taxon_update import UpdateTaxon, ReviewTaxonProposal
 from bims.api_views.reference import DeleteRecordsByReferenceId
 # from rest_framework.documentation import include_docs_urls
@@ -354,5 +358,17 @@
          name='test-email'),
     path('taxa-cites-status/',
          TaxaCitesStatusAPIView.as_view(),
-         name='taxa-cites-status')
+         name='taxa-cites-status'),
+    path('is-harvesting-geocontext/',
+         IsHarvestingGeocontext.as_view(),
+         name='is-harvesting-geocontext'),
+    path('harvest-geocontext/',
+         HarvestGeocontextView.as_view(),
+         name='harvest-geocontext'),
+    path('clear-harvesting-geocontext-cache/',
+         ClearHarvestingGeocontextCache.as_view(),
+         name='clear-harvesting-geocontext-cache'),
+    path('harvesting-geocontext-logs/',
+         GetGeocontextLogLinesView.as_view(),
+         name='get_log_lines'),
 ]

bims/api_views/geocontext.py

@@ -0,0 +1,104 @@
+import os
+
+from django.conf import settings
+from django.db import connection
+from django.http import JsonResponse
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from braces.views import SuperuserRequiredMixin
+
+from bims.cache import get_cache, HARVESTING_GEOCONTEXT, set_cache
+from bims.tasks import update_location_context
+from bims.utils.location_context import get_location_context_data
+
+
+class IsHarvestingGeocontext(SuperuserRequiredMixin, APIView):
+    """
+    API view to check if the geocontext is currently being harvested.
+    Only accessible to superusers.
+    """
+    def get(self, request, *args, **kwargs):
+        """
+        Handle GET request to determine the harvesting status.
+
+        :param request: HTTP request object
+        :return: JSON response with harvesting status
+        """
+        try:
+            is_harvesting = get_cache(HARVESTING_GEOCONTEXT, False)
+            return Response({'harvesting': is_harvesting})
+        except Exception as e:
+            return Response({'error': str(e)}, status=500)
+
+
+class HarvestGeocontextView(SuperuserRequiredMixin, APIView):
+
+    def harvest_geocontext(self, is_all=False):
+        update_location_context.delay(
+            location_site_id=None,
+            generate_site_code=False,
+            generate_filter=True,
+            only_empty=not is_all
+        )
+
+    def post(self, request, *args, **kwargs):
+        is_harvesting = get_cache(HARVESTING_GEOCONTEXT, False)
+        if is_harvesting:
+            return Response({'error': 'Harvesting is already in progress.'}, status=400)
+
+        # Set harvesting flag to true
+        set_cache(HARVESTING_GEOCONTEXT, True)
+
+        try:
+            # check if harvesting all or just empty
+            is_all = request.data.get('is_all', False)
+
+            self.harvest_geocontext(is_all)
+
+            return Response({'status': 'Harvesting started successfully.'}, status=200)
+        except Exception as e:
+            # Reset harvesting flag in case of exception
+            set_cache(HARVESTING_GEOCONTEXT, False)
+            return Response({'error': str(e)}, status=500)
+
+
+class ClearHarvestingGeocontextCache(SuperuserRequiredMixin, APIView):
+    def get(self, request, *args, **kwargs):
+        """
+        Handle GET request to determine the harvesting status.
+
+        :param request: HTTP request object
+        :return: JSON response with harvesting status
+        """
+        set_cache(HARVESTING_GEOCONTEXT, False)
+        try:
+            is_harvesting = get_cache(HARVESTING_GEOCONTEXT, False)
+            return Response({'harvesting': is_harvesting})
+        except Exception as e:
+            return Response({'error': str(e)}, status=500)
+
+
+def get_last_100_lines(file_path):
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+    return lines[-100:]
+
+
+class GetGeocontextLogLinesView(SuperuserRequiredMixin, APIView):
+    def get(self, request, *args, **kwargs):
+        tenant = connection.schema_name
+        tenant_name = str(tenant)
+        log_file_name = f'{tenant_name}_get_location_context_data.log'
+        log_file_path = os.path.join(settings.MEDIA_ROOT, log_file_name)
+
+        if not os.path.exists(log_file_path):
+            return JsonResponse(
+                {'error': 'Log file not found'}, status=404)
+
+        try:
+            last_100_lines = get_last_100_lines(log_file_path)
+            return JsonResponse(
+                {'log': last_100_lines})
+        except Exception as e:
+            return JsonResponse(
+                {'error': str(e)}, status=500)

bims/cache.py

@@ -4,6 +4,7 @@
 
 LANDING_PAGE_MODULE_SUMMARY_CACHE = 'LANDING_PAGE_MODULE_SUMMARY_CACHE'
 UPDATE_FILTERS_CACHE = 'UPDATE_FILTERS'
+HARVESTING_GEOCONTEXT = 'HARVESTING_GEOCONTEXT_3'
 
 
 def instance_cache_key(instance):

bims/tasks/location_site.py

@@ -4,6 +4,8 @@
 from django.db.models import signals
 
 from celery import shared_task
+
+from bims.cache import set_cache, HARVESTING_GEOCONTEXT
 from bims.utils.logger import log
 
 logger = logging.getLogger(__name__)
@@ -64,7 +66,8 @@ def update_location_context(
         self,
         location_site_id,
         generate_site_code=False,
-        generate_filter=True):
+        generate_filter=True,
+        only_empty=False):
     from bims.models import LocationSite, location_site_post_save_handler
     from bims.utils.location_context import get_location_context_data
     from bims.models.location_context_group import LocationContextGroup
@@ -75,6 +78,21 @@ def update_location_context(
 
     self.update_state(state='STARTED', meta={'process': 'Checking location site'})
 
+    if not location_site_id:
+        self.update_state(
+            state='PROGRESS',
+            meta={'process': 'Updating location context for multiple sites'})
+        try:
+            get_location_context_data(
+                group_keys=group_keys,
+                site_id=None,
+                only_empty=only_empty,
+                should_generate_site_code=generate_site_code
+            )
+        except Exception:
+            set_cache(HARVESTING_GEOCONTEXT, False)
+        return
+
     if isinstance(location_site_id, str):
         self.update_state(
             state='PROGRESS',
@@ -83,7 +101,7 @@ def update_location_context(
             get_location_context_data(
                 group_keys=group_keys,
                 site_id=str(location_site_id),
-                only_empty=False,
+                only_empty=only_empty,
                 should_generate_site_code=generate_site_code
             )
             return

bims/templates/admin/location_site_changelist.html

@@ -0,0 +1,164 @@
+{% extends "admin/change_list.html" %}
+{% load static %}
+
+{% block object-tools %}
+{{ block.super }}
+<div class="dropdown" id="update-container">
+    <button class="button custom-button" id="update-geo-context" disabled="disabled">Update GeoContext</button>
+    <div class="dropdown-content" id="dropdown-content">
+        <a href="#" id="update-all" class="disabled-link">Update All Records</a>
+        <a href="#" id="update-empty" class="disabled-link">Update Only Empty Records</a>
+    </div>
+</div>
+{% endblock %}
+
+{% block extrahead %}
+{{ block.super }}
+<style>
+     .custom-button {
+        display: inline-block;
+        height: 30px;
+        width: 200px;
+        margin-left: 10px;
+        font-size: 14px;
+        font-weight: 400;
+        line-height: 1.42857143;
+        text-align: center;
+        white-space: nowrap;
+        vertical-align: middle;
+        cursor: pointer;
+        background-color: #337ab7;
+        border: 1px solid transparent;
+        border-radius: 4px;
+        color: #fff;
+        text-decoration: none;
+    }
+    .custom-button:disabled {
+        background-color: #cccccc;
+        border-color: #aaaaaa;
+        cursor: not-allowed;
+    }
+    .custom-button:hover:not(:disabled) {
+        background-color: #286090;
+        border-color: #204d74;
+        text-decoration: none;
+        color: #fff;
+    }
+    .dropdown {
+        right: 170px;
+        position: absolute;
+        top: 90px;
+        display: inline-block;
+    }
+    .dropdown-content {
+        display: none;
+        position: absolute;
+        background-color: #f9f9f9;
+        min-width: 160px;
+        box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
+        z-index: 1;
+    }
+    .dropdown-content a {
+        color: black;
+        padding: 12px 16px;
+        text-decoration: none;
+        display: block;
+    }
+    .dropdown-content a:hover {
+        background-color: #f1f1f1;
+    }
+    .dropdown-content .disabled-link {
+        pointer-events: none;
+        color: #cccccc;
+    }
+    .dropdown:hover .dropdown-content {
+        display: block;
+    }
+</style>
+<script type="text/javascript">
+    document.addEventListener('DOMContentLoaded', function () {
+        const updateGeoContextButton = document.getElementById('update-geo-context');
+        const updateContainer = document.getElementById('update-container');
+        const dropdownContent = document.getElementById('dropdown-content');
+        const updateAllLink = document.getElementById('update-all');
+        const updateEmptyLink = document.getElementById('update-empty');
+
+        // Function to check if harvesting is ongoing
+        async function checkHarvestingStatus() {
+            try {
+                const response = await fetch('/api/is-harvesting-geocontext');
+                const data = await response.json();
+                if (data.harvesting) {
+                    updateGeoContextButton.disabled = true;
+                    updateGeoContextButton.textContent = 'Harvesting GeoContext...';
+                } else {
+                    updateGeoContextButton.disabled = false;
+                    var links = document.querySelectorAll('.dropdown-content a');
+                    links.forEach(function(link) {
+                        link.classList.remove('disabled-link');
+                    });
+                }
+            } catch (error) {
+                console.error('Error checking harvesting status:', error);
+            }
+        }
+
+            async function startHarvesting(isAll) {
+            try {
+                const response = await fetch('/api/harvest-geocontext/', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'X-CSRFToken': "{{ csrf_token }}"
+                    },
+                    body: JSON.stringify({ is_all: isAll }),
+                });
+                const data = await response.json();
+                if (response.ok) {
+                    alert('Harvesting started successfully.');
+                    checkHarvestingStatus();
+                } else {
+                    alert('Error: ' + data.error);
+                }
+            } catch (error) {
+                console.error('Error starting harvesting:', error);
+                alert('An error occurred while starting harvesting.');
+            }
+        }
+
+        checkHarvestingStatus();
+
+        updateContainer.addEventListener('mouseover', function (event) {
+            if (updateGeoContextButton.disabled) {
+                dropdownContent.style.display = 'none';
+            } else {
+                event.preventDefault();
+                dropdownContent.style.display = 'block';
+            }
+        });
+
+        updateContainer.addEventListener('mouseleave', function (event) {
+            if (updateGeoContextButton.disabled) {
+                dropdownContent.style.display = 'none';
+            } else {
+                event.preventDefault();
+                dropdownContent.style.display = 'none';
+            }
+        });
+
+        updateAllLink.addEventListener('click', function (event) {
+            if (!event.target.classList.contains('disabled-link')) {
+                event.preventDefault();
+                startHarvesting(true);
+            }
+        });
+
+        updateEmptyLink.addEventListener('click', function (event) {
+            if (!event.target.classList.contains('disabled-link')) {
+                event.preventDefault();
+                startHarvesting(false);
+            }
+        });
+    });
+</script>
+{% endblock %}