From 971c36a25e358c513be9f434a49308d9e498906a Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Wed, 11 Oct 2023 00:14:28 +0200 Subject: [PATCH 1/6] Updating libraries and fixing errors - Part 1 --- app/fetcher/fetcher.py | 5 ++++- app/fetcher/fetcher_launcher.py | 17 +++++++++-------- app/fetcher/reservations_fetcher.py | 2 +- migrations/env.py | 2 ++ requirements.txt | 2 +- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/app/fetcher/fetcher.py b/app/fetcher/fetcher.py index ec2e841b..8ce7e7a2 100644 --- a/app/fetcher/fetcher.py +++ b/app/fetcher/fetcher.py @@ -3,6 +3,7 @@ from email import utils as eut from io import StringIO from typing import Optional +from sqlalchemy import text import requests @@ -30,7 +31,9 @@ def fetch(self, import_id: int) -> None: pass def _truncate(self) -> None: - db.engine.execute('truncate table {}'.format(self._table)) + with db.engine.connect() as conn: + result = conn.execute(text('TRUNCATE table {}'.format(self._table))) + conn.commit() def _log_download_finished(self): app.logger.info(f"Fetcher '{type(self).__name__}' source file downloaded.") diff --git a/app/fetcher/fetcher_launcher.py b/app/fetcher/fetcher_launcher.py index 8d0db2fb..ba061005 100644 --- a/app/fetcher/fetcher_launcher.py +++ b/app/fetcher/fetcher_launcher.py @@ -85,15 +85,15 @@ def _init_fetchers(self, dataset: str) -> None: # inits requested fetchers if dataset == 'all': # self._fetchers.append(CentersFetcher()) # replaced by CentersApiFetcher - self._fetchers.append(CentersApiFetcher()) - self._fetchers.append(OpeningHoursFetcher()) - self._fetchers.append(HealthFacilitiesFetcher()) + # self._fetchers.append(CentersApiFetcher()) + # self._fetchers.append(OpeningHoursFetcher()) + # self._fetchers.append(HealthFacilitiesFetcher()) # self._fetchers.append(DistributedFetcher()) # not updated since 2022-11-29 # self._fetchers.append(UsedFetcher()) # not updated since 2022-02-16 - self._fetchers.append(RegistrationsFetcher()) - self._fetchers.append(ReservationsFetcher()) + ## self._fetchers.append(RegistrationsFetcher()) + ## self._fetchers.append(ReservationsFetcher()) # self._fetchers.append(ReservationsApiFetcher()) # API not available anymore - self._fetchers.append(VaccinatedFetcher()) + ## self._fetchers.append(VaccinatedFetcher()) self._fetchers.append(InfectedFetcher()) self._fetchers.append(DeathsFetcher()) self._fetchers.append(MunicipalCharacteristicsFetcher()) @@ -264,5 +264,6 @@ def _set_import_finished(self) -> None: if __name__ == '__main__': - fetcher = FetcherLauncher() - fetcher.fetch(sys.argv[1]) + with app.app_context(): + fetcher = FetcherLauncher() + fetcher.fetch(sys.argv[1]) diff --git a/app/fetcher/reservations_fetcher.py b/app/fetcher/reservations_fetcher.py index 84162ed2..43edc7a0 100644 --- a/app/fetcher/reservations_fetcher.py +++ b/app/fetcher/reservations_fetcher.py @@ -18,7 +18,7 @@ def __init__(self): def fetch(self, import_id: int) -> None: df = pd.read_csv(self._url) - df = df.drop(['ockovaci_misto_nazev', 'kraj_nuts_kod', 'kraj_nazev'], axis=1) + df = df.drop(['ockovaci_misto_nazev', 'kraj_nuts_kod', 'kraj_nazev', 'id'], axis=1) df = df.groupby(['datum', 'ockovaci_misto_id', 'kalendar_ockovani'], dropna=False).sum().reset_index() diff --git a/migrations/env.py b/migrations/env.py index 5729258e..ba726c8c 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -32,6 +32,7 @@ exclude_tables = config.get_section('alembic:exclude').get('tables', '').split(',') + def include_object(object, name, type_, *args, **kwargs): if type_ == 'table' and name in exclude_tables: return False @@ -41,6 +42,7 @@ def include_object(object, name, type_, *args, **kwargs): else: return True + def run_migrations_offline(): """Run migrations in 'offline' mode. diff --git a/requirements.txt b/requirements.txt index 907f7185..376d06af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,4 @@ setuptools>=54.1.0 alembic>=1.5.0 urllib3>=1.26.0 openpyxl>=3.0.0 -pandas==1.3.5 +pandas>=2.1.0 From 3714bb335256fe02a28023c2177e132f4ef46f4f Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Thu, 12 Oct 2023 22:21:14 +0200 Subject: [PATCH 2/6] Fix - Fetchers running --- app/fetcher/deaths_fetcher.py | 3 ++- app/fetcher/fetcher_launcher.py | 6 +++--- app/fetcher/tests_fetcher.py | 18 +++++++++++++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/app/fetcher/deaths_fetcher.py b/app/fetcher/deaths_fetcher.py index a7bd4ebe..40a968e4 100644 --- a/app/fetcher/deaths_fetcher.py +++ b/app/fetcher/deaths_fetcher.py @@ -21,7 +21,8 @@ def fetch(self, import_id: int) -> None: df = df[['datum', 'vek', 'kraj_nuts_kod']] df['kraj_nuts_kod'] = df['kraj_nuts_kod'].fillna('-') - + df = df.dropna(subset=['vek']) + df['vek'] = df['vek'].astype('int') df = df.groupby(df.columns.tolist(), dropna=False).size().reset_index(name='pocet') self._truncate() diff --git a/app/fetcher/fetcher_launcher.py b/app/fetcher/fetcher_launcher.py index ba061005..3e74fe30 100644 --- a/app/fetcher/fetcher_launcher.py +++ b/app/fetcher/fetcher_launcher.py @@ -90,10 +90,10 @@ def _init_fetchers(self, dataset: str) -> None: # self._fetchers.append(HealthFacilitiesFetcher()) # self._fetchers.append(DistributedFetcher()) # not updated since 2022-11-29 # self._fetchers.append(UsedFetcher()) # not updated since 2022-02-16 - ## self._fetchers.append(RegistrationsFetcher()) - ## self._fetchers.append(ReservationsFetcher()) + self._fetchers.append(RegistrationsFetcher()) + self._fetchers.append(ReservationsFetcher()) # self._fetchers.append(ReservationsApiFetcher()) # API not available anymore - ## self._fetchers.append(VaccinatedFetcher()) + self._fetchers.append(VaccinatedFetcher()) self._fetchers.append(InfectedFetcher()) self._fetchers.append(DeathsFetcher()) self._fetchers.append(MunicipalCharacteristicsFetcher()) diff --git a/app/fetcher/tests_fetcher.py b/app/fetcher/tests_fetcher.py index 98e845ec..75d17c17 100644 --- a/app/fetcher/tests_fetcher.py +++ b/app/fetcher/tests_fetcher.py @@ -17,7 +17,23 @@ def __init__(self): def fetch(self, import_id: int) -> None: df = pd.read_csv(self._url) - + df['incidence_pozitivni'] = df['incidence_pozitivni'].fillna(-1).astype(int) + df['pozit_typologie_test_indik_diagnosticka'] = df['pozit_typologie_test_indik_diagnosticka'].fillna(-1).astype(int) + df['pozit_typologie_test_indik_epidemiologicka'] = df['pozit_typologie_test_indik_epidemiologicka'].fillna(-1).astype(int) + df['pozit_typologie_test_indik_preventivni'] = df['pozit_typologie_test_indik_preventivni'].fillna( + -1).astype(int) + df['pozit_typologie_test_indik_ostatni'] = df['pozit_typologie_test_indik_ostatni'].fillna(-1).astype( + int) + df['PCR_pozit_sympt'] = df['PCR_pozit_sympt'].fillna(-1).astype( + int) + df['AG_pozit_symp'] = df['AG_pozit_symp'].fillna(-1).astype( + int) + df['AG_pozit_asymp_PCR_conf'] = df['AG_pozit_asymp_PCR_conf'].fillna(-1).astype( + int) + df['PCR_pozit_asymp'] = df['PCR_pozit_asymp'].fillna(-1).astype( + int) + df['PCR_pozit_sympt'] = df['PCR_pozit_sympt'].fillna(-1).astype( + int) df = df.drop(columns=['id']) df = df.rename(columns=str.lower) From 8d18797119196a9b09766d1d67013869bfd2c5aa Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Thu, 12 Oct 2023 23:59:03 +0200 Subject: [PATCH 3/6] Compute metrics fix - new case syntax --- app/etl/center_metrics_etl.py | 50 +++++++++++++++++------------------ app/etl/cr_metrics_etl.py | 30 ++++++++++----------- app/etl/kraj_metrics_etl.py | 36 ++++++++++++------------- app/etl/okres_metrics_etl.py | 20 +++++++------- 4 files changed, 68 insertions(+), 68 deletions(-) diff --git a/app/etl/center_metrics_etl.py b/app/etl/center_metrics_etl.py index a88dfed4..2c458d7f 100644 --- a/app/etl/center_metrics_etl.py +++ b/app/etl/center_metrics_etl.py @@ -44,9 +44,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for each vaccination center.""" registrations = db.session.query( OckovaciMisto.id, func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).outerjoin(OckovaniRegistrace, and_(OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id, OckovaniRegistrace.import_id == self._import_id)) \ .group_by(OckovaciMisto.id) \ .all() @@ -68,15 +68,15 @@ def _compute_reservations(self): reservations = db.session.query( OckovaciMisto.id, func.coalesce(func.sum(OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), 0).label('rezervace_celkem'), - func.coalesce(func.sum(case([(OckovaniRezervace.datum >= self._date, OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici_1"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici_2"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici_3"), - func.coalesce(func.sum(case([(OckovaniRezervace.datum == self._date, OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita_1"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita_2"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita_3"), - func.min(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1', OckovaniRezervace.volna_kapacita > 0), OckovaniRezervace.datum)], else_=None)).label("rezervace_nejblizsi_volno") + func.coalesce(func.sum(case((OckovaniRezervace.datum >= self._date, OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici_1"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici_2"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici_3"), + func.coalesce(func.sum(case((OckovaniRezervace.datum == self._date, OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita_1"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita_2"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita_3"), + func.min(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1', OckovaniRezervace.volna_kapacita > 0), OckovaniRezervace.datum), else_=None)).label("rezervace_nejblizsi_volno") ).outerjoin(OckovaniRezervace, OckovaciMisto.id == OckovaniRezervace.ockovaci_misto_id) \ .group_by(OckovaciMisto.id) \ .all() @@ -102,10 +102,10 @@ def _compute_reservations(self): def _compute_vaccinated(self): """Computes metrics based on vaccinated people dataset for each vaccination center.""" vaccinated = db.session.query(OckovaciMisto.id, func.coalesce(func.sum(OckovaniLide.pocet), 0).label('ockovani_pocet_davek'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4')) \ + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4')) \ .outerjoin(OckovaniLide, and_(OckovaciMisto.nrpzs_kod == OckovaniLide.zarizeni_kod, OckovaniLide.datum < self._date)) \ .join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .filter(or_(and_(OckovaciMisto.status == True, OckovaciMisto.nrpzs_kod.in_(queries.unique_nrpzs_active_subquery())), @@ -130,9 +130,9 @@ def _compute_distributed(self): """Computes metrics based on distributed vaccines dataset for each vaccination center.""" distributed = db.session.query( OckovaciMisto.id, ( - func.coalesce(func.sum(case([(and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Příjem'), OckovaniDistribuce.pocet_davek)], else_=0)), 0) - + func.coalesce(func.sum(case([(and_(OckovaciMisto.id == OckovaniDistribuce.cilove_ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek)], else_=0)), 0) - - func.coalesce(func.sum(case([(and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek)], else_=0)), 0)) + func.coalesce(func.sum(case((and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Příjem'), OckovaniDistribuce.pocet_davek), else_=0)), 0) + + func.coalesce(func.sum(case((and_(OckovaciMisto.id == OckovaniDistribuce.cilove_ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek), else_=0)), 0) + - func.coalesce(func.sum(case((and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek), else_=0)), 0)) .label('vakciny_prijate_pocet') ).outerjoin(OckovaniDistribuce, and_( or_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaciMisto.id == OckovaniDistribuce.cilove_ockovaci_misto_id), @@ -230,8 +230,8 @@ def _compute_derived(self): success_ratio_7 = db.session.query( OckovaciMisto.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(7)) \ @@ -247,8 +247,8 @@ def _compute_derived(self): success_ratio_14 = db.session.query( OckovaciMisto.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(14)) \ @@ -264,8 +264,8 @@ def _compute_derived(self): success_ratio_30 = db.session.query( OckovaciMisto.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(30)) \ diff --git a/app/etl/cr_metrics_etl.py b/app/etl/cr_metrics_etl.py index 06502aab..316099a9 100644 --- a/app/etl/cr_metrics_etl.py +++ b/app/etl/cr_metrics_etl.py @@ -45,7 +45,7 @@ def _compute_population(self): """Computes metrics based on population for cr.""" population = db.session.query( func.sum(Populace.pocet).label('pocet_obyvatel_celkem'), - func.sum(case([(Populace.vek >= 18, Populace.pocet)], else_=0)).label('pocet_obyvatel_dospeli') + func.sum(case((Populace.vek >= 18, Populace.pocet), else_=0)).label('pocet_obyvatel_dospeli') ).filter(Populace.orp_kod == 'CZ0') \ .one() @@ -61,9 +61,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for cr.""" registrations = db.session.query( func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .one() @@ -111,10 +111,10 @@ def _compute_vaccinated(self): """Computes metrics based on vaccinated people dataset for cr.""" vaccinated = db.session.query( func.coalesce(func.sum(OckovaniLide.pocet), 0).label('ockovani_pocet_davek'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4') + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4') ).join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .filter(OckovaniLide.datum < self._date) \ .one() @@ -133,7 +133,7 @@ def _compute_vaccinated(self): def _compute_distributed(self): """Computes metrics based on distributed vaccines dataset for cr.""" distributed = db.session.query( - func.sum(case([(OckovaniDistribuce.akce == 'Příjem', OckovaniDistribuce.pocet_davek)], else_=0)).label('vakciny_prijate_pocet') + func.sum(case((OckovaniDistribuce.akce == 'Příjem', OckovaniDistribuce.pocet_davek), else_=0)).label('vakciny_prijate_pocet') ).filter(OckovaniDistribuce.datum < self._date) \ .one() @@ -189,8 +189,8 @@ def _compute_derived(self): )) success_ratio_7 = db.session.query( - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(7)) \ .one() @@ -201,8 +201,8 @@ def _compute_derived(self): )) success_ratio_14 = db.session.query( - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(14)) \ .one() @@ -213,8 +213,8 @@ def _compute_derived(self): )) success_ratio_30 = db.session.query( - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(30)) \ .one() diff --git a/app/etl/kraj_metrics_etl.py b/app/etl/kraj_metrics_etl.py index 2ac8e26a..2729312e 100644 --- a/app/etl/kraj_metrics_etl.py +++ b/app/etl/kraj_metrics_etl.py @@ -45,7 +45,7 @@ def _compute_population(self): """Computes metrics based on population for each kraj.""" population = db.session.query( Kraj.id, func.sum(Populace.pocet).label('pocet_obyvatel_celkem'), - func.sum(case([(Populace.vek >= 18, Populace.pocet)], else_=0)).label('pocet_obyvatel_dospeli') + func.sum(case((Populace.vek >= 18, Populace.pocet), else_=0)).label('pocet_obyvatel_dospeli') ).join(Populace, Populace.orp_kod == Kraj.id) \ .group_by(Kraj.id) @@ -63,9 +63,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for each kraj.""" registrations = db.session.query( Okres.kraj_id, func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).join(OckovaciMisto, OckovaciMisto.okres_id == Okres.id) \ .outerjoin(OckovaniRegistrace, and_(OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id, OckovaniRegistrace.import_id == self._import_id)) \ .group_by(Okres.kraj_id) \ @@ -125,10 +125,10 @@ def _compute_vaccinated(self): """Computes metrics based on vaccinated people dataset for each kraj.""" vaccinated = db.session.query( Kraj.id, func.coalesce(func.sum(OckovaniLide.pocet), 0).label('ockovani_pocet_davek'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4') + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4') ).outerjoin(OckovaniLide, and_(OckovaniLide.kraj_nuts_kod == Kraj.id, OckovaniLide.datum < self._date)) \ .join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .group_by(Kraj.id) \ @@ -147,10 +147,10 @@ def _compute_vaccinated(self): vaccinated_bydl = db.session.query( Kraj.id, - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne_bydl'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne_bydl'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3_bydl'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4_bydl') + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne_bydl'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne_bydl'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3_bydl'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4_bydl') ).outerjoin(OckovaniLide, and_(OckovaniLide.kraj_bydl_nuts == Kraj.id, OckovaniLide.datum < self._date)) \ .join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .group_by(Kraj.id) \ @@ -276,8 +276,8 @@ def _compute_derived(self): success_ratio_7 = db.session.query( Kraj.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).join(Okres, Okres.kraj_id == Kraj.id) \ .join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ @@ -295,8 +295,8 @@ def _compute_derived(self): success_ratio_14 = db.session.query( Kraj.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).join(Okres, Okres.kraj_id == Kraj.id) \ .join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ @@ -314,8 +314,8 @@ def _compute_derived(self): success_ratio_30 = db.session.query( Kraj.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).join(Okres, Okres.kraj_id == Kraj.id) \ .join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ diff --git a/app/etl/okres_metrics_etl.py b/app/etl/okres_metrics_etl.py index 82e69963..e7154438 100644 --- a/app/etl/okres_metrics_etl.py +++ b/app/etl/okres_metrics_etl.py @@ -43,7 +43,7 @@ def _compute_population(self): """Computes metrics based on population for each okres.""" population = db.session.query( Okres.id, func.sum(Populace.pocet).label('pocet_obyvatel_celkem'), - func.sum(case([(Populace.vek >= 18, Populace.pocet)], else_=0)).label('pocet_obyvatel_dospeli') + func.sum(case((Populace.vek >= 18, Populace.pocet), else_=0)).label('pocet_obyvatel_dospeli') ).join(Populace, Populace.orp_kod == Okres.id) \ .group_by(Okres.id) @@ -61,9 +61,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for each okres.""" registrations = db.session.query( OckovaciMisto.okres_id, func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).outerjoin(OckovaniRegistrace, and_(OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id, OckovaniRegistrace.import_id == self._import_id)) \ .filter(OckovaciMisto.okres_id != None) \ .group_by(OckovaciMisto.okres_id) \ @@ -222,8 +222,8 @@ def _compute_derived(self): success_ratio_7 = db.session.query( Okres.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ @@ -240,8 +240,8 @@ def _compute_derived(self): success_ratio_14 = db.session.query( Okres.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ @@ -258,8 +258,8 @@ def _compute_derived(self): success_ratio_30 = db.session.query( Okres.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ From ad2bbe8c08525acf06ca1f0e2b42b3c3521a3b2a Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Fri, 13 Oct 2023 16:05:56 +0200 Subject: [PATCH 4/6] Oprava dataframe pro rolling --- app/queries.py | 69 +++++++++++++++++++++++++++-------------- app/templates/base.html | 2 +- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/app/queries.py b/app/queries.py index 689f31d8..5a9baa38 100644 --- a/app/queries.py +++ b/app/queries.py @@ -192,7 +192,8 @@ def find_free_vaccines_available(nrpzs_kod=None, okres_id=None, kraj_id=None): PrakticiKapacity.kontakt_email, PrakticiKapacity.expirace, PrakticiKapacity.poznamka, PrakticiKapacity.kraj, ZdravotnickeStredisko.nrpzs_kod, ZdravotnickeStredisko.latitude, ZdravotnickeStredisko.longitude) \ - .outerjoin(ZdravotnickeStredisko, ZdravotnickeStredisko.zdravotnicke_zarizeni_kod == PrakticiKapacity.zdravotnicke_zarizeni_kod) \ + .outerjoin(ZdravotnickeStredisko, + ZdravotnickeStredisko.zdravotnicke_zarizeni_kod == PrakticiKapacity.zdravotnicke_zarizeni_kod) \ .filter(or_(func.left(PrakticiKapacity.zdravotnicke_zarizeni_kod, 11) == nrpzs_kod, nrpzs_kod is None)) \ .filter(or_(ZdravotnickeStredisko.okres_kod == okres_id, okres_id is None)) \ .filter(or_(ZdravotnickeStredisko.kraj_kod == kraj_id, kraj_id is None)) \ @@ -477,17 +478,21 @@ def count_registrations(filter_column, filter_value): df['fronta_pocet'] = df[['pocet']].where(df['rezervace'] == False).fillna(0).astype('int') df['fronta_cekani'] = (df['dnes'] - df['datum']).astype('timedelta64[ns]').dt.days df['fronta_pocet_x_cekani'] = df['fronta_pocet'] * df['fronta_cekani'] - df['s_terminem_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['ockovani'] < 1)).fillna(0).astype('int') + df['s_terminem_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['ockovani'] < 1)).fillna(0).astype( + 'int') df['registrace_7'] = df[['pocet']].where(df['datum'] >= get_import_date() - timedelta(7)) - df['registrace_7_rez'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(7))) + df['registrace_7_rez'] = df[['pocet']].where( + (df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(7))) # df['registrace_14'] = df[['pocet']].where(df['datum'] >= get_import_date() - timedelta(14)) # df['registrace_14_rez'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(14))) # df['registrace_30'] = df[['pocet']].where(df['datum'] >= get_import_date() - timedelta(30)) # df['registrace_30_rez'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(30))) df['rezervace_7_cekani'] = (df['datum_rezervace_fix'] - df['datum']).astype('timedelta64[ns]').dt.days - df['rezervace_7_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(7))) + df['rezervace_7_pocet'] = df[['pocet']].where( + (df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(7))) df['rezervace_7_pocet_x_cekani'] = df['rezervace_7_cekani'] * df['rezervace_7_pocet'] - df['rezervace_14_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(14))) + df['rezervace_14_pocet'] = df[['pocet']].where( + (df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(14))) df = df.groupby(['vekova_skupina', 'povolani']).sum() @@ -501,9 +506,12 @@ def count_registrations(filter_column, filter_value): # df['registrace_30'] = df['registrace_30'].astype('int') # df['registrace_30_rez'] = df['registrace_30_rez'].astype('int') df['fronta_prumer_cekani'] = ((df['fronta_pocet_x_cekani'] / df['fronta_pocet']) / 7).replace({np.nan: None}) - df['rezervace_prumer_cekani'] = ((df['rezervace_7_pocet_x_cekani'] / df['rezervace_7_pocet']) / 7).replace({np.nan: None}) + df['rezervace_prumer_cekani'] = ((df['rezervace_7_pocet_x_cekani'] / df['rezervace_7_pocet']) / 7).replace( + {np.nan: None}) - df = df[(df['fronta_pocet'] > 0) | df['fronta_prumer_cekani'].notnull() | df['rezervace_prumer_cekani'].notnull() | df['uspesnost_7'].notnull()] + df = df[ + (df['fronta_pocet'] > 0) | df['fronta_prumer_cekani'].notnull() | df['rezervace_prumer_cekani'].notnull() | df[ + 'uspesnost_7'].notnull()] return df.reset_index().sort_values(by=['vekova_skupina', 'povolani']) @@ -822,7 +830,8 @@ def count_free_slots(center_id=None): def count_vaccinated_week(): - return db.session.query(column('datum'), column('pocet_1'), column('pocet_2'), column('pocet_3'), column('pocet_4'), column('pocet_celkem')).from_statement(text( + return db.session.query(column('datum'), column('pocet_1'), column('pocet_2'), column('pocet_3'), column('pocet_4'), + column('pocet_celkem')).from_statement(text( f""" select datum, sum(case when poradi_davky = 1 then pocet else 0 end) pocet_1, @@ -1009,7 +1018,8 @@ def count_vaccinated_unvaccinated_comparison_age(): .replace({np.nan: 0}) for g in ['plne', 'posilujici']: - df_norm[d + '_' + g + '_ratio'] = (df_norm[d + '_bez_norm'] / df_norm[d + '_' + g + '_norm']).replace({np.inf: np.nan}) + df_norm[d + '_' + g + '_ratio'] = (df_norm[d + '_bez_norm'] / df_norm[d + '_' + g + '_norm']).replace( + {np.inf: np.nan}) df_norm.loc[df_norm['populace_' + g + '_zastoupeni'] < 0.05, d + '_' + g + '_ratio'] = np.nan df_norm[d + '_' + g + '_ratio'] = df_norm[d + '_' + g + '_ratio'].replace({np.nan: None}) @@ -1297,9 +1307,8 @@ def get_infected_graph_data(): {'0-11': '0-17', '12-15': '0-17', '16-17': '0-17', '18-24': '18-29', '25-29': '18-29', '30-34': '30-39', '35-39': '30-39', '40-44': '40-49', '45-49': '40-49', '50-54': '50-59', '55-59': '50-59', '60-64': '60-69', '65-69': '60-69', '70-74': '70-79', '75-79': '70-79'}) - df = df.groupby(['vekova_skupina_grp', 'datum']).sum() - df_sum = df.rolling(7).sum() + df_sum = df[['pocet_vek', 'pocet_nakazeni']].rolling(7).sum() df_sum['pocet_nakazeni_norm'] = ((df_sum['pocet_nakazeni'] / df_sum['pocet_vek']) * 100000) @@ -1350,7 +1359,7 @@ def get_deaths_graph_data(): '65-69': '60-69', '70-74': '70-79', '75-79': '70-79'}) df = df.groupby(['vekova_skupina_grp', 'datum']).sum() - df_sum = df.rolling(7).sum() + df_sum = df[['pocet_vek', 'pocet_umrti']].rolling(7).sum() df_sum['pocet_umrti_norm'] = ((df_sum['pocet_umrti'] / df_sum['pocet_vek']) * 100000) @@ -1375,11 +1384,15 @@ def get_tests_graph_data(): df = df.set_index('datum') df_sum = df.rolling(7).sum() - df_sum['pozitivita_diagnosticka'] = (df_sum['pozit_typologie_test_indik_diagnosticka'] / df_sum['typologie_test_indik_diagnosticka']) * 100 - df_sum['pozitivita_epidemiologicka'] = (df_sum['pozit_typologie_test_indik_epidemiologicka'] / df_sum['typologie_test_indik_epidemiologicka']) * 100 - df_sum['pozitivita_preventivni'] = (df_sum['pozit_typologie_test_indik_preventivni'] / df_sum['typologie_test_indik_preventivni']) * 100 + df_sum['pozitivita_diagnosticka'] = (df_sum['pozit_typologie_test_indik_diagnosticka'] / df_sum[ + 'typologie_test_indik_diagnosticka']) * 100 + df_sum['pozitivita_epidemiologicka'] = (df_sum['pozit_typologie_test_indik_epidemiologicka'] / df_sum[ + 'typologie_test_indik_epidemiologicka']) * 100 + df_sum['pozitivita_preventivni'] = (df_sum['pozit_typologie_test_indik_preventivni'] / df_sum[ + 'typologie_test_indik_preventivni']) * 100 - df = pd.merge(df_sum[['pozitivita_diagnosticka', 'pozitivita_epidemiologicka', 'pozitivita_preventivni']], df, left_index=True, right_index=True) + df = pd.merge(df_sum[['pozitivita_diagnosticka', 'pozitivita_epidemiologicka', 'pozitivita_preventivni']], df, + left_index=True, right_index=True) df = df[df.index >= df.index.min() + timedelta(7)] @@ -1457,18 +1470,26 @@ def get_tests_orp_graph_data(): def get_hospital_capacities_graph_data(): - capacities = pd.read_sql_query(f"select * from kapacity_nemocnic where datum < '{get_import_date()}'", db.engine).fillna(0) - capacities['luzka_standard_kyslik_kapacita_volna_covid_pozitivni'] += capacities['inf_luzka_kyslik_kapacita_volna_covid_pozitivni'] - capacities['luzka_standard_kyslik_kapacita_volna_covid_negativni'] += capacities['inf_luzka_kyslik_kapacita_volna_covid_negativni'] + capacities = pd.read_sql_query(f"select * from kapacity_nemocnic where datum < '{get_import_date()}'", + db.engine).fillna(0) + capacities['luzka_standard_kyslik_kapacita_volna_covid_pozitivni'] += capacities[ + 'inf_luzka_kyslik_kapacita_volna_covid_pozitivni'] + capacities['luzka_standard_kyslik_kapacita_volna_covid_negativni'] += capacities[ + 'inf_luzka_kyslik_kapacita_volna_covid_negativni'] capacities['luzka_standard_kyslik_kapacita_celkem'] += capacities['inf_luzka_kyslik_kapacita_celkem'] - capacities['luzka_hfno_cpap_kapacita_volna_covid_pozitivni'] += capacities['inf_luzka_hfno_kapacita_volna_covid_pozitivni'] - capacities['luzka_hfno_cpap_kapacita_volna_covid_negativni'] += capacities['inf_luzka_hfno_kapacita_volna_covid_negativni'] + capacities['luzka_hfno_cpap_kapacita_volna_covid_pozitivni'] += capacities[ + 'inf_luzka_hfno_kapacita_volna_covid_pozitivni'] + capacities['luzka_hfno_cpap_kapacita_volna_covid_negativni'] += capacities[ + 'inf_luzka_hfno_kapacita_volna_covid_negativni'] capacities['luzka_hfno_cpap_kapacita_celkem'] += capacities['inf_luzka_hfno_kapacita_celkem'] - capacities['luzka_upv_niv_kapacita_volna_covid_pozitivni'] += capacities['inf_luzka_upv_kapacita_volna_covid_pozitivni'] - capacities['luzka_upv_niv_kapacita_volna_covid_negativni'] += capacities['inf_luzka_upv_kapacita_volna_covid_negativni'] + capacities['luzka_upv_niv_kapacita_volna_covid_pozitivni'] += capacities[ + 'inf_luzka_upv_kapacita_volna_covid_pozitivni'] + capacities['luzka_upv_niv_kapacita_volna_covid_negativni'] += capacities[ + 'inf_luzka_upv_kapacita_volna_covid_negativni'] capacities['luzka_upv_niv_kapacita_celkem'] += capacities['inf_luzka_upv_kapacita_celkem'] - capacities_21 = pd.read_sql_query("select * from kapacity_nemocnic_21 where datum >= '2021-03-30'", db.engine).fillna(0) + capacities_21 = pd.read_sql_query("select * from kapacity_nemocnic_21 where datum >= '2021-03-30'", + db.engine).fillna(0) capacities_20 = pd.read_sql_query("select * from kapacity_nemocnic_20", db.engine).fillna(0) diff --git a/app/templates/base.html b/app/templates/base.html index da84f4ad..bca7a7c5 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -101,7 +101,7 @@ diff --git a/app/templates/index.html b/app/templates/index.html index 31738812..2e8abbd8 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -115,6 +115,9 @@

COVID-19 data o očkování

Aktuality a důležité informace

    + {{ zprava('Znovuspuštění aktualizace webu', '13. 10. 2023', + 'Aktualizace webu byla 13. 10. 2023 znovu zahájena, zatím v pilotním režimu.') }} + {{ zprava('Ukončení aktualizace webu', '1. 7. 2023', 'Aktualizace webu byla 1. 7. 2023 ukončena.') }} From d4de0fd78cb66f651480a0f95da7289c53bfb2f0 Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Wed, 25 Oct 2023 00:04:17 +0200 Subject: [PATCH 6/6] Fix for new behaviour of groupby pandas --- app/queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/queries.py b/app/queries.py index 5a9baa38..0a5785ab 100644 --- a/app/queries.py +++ b/app/queries.py @@ -494,7 +494,7 @@ def count_registrations(filter_column, filter_value): df['rezervace_14_pocet'] = df[['pocet']].where( (df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(14))) - df = df.groupby(['vekova_skupina', 'povolani']).sum() + df = df.groupby(['vekova_skupina', 'povolani'], group_keys=False, level=0).sum() df['uspesnost_7'] = ((df['registrace_7_rez'] / df['registrace_7']) * 100).replace({np.nan: None}) # df['uspesnost_14'] = ((df['registrace_14_rez'] / df['registrace_14']) * 100).replace({np.nan: None})