From 971c36a25e358c513be9f434a49308d9e498906a Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Wed, 11 Oct 2023 00:14:28 +0200 Subject: [PATCH 01/11] Updating libraries and fixing errors - Part 1 --- app/fetcher/fetcher.py | 5 ++++- app/fetcher/fetcher_launcher.py | 17 +++++++++-------- app/fetcher/reservations_fetcher.py | 2 +- migrations/env.py | 2 ++ requirements.txt | 2 +- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/app/fetcher/fetcher.py b/app/fetcher/fetcher.py index ec2e841b..8ce7e7a2 100644 --- a/app/fetcher/fetcher.py +++ b/app/fetcher/fetcher.py @@ -3,6 +3,7 @@ from email import utils as eut from io import StringIO from typing import Optional +from sqlalchemy import text import requests @@ -30,7 +31,9 @@ def fetch(self, import_id: int) -> None: pass def _truncate(self) -> None: - db.engine.execute('truncate table {}'.format(self._table)) + with db.engine.connect() as conn: + result = conn.execute(text('TRUNCATE table {}'.format(self._table))) + conn.commit() def _log_download_finished(self): app.logger.info(f"Fetcher '{type(self).__name__}' source file downloaded.") diff --git a/app/fetcher/fetcher_launcher.py b/app/fetcher/fetcher_launcher.py index 8d0db2fb..ba061005 100644 --- a/app/fetcher/fetcher_launcher.py +++ b/app/fetcher/fetcher_launcher.py @@ -85,15 +85,15 @@ def _init_fetchers(self, dataset: str) -> None: # inits requested fetchers if dataset == 'all': # self._fetchers.append(CentersFetcher()) # replaced by CentersApiFetcher - self._fetchers.append(CentersApiFetcher()) - self._fetchers.append(OpeningHoursFetcher()) - self._fetchers.append(HealthFacilitiesFetcher()) + # self._fetchers.append(CentersApiFetcher()) + # self._fetchers.append(OpeningHoursFetcher()) + # self._fetchers.append(HealthFacilitiesFetcher()) # self._fetchers.append(DistributedFetcher()) # not updated since 2022-11-29 # self._fetchers.append(UsedFetcher()) # not updated since 2022-02-16 - self._fetchers.append(RegistrationsFetcher()) - self._fetchers.append(ReservationsFetcher()) + ## self._fetchers.append(RegistrationsFetcher()) + ## self._fetchers.append(ReservationsFetcher()) # self._fetchers.append(ReservationsApiFetcher()) # API not available anymore - self._fetchers.append(VaccinatedFetcher()) + ## self._fetchers.append(VaccinatedFetcher()) self._fetchers.append(InfectedFetcher()) self._fetchers.append(DeathsFetcher()) self._fetchers.append(MunicipalCharacteristicsFetcher()) @@ -264,5 +264,6 @@ def _set_import_finished(self) -> None: if __name__ == '__main__': - fetcher = FetcherLauncher() - fetcher.fetch(sys.argv[1]) + with app.app_context(): + fetcher = FetcherLauncher() + fetcher.fetch(sys.argv[1]) diff --git a/app/fetcher/reservations_fetcher.py b/app/fetcher/reservations_fetcher.py index 84162ed2..43edc7a0 100644 --- a/app/fetcher/reservations_fetcher.py +++ b/app/fetcher/reservations_fetcher.py @@ -18,7 +18,7 @@ def __init__(self): def fetch(self, import_id: int) -> None: df = pd.read_csv(self._url) - df = df.drop(['ockovaci_misto_nazev', 'kraj_nuts_kod', 'kraj_nazev'], axis=1) + df = df.drop(['ockovaci_misto_nazev', 'kraj_nuts_kod', 'kraj_nazev', 'id'], axis=1) df = df.groupby(['datum', 'ockovaci_misto_id', 'kalendar_ockovani'], dropna=False).sum().reset_index() diff --git a/migrations/env.py b/migrations/env.py index 5729258e..ba726c8c 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -32,6 +32,7 @@ exclude_tables = config.get_section('alembic:exclude').get('tables', '').split(',') + def include_object(object, name, type_, *args, **kwargs): if type_ == 'table' and name in exclude_tables: return False @@ -41,6 +42,7 @@ def include_object(object, name, type_, *args, **kwargs): else: return True + def run_migrations_offline(): """Run migrations in 'offline' mode. diff --git a/requirements.txt b/requirements.txt index 907f7185..376d06af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,4 @@ setuptools>=54.1.0 alembic>=1.5.0 urllib3>=1.26.0 openpyxl>=3.0.0 -pandas==1.3.5 +pandas>=2.1.0 From 3714bb335256fe02a28023c2177e132f4ef46f4f Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Thu, 12 Oct 2023 22:21:14 +0200 Subject: [PATCH 02/11] Fix - Fetchers running --- app/fetcher/deaths_fetcher.py | 3 ++- app/fetcher/fetcher_launcher.py | 6 +++--- app/fetcher/tests_fetcher.py | 18 +++++++++++++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/app/fetcher/deaths_fetcher.py b/app/fetcher/deaths_fetcher.py index a7bd4ebe..40a968e4 100644 --- a/app/fetcher/deaths_fetcher.py +++ b/app/fetcher/deaths_fetcher.py @@ -21,7 +21,8 @@ def fetch(self, import_id: int) -> None: df = df[['datum', 'vek', 'kraj_nuts_kod']] df['kraj_nuts_kod'] = df['kraj_nuts_kod'].fillna('-') - + df = df.dropna(subset=['vek']) + df['vek'] = df['vek'].astype('int') df = df.groupby(df.columns.tolist(), dropna=False).size().reset_index(name='pocet') self._truncate() diff --git a/app/fetcher/fetcher_launcher.py b/app/fetcher/fetcher_launcher.py index ba061005..3e74fe30 100644 --- a/app/fetcher/fetcher_launcher.py +++ b/app/fetcher/fetcher_launcher.py @@ -90,10 +90,10 @@ def _init_fetchers(self, dataset: str) -> None: # self._fetchers.append(HealthFacilitiesFetcher()) # self._fetchers.append(DistributedFetcher()) # not updated since 2022-11-29 # self._fetchers.append(UsedFetcher()) # not updated since 2022-02-16 - ## self._fetchers.append(RegistrationsFetcher()) - ## self._fetchers.append(ReservationsFetcher()) + self._fetchers.append(RegistrationsFetcher()) + self._fetchers.append(ReservationsFetcher()) # self._fetchers.append(ReservationsApiFetcher()) # API not available anymore - ## self._fetchers.append(VaccinatedFetcher()) + self._fetchers.append(VaccinatedFetcher()) self._fetchers.append(InfectedFetcher()) self._fetchers.append(DeathsFetcher()) self._fetchers.append(MunicipalCharacteristicsFetcher()) diff --git a/app/fetcher/tests_fetcher.py b/app/fetcher/tests_fetcher.py index 98e845ec..75d17c17 100644 --- a/app/fetcher/tests_fetcher.py +++ b/app/fetcher/tests_fetcher.py @@ -17,7 +17,23 @@ def __init__(self): def fetch(self, import_id: int) -> None: df = pd.read_csv(self._url) - + df['incidence_pozitivni'] = df['incidence_pozitivni'].fillna(-1).astype(int) + df['pozit_typologie_test_indik_diagnosticka'] = df['pozit_typologie_test_indik_diagnosticka'].fillna(-1).astype(int) + df['pozit_typologie_test_indik_epidemiologicka'] = df['pozit_typologie_test_indik_epidemiologicka'].fillna(-1).astype(int) + df['pozit_typologie_test_indik_preventivni'] = df['pozit_typologie_test_indik_preventivni'].fillna( + -1).astype(int) + df['pozit_typologie_test_indik_ostatni'] = df['pozit_typologie_test_indik_ostatni'].fillna(-1).astype( + int) + df['PCR_pozit_sympt'] = df['PCR_pozit_sympt'].fillna(-1).astype( + int) + df['AG_pozit_symp'] = df['AG_pozit_symp'].fillna(-1).astype( + int) + df['AG_pozit_asymp_PCR_conf'] = df['AG_pozit_asymp_PCR_conf'].fillna(-1).astype( + int) + df['PCR_pozit_asymp'] = df['PCR_pozit_asymp'].fillna(-1).astype( + int) + df['PCR_pozit_sympt'] = df['PCR_pozit_sympt'].fillna(-1).astype( + int) df = df.drop(columns=['id']) df = df.rename(columns=str.lower) From 8d18797119196a9b09766d1d67013869bfd2c5aa Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Thu, 12 Oct 2023 23:59:03 +0200 Subject: [PATCH 03/11] Compute metrics fix - new case syntax --- app/etl/center_metrics_etl.py | 50 +++++++++++++++++------------------ app/etl/cr_metrics_etl.py | 30 ++++++++++----------- app/etl/kraj_metrics_etl.py | 36 ++++++++++++------------- app/etl/okres_metrics_etl.py | 20 +++++++------- 4 files changed, 68 insertions(+), 68 deletions(-) diff --git a/app/etl/center_metrics_etl.py b/app/etl/center_metrics_etl.py index a88dfed4..2c458d7f 100644 --- a/app/etl/center_metrics_etl.py +++ b/app/etl/center_metrics_etl.py @@ -44,9 +44,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for each vaccination center.""" registrations = db.session.query( OckovaciMisto.id, func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).outerjoin(OckovaniRegistrace, and_(OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id, OckovaniRegistrace.import_id == self._import_id)) \ .group_by(OckovaciMisto.id) \ .all() @@ -68,15 +68,15 @@ def _compute_reservations(self): reservations = db.session.query( OckovaciMisto.id, func.coalesce(func.sum(OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), 0).label('rezervace_celkem'), - func.coalesce(func.sum(case([(OckovaniRezervace.datum >= self._date, OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici_1"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici_2"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici_3"), - func.coalesce(func.sum(case([(OckovaniRezervace.datum == self._date, OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita_1"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita_2"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita_3"), - func.min(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1', OckovaniRezervace.volna_kapacita > 0), OckovaniRezervace.datum)], else_=None)).label("rezervace_nejblizsi_volno") + func.coalesce(func.sum(case((OckovaniRezervace.datum >= self._date, OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici_1"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici_2"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici_3"), + func.coalesce(func.sum(case((OckovaniRezervace.datum == self._date, OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita_1"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita_2"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita_3"), + func.min(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1', OckovaniRezervace.volna_kapacita > 0), OckovaniRezervace.datum), else_=None)).label("rezervace_nejblizsi_volno") ).outerjoin(OckovaniRezervace, OckovaciMisto.id == OckovaniRezervace.ockovaci_misto_id) \ .group_by(OckovaciMisto.id) \ .all() @@ -102,10 +102,10 @@ def _compute_reservations(self): def _compute_vaccinated(self): """Computes metrics based on vaccinated people dataset for each vaccination center.""" vaccinated = db.session.query(OckovaciMisto.id, func.coalesce(func.sum(OckovaniLide.pocet), 0).label('ockovani_pocet_davek'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4')) \ + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4')) \ .outerjoin(OckovaniLide, and_(OckovaciMisto.nrpzs_kod == OckovaniLide.zarizeni_kod, OckovaniLide.datum < self._date)) \ .join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .filter(or_(and_(OckovaciMisto.status == True, OckovaciMisto.nrpzs_kod.in_(queries.unique_nrpzs_active_subquery())), @@ -130,9 +130,9 @@ def _compute_distributed(self): """Computes metrics based on distributed vaccines dataset for each vaccination center.""" distributed = db.session.query( OckovaciMisto.id, ( - func.coalesce(func.sum(case([(and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Příjem'), OckovaniDistribuce.pocet_davek)], else_=0)), 0) - + func.coalesce(func.sum(case([(and_(OckovaciMisto.id == OckovaniDistribuce.cilove_ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek)], else_=0)), 0) - - func.coalesce(func.sum(case([(and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek)], else_=0)), 0)) + func.coalesce(func.sum(case((and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Příjem'), OckovaniDistribuce.pocet_davek), else_=0)), 0) + + func.coalesce(func.sum(case((and_(OckovaciMisto.id == OckovaniDistribuce.cilove_ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek), else_=0)), 0) + - func.coalesce(func.sum(case((and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek), else_=0)), 0)) .label('vakciny_prijate_pocet') ).outerjoin(OckovaniDistribuce, and_( or_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaciMisto.id == OckovaniDistribuce.cilove_ockovaci_misto_id), @@ -230,8 +230,8 @@ def _compute_derived(self): success_ratio_7 = db.session.query( OckovaciMisto.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(7)) \ @@ -247,8 +247,8 @@ def _compute_derived(self): success_ratio_14 = db.session.query( OckovaciMisto.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(14)) \ @@ -264,8 +264,8 @@ def _compute_derived(self): success_ratio_30 = db.session.query( OckovaciMisto.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(30)) \ diff --git a/app/etl/cr_metrics_etl.py b/app/etl/cr_metrics_etl.py index 06502aab..316099a9 100644 --- a/app/etl/cr_metrics_etl.py +++ b/app/etl/cr_metrics_etl.py @@ -45,7 +45,7 @@ def _compute_population(self): """Computes metrics based on population for cr.""" population = db.session.query( func.sum(Populace.pocet).label('pocet_obyvatel_celkem'), - func.sum(case([(Populace.vek >= 18, Populace.pocet)], else_=0)).label('pocet_obyvatel_dospeli') + func.sum(case((Populace.vek >= 18, Populace.pocet), else_=0)).label('pocet_obyvatel_dospeli') ).filter(Populace.orp_kod == 'CZ0') \ .one() @@ -61,9 +61,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for cr.""" registrations = db.session.query( func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .one() @@ -111,10 +111,10 @@ def _compute_vaccinated(self): """Computes metrics based on vaccinated people dataset for cr.""" vaccinated = db.session.query( func.coalesce(func.sum(OckovaniLide.pocet), 0).label('ockovani_pocet_davek'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4') + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4') ).join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .filter(OckovaniLide.datum < self._date) \ .one() @@ -133,7 +133,7 @@ def _compute_vaccinated(self): def _compute_distributed(self): """Computes metrics based on distributed vaccines dataset for cr.""" distributed = db.session.query( - func.sum(case([(OckovaniDistribuce.akce == 'Příjem', OckovaniDistribuce.pocet_davek)], else_=0)).label('vakciny_prijate_pocet') + func.sum(case((OckovaniDistribuce.akce == 'Příjem', OckovaniDistribuce.pocet_davek), else_=0)).label('vakciny_prijate_pocet') ).filter(OckovaniDistribuce.datum < self._date) \ .one() @@ -189,8 +189,8 @@ def _compute_derived(self): )) success_ratio_7 = db.session.query( - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(7)) \ .one() @@ -201,8 +201,8 @@ def _compute_derived(self): )) success_ratio_14 = db.session.query( - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(14)) \ .one() @@ -213,8 +213,8 @@ def _compute_derived(self): )) success_ratio_30 = db.session.query( - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(30)) \ .one() diff --git a/app/etl/kraj_metrics_etl.py b/app/etl/kraj_metrics_etl.py index 2ac8e26a..2729312e 100644 --- a/app/etl/kraj_metrics_etl.py +++ b/app/etl/kraj_metrics_etl.py @@ -45,7 +45,7 @@ def _compute_population(self): """Computes metrics based on population for each kraj.""" population = db.session.query( Kraj.id, func.sum(Populace.pocet).label('pocet_obyvatel_celkem'), - func.sum(case([(Populace.vek >= 18, Populace.pocet)], else_=0)).label('pocet_obyvatel_dospeli') + func.sum(case((Populace.vek >= 18, Populace.pocet), else_=0)).label('pocet_obyvatel_dospeli') ).join(Populace, Populace.orp_kod == Kraj.id) \ .group_by(Kraj.id) @@ -63,9 +63,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for each kraj.""" registrations = db.session.query( Okres.kraj_id, func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).join(OckovaciMisto, OckovaciMisto.okres_id == Okres.id) \ .outerjoin(OckovaniRegistrace, and_(OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id, OckovaniRegistrace.import_id == self._import_id)) \ .group_by(Okres.kraj_id) \ @@ -125,10 +125,10 @@ def _compute_vaccinated(self): """Computes metrics based on vaccinated people dataset for each kraj.""" vaccinated = db.session.query( Kraj.id, func.coalesce(func.sum(OckovaniLide.pocet), 0).label('ockovani_pocet_davek'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4') + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4') ).outerjoin(OckovaniLide, and_(OckovaniLide.kraj_nuts_kod == Kraj.id, OckovaniLide.datum < self._date)) \ .join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .group_by(Kraj.id) \ @@ -147,10 +147,10 @@ def _compute_vaccinated(self): vaccinated_bydl = db.session.query( Kraj.id, - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne_bydl'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne_bydl'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3_bydl'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4_bydl') + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne_bydl'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne_bydl'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3_bydl'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4_bydl') ).outerjoin(OckovaniLide, and_(OckovaniLide.kraj_bydl_nuts == Kraj.id, OckovaniLide.datum < self._date)) \ .join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .group_by(Kraj.id) \ @@ -276,8 +276,8 @@ def _compute_derived(self): success_ratio_7 = db.session.query( Kraj.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).join(Okres, Okres.kraj_id == Kraj.id) \ .join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ @@ -295,8 +295,8 @@ def _compute_derived(self): success_ratio_14 = db.session.query( Kraj.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).join(Okres, Okres.kraj_id == Kraj.id) \ .join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ @@ -314,8 +314,8 @@ def _compute_derived(self): success_ratio_30 = db.session.query( Kraj.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).join(Okres, Okres.kraj_id == Kraj.id) \ .join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ diff --git a/app/etl/okres_metrics_etl.py b/app/etl/okres_metrics_etl.py index 82e69963..e7154438 100644 --- a/app/etl/okres_metrics_etl.py +++ b/app/etl/okres_metrics_etl.py @@ -43,7 +43,7 @@ def _compute_population(self): """Computes metrics based on population for each okres.""" population = db.session.query( Okres.id, func.sum(Populace.pocet).label('pocet_obyvatel_celkem'), - func.sum(case([(Populace.vek >= 18, Populace.pocet)], else_=0)).label('pocet_obyvatel_dospeli') + func.sum(case((Populace.vek >= 18, Populace.pocet), else_=0)).label('pocet_obyvatel_dospeli') ).join(Populace, Populace.orp_kod == Okres.id) \ .group_by(Okres.id) @@ -61,9 +61,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for each okres.""" registrations = db.session.query( OckovaciMisto.okres_id, func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).outerjoin(OckovaniRegistrace, and_(OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id, OckovaniRegistrace.import_id == self._import_id)) \ .filter(OckovaciMisto.okres_id != None) \ .group_by(OckovaciMisto.okres_id) \ @@ -222,8 +222,8 @@ def _compute_derived(self): success_ratio_7 = db.session.query( Okres.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ @@ -240,8 +240,8 @@ def _compute_derived(self): success_ratio_14 = db.session.query( Okres.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ @@ -258,8 +258,8 @@ def _compute_derived(self): success_ratio_30 = db.session.query( Okres.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ From ad2bbe8c08525acf06ca1f0e2b42b3c3521a3b2a Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Fri, 13 Oct 2023 16:05:56 +0200 Subject: [PATCH 04/11] Oprava dataframe pro rolling --- app/queries.py | 69 +++++++++++++++++++++++++++-------------- app/templates/base.html | 2 +- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/app/queries.py b/app/queries.py index 689f31d8..5a9baa38 100644 --- a/app/queries.py +++ b/app/queries.py @@ -192,7 +192,8 @@ def find_free_vaccines_available(nrpzs_kod=None, okres_id=None, kraj_id=None): PrakticiKapacity.kontakt_email, PrakticiKapacity.expirace, PrakticiKapacity.poznamka, PrakticiKapacity.kraj, ZdravotnickeStredisko.nrpzs_kod, ZdravotnickeStredisko.latitude, ZdravotnickeStredisko.longitude) \ - .outerjoin(ZdravotnickeStredisko, ZdravotnickeStredisko.zdravotnicke_zarizeni_kod == PrakticiKapacity.zdravotnicke_zarizeni_kod) \ + .outerjoin(ZdravotnickeStredisko, + ZdravotnickeStredisko.zdravotnicke_zarizeni_kod == PrakticiKapacity.zdravotnicke_zarizeni_kod) \ .filter(or_(func.left(PrakticiKapacity.zdravotnicke_zarizeni_kod, 11) == nrpzs_kod, nrpzs_kod is None)) \ .filter(or_(ZdravotnickeStredisko.okres_kod == okres_id, okres_id is None)) \ .filter(or_(ZdravotnickeStredisko.kraj_kod == kraj_id, kraj_id is None)) \ @@ -477,17 +478,21 @@ def count_registrations(filter_column, filter_value): df['fronta_pocet'] = df[['pocet']].where(df['rezervace'] == False).fillna(0).astype('int') df['fronta_cekani'] = (df['dnes'] - df['datum']).astype('timedelta64[ns]').dt.days df['fronta_pocet_x_cekani'] = df['fronta_pocet'] * df['fronta_cekani'] - df['s_terminem_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['ockovani'] < 1)).fillna(0).astype('int') + df['s_terminem_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['ockovani'] < 1)).fillna(0).astype( + 'int') df['registrace_7'] = df[['pocet']].where(df['datum'] >= get_import_date() - timedelta(7)) - df['registrace_7_rez'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(7))) + df['registrace_7_rez'] = df[['pocet']].where( + (df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(7))) # df['registrace_14'] = df[['pocet']].where(df['datum'] >= get_import_date() - timedelta(14)) # df['registrace_14_rez'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(14))) # df['registrace_30'] = df[['pocet']].where(df['datum'] >= get_import_date() - timedelta(30)) # df['registrace_30_rez'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(30))) df['rezervace_7_cekani'] = (df['datum_rezervace_fix'] - df['datum']).astype('timedelta64[ns]').dt.days - df['rezervace_7_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(7))) + df['rezervace_7_pocet'] = df[['pocet']].where( + (df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(7))) df['rezervace_7_pocet_x_cekani'] = df['rezervace_7_cekani'] * df['rezervace_7_pocet'] - df['rezervace_14_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(14))) + df['rezervace_14_pocet'] = df[['pocet']].where( + (df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(14))) df = df.groupby(['vekova_skupina', 'povolani']).sum() @@ -501,9 +506,12 @@ def count_registrations(filter_column, filter_value): # df['registrace_30'] = df['registrace_30'].astype('int') # df['registrace_30_rez'] = df['registrace_30_rez'].astype('int') df['fronta_prumer_cekani'] = ((df['fronta_pocet_x_cekani'] / df['fronta_pocet']) / 7).replace({np.nan: None}) - df['rezervace_prumer_cekani'] = ((df['rezervace_7_pocet_x_cekani'] / df['rezervace_7_pocet']) / 7).replace({np.nan: None}) + df['rezervace_prumer_cekani'] = ((df['rezervace_7_pocet_x_cekani'] / df['rezervace_7_pocet']) / 7).replace( + {np.nan: None}) - df = df[(df['fronta_pocet'] > 0) | df['fronta_prumer_cekani'].notnull() | df['rezervace_prumer_cekani'].notnull() | df['uspesnost_7'].notnull()] + df = df[ + (df['fronta_pocet'] > 0) | df['fronta_prumer_cekani'].notnull() | df['rezervace_prumer_cekani'].notnull() | df[ + 'uspesnost_7'].notnull()] return df.reset_index().sort_values(by=['vekova_skupina', 'povolani']) @@ -822,7 +830,8 @@ def count_free_slots(center_id=None): def count_vaccinated_week(): - return db.session.query(column('datum'), column('pocet_1'), column('pocet_2'), column('pocet_3'), column('pocet_4'), column('pocet_celkem')).from_statement(text( + return db.session.query(column('datum'), column('pocet_1'), column('pocet_2'), column('pocet_3'), column('pocet_4'), + column('pocet_celkem')).from_statement(text( f""" select datum, sum(case when poradi_davky = 1 then pocet else 0 end) pocet_1, @@ -1009,7 +1018,8 @@ def count_vaccinated_unvaccinated_comparison_age(): .replace({np.nan: 0}) for g in ['plne', 'posilujici']: - df_norm[d + '_' + g + '_ratio'] = (df_norm[d + '_bez_norm'] / df_norm[d + '_' + g + '_norm']).replace({np.inf: np.nan}) + df_norm[d + '_' + g + '_ratio'] = (df_norm[d + '_bez_norm'] / df_norm[d + '_' + g + '_norm']).replace( + {np.inf: np.nan}) df_norm.loc[df_norm['populace_' + g + '_zastoupeni'] < 0.05, d + '_' + g + '_ratio'] = np.nan df_norm[d + '_' + g + '_ratio'] = df_norm[d + '_' + g + '_ratio'].replace({np.nan: None}) @@ -1297,9 +1307,8 @@ def get_infected_graph_data(): {'0-11': '0-17', '12-15': '0-17', '16-17': '0-17', '18-24': '18-29', '25-29': '18-29', '30-34': '30-39', '35-39': '30-39', '40-44': '40-49', '45-49': '40-49', '50-54': '50-59', '55-59': '50-59', '60-64': '60-69', '65-69': '60-69', '70-74': '70-79', '75-79': '70-79'}) - df = df.groupby(['vekova_skupina_grp', 'datum']).sum() - df_sum = df.rolling(7).sum() + df_sum = df[['pocet_vek', 'pocet_nakazeni']].rolling(7).sum() df_sum['pocet_nakazeni_norm'] = ((df_sum['pocet_nakazeni'] / df_sum['pocet_vek']) * 100000) @@ -1350,7 +1359,7 @@ def get_deaths_graph_data(): '65-69': '60-69', '70-74': '70-79', '75-79': '70-79'}) df = df.groupby(['vekova_skupina_grp', 'datum']).sum() - df_sum = df.rolling(7).sum() + df_sum = df[['pocet_vek', 'pocet_umrti']].rolling(7).sum() df_sum['pocet_umrti_norm'] = ((df_sum['pocet_umrti'] / df_sum['pocet_vek']) * 100000) @@ -1375,11 +1384,15 @@ def get_tests_graph_data(): df = df.set_index('datum') df_sum = df.rolling(7).sum() - df_sum['pozitivita_diagnosticka'] = (df_sum['pozit_typologie_test_indik_diagnosticka'] / df_sum['typologie_test_indik_diagnosticka']) * 100 - df_sum['pozitivita_epidemiologicka'] = (df_sum['pozit_typologie_test_indik_epidemiologicka'] / df_sum['typologie_test_indik_epidemiologicka']) * 100 - df_sum['pozitivita_preventivni'] = (df_sum['pozit_typologie_test_indik_preventivni'] / df_sum['typologie_test_indik_preventivni']) * 100 + df_sum['pozitivita_diagnosticka'] = (df_sum['pozit_typologie_test_indik_diagnosticka'] / df_sum[ + 'typologie_test_indik_diagnosticka']) * 100 + df_sum['pozitivita_epidemiologicka'] = (df_sum['pozit_typologie_test_indik_epidemiologicka'] / df_sum[ + 'typologie_test_indik_epidemiologicka']) * 100 + df_sum['pozitivita_preventivni'] = (df_sum['pozit_typologie_test_indik_preventivni'] / df_sum[ + 'typologie_test_indik_preventivni']) * 100 - df = pd.merge(df_sum[['pozitivita_diagnosticka', 'pozitivita_epidemiologicka', 'pozitivita_preventivni']], df, left_index=True, right_index=True) + df = pd.merge(df_sum[['pozitivita_diagnosticka', 'pozitivita_epidemiologicka', 'pozitivita_preventivni']], df, + left_index=True, right_index=True) df = df[df.index >= df.index.min() + timedelta(7)] @@ -1457,18 +1470,26 @@ def get_tests_orp_graph_data(): def get_hospital_capacities_graph_data(): - capacities = pd.read_sql_query(f"select * from kapacity_nemocnic where datum < '{get_import_date()}'", db.engine).fillna(0) - capacities['luzka_standard_kyslik_kapacita_volna_covid_pozitivni'] += capacities['inf_luzka_kyslik_kapacita_volna_covid_pozitivni'] - capacities['luzka_standard_kyslik_kapacita_volna_covid_negativni'] += capacities['inf_luzka_kyslik_kapacita_volna_covid_negativni'] + capacities = pd.read_sql_query(f"select * from kapacity_nemocnic where datum < '{get_import_date()}'", + db.engine).fillna(0) + capacities['luzka_standard_kyslik_kapacita_volna_covid_pozitivni'] += capacities[ + 'inf_luzka_kyslik_kapacita_volna_covid_pozitivni'] + capacities['luzka_standard_kyslik_kapacita_volna_covid_negativni'] += capacities[ + 'inf_luzka_kyslik_kapacita_volna_covid_negativni'] capacities['luzka_standard_kyslik_kapacita_celkem'] += capacities['inf_luzka_kyslik_kapacita_celkem'] - capacities['luzka_hfno_cpap_kapacita_volna_covid_pozitivni'] += capacities['inf_luzka_hfno_kapacita_volna_covid_pozitivni'] - capacities['luzka_hfno_cpap_kapacita_volna_covid_negativni'] += capacities['inf_luzka_hfno_kapacita_volna_covid_negativni'] + capacities['luzka_hfno_cpap_kapacita_volna_covid_pozitivni'] += capacities[ + 'inf_luzka_hfno_kapacita_volna_covid_pozitivni'] + capacities['luzka_hfno_cpap_kapacita_volna_covid_negativni'] += capacities[ + 'inf_luzka_hfno_kapacita_volna_covid_negativni'] capacities['luzka_hfno_cpap_kapacita_celkem'] += capacities['inf_luzka_hfno_kapacita_celkem'] - capacities['luzka_upv_niv_kapacita_volna_covid_pozitivni'] += capacities['inf_luzka_upv_kapacita_volna_covid_pozitivni'] - capacities['luzka_upv_niv_kapacita_volna_covid_negativni'] += capacities['inf_luzka_upv_kapacita_volna_covid_negativni'] + capacities['luzka_upv_niv_kapacita_volna_covid_pozitivni'] += capacities[ + 'inf_luzka_upv_kapacita_volna_covid_pozitivni'] + capacities['luzka_upv_niv_kapacita_volna_covid_negativni'] += capacities[ + 'inf_luzka_upv_kapacita_volna_covid_negativni'] capacities['luzka_upv_niv_kapacita_celkem'] += capacities['inf_luzka_upv_kapacita_celkem'] - capacities_21 = pd.read_sql_query("select * from kapacity_nemocnic_21 where datum >= '2021-03-30'", db.engine).fillna(0) + capacities_21 = pd.read_sql_query("select * from kapacity_nemocnic_21 where datum >= '2021-03-30'", + db.engine).fillna(0) capacities_20 = pd.read_sql_query("select * from kapacity_nemocnic_20", db.engine).fillna(0) diff --git a/app/templates/base.html b/app/templates/base.html index da84f4ad..bca7a7c5 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -101,7 +101,7 @@ diff --git a/app/templates/index.html b/app/templates/index.html index 31738812..2e8abbd8 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -115,6 +115,9 @@

COVID-19 data o očkování

Aktuality a důležité informace

    + {{ zprava('Znovuspuštění aktualizace webu', '13. 10. 2023', + 'Aktualizace webu byla 13. 10. 2023 znovu zahájena, zatím v pilotním režimu.') }} + {{ zprava('Ukončení aktualizace webu', '1. 7. 2023', 'Aktualizace webu byla 1. 7. 2023 ukončena.') }} From d4de0fd78cb66f651480a0f95da7289c53bfb2f0 Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Wed, 25 Oct 2023 00:04:17 +0200 Subject: [PATCH 06/11] Fix for new behaviour of groupby pandas --- app/queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/queries.py b/app/queries.py index 5a9baa38..0a5785ab 100644 --- a/app/queries.py +++ b/app/queries.py @@ -494,7 +494,7 @@ def count_registrations(filter_column, filter_value): df['rezervace_14_pocet'] = df[['pocet']].where( (df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(14))) - df = df.groupby(['vekova_skupina', 'povolani']).sum() + df = df.groupby(['vekova_skupina', 'povolani'], group_keys=False, level=0).sum() df['uspesnost_7'] = ((df['registrace_7_rez'] / df['registrace_7']) * 100).replace({np.nan: None}) # df['uspesnost_14'] = ((df['registrace_14_rez'] / df['registrace_14']) * 100).replace({np.nan: None}) From c4886379706c1761fd926eca2e6d4d468be070ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Stan=C4=9Bk?= Date: Thu, 18 Apr 2024 11:05:06 +0200 Subject: [PATCH 07/11] migrations fix --- migrations/versions/130cce777d35_.py | 9 +++++---- migrations/versions/1ed9516441be_.py | 13 +++++++------ migrations/versions/5480e96df417_.py | 18 +++++++++--------- migrations/versions/6d30bda90e50_.py | 3 ++- migrations/versions/b3623a352bec_.py | 7 ++++--- migrations/versions/b8604a1102c8_.py | 7 ++++--- migrations/versions/b8e5763ba835_.py | 17 +++++++++-------- migrations/versions/c703c687dcbf_.py | 15 ++++++++------- migrations/versions/cb94fb60af8b_.py | 4 ++-- migrations/versions/d0ff680fad37_.py | 3 ++- migrations/versions/dd9f6e3ad7c1_.py | 5 +++-- migrations/versions/eb41821d2f4c_.py | 15 ++++++++------- migrations/versions/f238461c71d7_.py | 7 ++++--- migrations/versions/f636a63c48d6_.py | 5 +++-- 14 files changed, 70 insertions(+), 58 deletions(-) diff --git a/migrations/versions/130cce777d35_.py b/migrations/versions/130cce777d35_.py index 487421d0..bddbc18f 100644 --- a/migrations/versions/130cce777d35_.py +++ b/migrations/versions/130cce777d35_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -42,16 +43,16 @@ def upgrade(): op.create_index(op.f('ix_ockovani_lide_kraj_bydl_nuts'), 'ockovani_lide', ['kraj_bydl_nuts'], unique=False) op.drop_column('ockovani_lide', 'kraj_nazev') connection = op.get_bind() - connection.execute("alter table ockovani_lide drop CONSTRAINT ockovani_lide_pkey") - connection.execute("alter table ockovani_lide add CONSTRAINT ockovani_lide_pkey PRIMARY KEY (datum, vakcina, zarizeni_kod, poradi_davky, vekova_skupina, kraj_bydl_nuts, indikace_bezpecnostni_infrastruktura, indikace_chronicke_onemocneni, indikace_ostatni, indikace_pedagog, indikace_skolstvi_ostatni, indikace_socialni_sluzby, indikace_zdravotnik)") + connection.execute(text("alter table ockovani_lide drop CONSTRAINT ockovani_lide_pkey")) + connection.execute(text("alter table ockovani_lide add CONSTRAINT ockovani_lide_pkey PRIMARY KEY (datum, vakcina, zarizeni_kod, poradi_davky, vekova_skupina, kraj_bydl_nuts, indikace_bezpecnostni_infrastruktura, indikace_chronicke_onemocneni, indikace_ostatni, indikace_pedagog, indikace_skolstvi_ostatni, indikace_socialni_sluzby, indikace_zdravotnik)")) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### connection = op.get_bind() - connection.execute("alter table ockovani_lide drop CONSTRAINT ockovani_lide_pkey") - connection.execute("alter table ockovani_lide add CONSTRAINT ockovani_lide_pkey PRIMARY KEY (datum, vakcina, zarizeni_kod, poradi_davky, vekova_skupina)") + connection.execute(text("alter table ockovani_lide drop CONSTRAINT ockovani_lide_pkey")) + connection.execute(text("alter table ockovani_lide add CONSTRAINT ockovani_lide_pkey PRIMARY KEY (datum, vakcina, zarizeni_kod, poradi_davky, vekova_skupina)")) op.add_column('ockovani_lide', sa.Column('kraj_nazev', sa.VARCHAR(), autoincrement=False, nullable=True)) op.drop_index(op.f('ix_ockovani_lide_kraj_bydl_nuts'), table_name='ockovani_lide') op.drop_column('ockovani_lide', 'kraj_bydl_nuts') diff --git a/migrations/versions/1ed9516441be_.py b/migrations/versions/1ed9516441be_.py index 1d7254c0..2b99144d 100644 --- a/migrations/versions/1ed9516441be_.py +++ b/migrations/versions/1ed9516441be_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -18,13 +19,13 @@ def upgrade(): connection = op.get_bind() - connection.execute("alter table ockovani_registrace add column ockovani smallint default -1") - connection.execute("alter table ockovani_registrace drop CONSTRAINT ockovani_registrace_part_pkey") - connection.execute("alter table ockovani_registrace add CONSTRAINT ockovani_registrace_part_pkey PRIMARY KEY (datum, ockovaci_misto_id, vekova_skupina, povolani, stat, rezervace, datum_rezervace, import_id, ockovani)") + connection.execute(text("alter table ockovani_registrace add column ockovani smallint default -1")) + connection.execute(text("alter table ockovani_registrace drop CONSTRAINT ockovani_registrace_part_pkey")) + connection.execute(text("alter table ockovani_registrace add CONSTRAINT ockovani_registrace_part_pkey PRIMARY KEY (datum, ockovaci_misto_id, vekova_skupina, povolani, stat, rezervace, datum_rezervace, import_id, ockovani)")) def downgrade(): connection = op.get_bind() - connection.execute("alter table ockovani_registrace drop column ockovani") - connection.execute("alter table ockovani_registrace drop CONSTRAINT ockovani_registrace_part_pkey") - connection.execute("alter table ockovani_registrace add CONSTRAINT ockovani_registrace_part_pkey PRIMARY KEY (datum, ockovaci_misto_id, vekova_skupina, povolani, stat, rezervace, datum_rezervace, import_id)") + connection.execute(text("alter table ockovani_registrace drop column ockovani")) + connection.execute(text("alter table ockovani_registrace drop CONSTRAINT ockovani_registrace_part_pkey")) + connection.execute(text("alter table ockovani_registrace add CONSTRAINT ockovani_registrace_part_pkey PRIMARY KEY (datum, ockovaci_misto_id, vekova_skupina, povolani, stat, rezervace, datum_rezervace, import_id)")) diff --git a/migrations/versions/5480e96df417_.py b/migrations/versions/5480e96df417_.py index 5f2b70f6..ffdcd8a4 100644 --- a/migrations/versions/5480e96df417_.py +++ b/migrations/versions/5480e96df417_.py @@ -6,7 +6,7 @@ """ from alembic import op -import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -18,15 +18,15 @@ def upgrade(): connection = op.get_bind() - connection.execute("DELETE FROM populace_kategorie WHERE vekova_skupina='0-17'") - connection.execute("INSERT INTO populace_kategorie (vekova_skupina,min_vek,max_vek) VALUES ('0-11',0,11)") - connection.execute("INSERT INTO populace_kategorie (vekova_skupina,min_vek,max_vek) VALUES ('12-15',12,15)") - connection.execute("INSERT INTO populace_kategorie (vekova_skupina,min_vek,max_vek) VALUES ('16-17',16,17)") + connection.execute(text("DELETE FROM populace_kategorie WHERE vekova_skupina='0-17'")) + connection.execute(text("INSERT INTO populace_kategorie (vekova_skupina,min_vek,max_vek) VALUES ('0-11',0,11)")) + connection.execute(text("INSERT INTO populace_kategorie (vekova_skupina,min_vek,max_vek) VALUES ('12-15',12,15)")) + connection.execute(text("INSERT INTO populace_kategorie (vekova_skupina,min_vek,max_vek) VALUES ('16-17',16,17)")) def downgrade(): connection = op.get_bind() - connection.execute("DELETE FROM populace_kategorie WHERE vekova_skupina='0-11'") - connection.execute("DELETE FROM populace_kategorie WHERE vekova_skupina='12-15'") - connection.execute("DELETE FROM populace_kategorie WHERE vekova_skupina='16-17'") - connection.execute("INSERT INTO populace_kategorie (vekova_skupina,min_vek,max_vek) VALUES ('0-17',0,17)") + connection.execute(text("DELETE FROM populace_kategorie WHERE vekova_skupina='0-11'")) + connection.execute(text("DELETE FROM populace_kategorie WHERE vekova_skupina='12-15'")) + connection.execute(text("DELETE FROM populace_kategorie WHERE vekova_skupina='16-17'")) + connection.execute(text("INSERT INTO populace_kategorie (vekova_skupina,min_vek,max_vek) VALUES ('0-17',0,17)")) diff --git a/migrations/versions/6d30bda90e50_.py b/migrations/versions/6d30bda90e50_.py index 4df2638a..128639b8 100644 --- a/migrations/versions/6d30bda90e50_.py +++ b/migrations/versions/6d30bda90e50_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -18,7 +19,7 @@ def upgrade(): connection = op.get_bind() - connection.execute("update ockovani_lide set zarizeni_kod=lpad(zarizeni_kod,11,'0')") + connection.execute(text("update ockovani_lide set zarizeni_kod=lpad(zarizeni_kod,11,'0')")) def downgrade(): diff --git a/migrations/versions/b3623a352bec_.py b/migrations/versions/b3623a352bec_.py index 177dec75..bb74b52b 100644 --- a/migrations/versions/b3623a352bec_.py +++ b/migrations/versions/b3623a352bec_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -18,9 +19,9 @@ def upgrade(): connection = op.get_bind() - connection.execute("INSERT INTO ockovaci_mista (id, nazev, okres_id, status, nrpzs_kod) VALUES('4561f6a6-a019-4fb6-8552-6e15c74a3d04', 'EUC klinika Ostrava', 'CZ0806', false, '25860836000')") - connection.execute("INSERT INTO ockovaci_mista (id, nazev, okres_id, status, nrpzs_kod) VALUES('68fcce7d-b37b-4073-a8a0-c21a62976442', 'VITA, s.r.o.', 'CZ0426', false, '61537713000')") - connection.execute("INSERT INTO ockovaci_mista (id, nazev, okres_id, status, nrpzs_kod) VALUES('9d29801f-7265-47cc-abfc-0167f8f58b2d', 'HNsP Bílina', 'CZ0426', false, '61325422000')") + connection.execute(text("INSERT INTO ockovaci_mista (id, nazev, okres_id, status, nrpzs_kod) VALUES('4561f6a6-a019-4fb6-8552-6e15c74a3d04', 'EUC klinika Ostrava', 'CZ0806', false, '25860836000')")) + connection.execute(text("INSERT INTO ockovaci_mista (id, nazev, okres_id, status, nrpzs_kod) VALUES('68fcce7d-b37b-4073-a8a0-c21a62976442', 'VITA, s.r.o.', 'CZ0426', false, '61537713000')")) + connection.execute(text("INSERT INTO ockovaci_mista (id, nazev, okres_id, status, nrpzs_kod) VALUES('9d29801f-7265-47cc-abfc-0167f8f58b2d', 'HNsP Bílina', 'CZ0426', false, '61325422000')")) def downgrade(): pass diff --git a/migrations/versions/b8604a1102c8_.py b/migrations/versions/b8604a1102c8_.py index d0d74aa1..c820f1a6 100644 --- a/migrations/versions/b8604a1102c8_.py +++ b/migrations/versions/b8604a1102c8_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -18,9 +19,9 @@ def upgrade(): connection = op.get_bind() - connection.execute("update importy set date=start") - connection.execute("delete from importy where status='FAILED'") - connection.execute("delete from importy where id in (select id from importy except select max(id) from importy group by date)") + connection.execute(text("update importy set date=start")) + connection.execute(text("delete from importy where status='FAILED'")) + connection.execute(text("delete from importy where id in (select id from importy except select max(id) from importy group by date)")) def downgrade(): pass diff --git a/migrations/versions/b8e5763ba835_.py b/migrations/versions/b8e5763ba835_.py index 0803a326..707e6126 100644 --- a/migrations/versions/b8e5763ba835_.py +++ b/migrations/versions/b8e5763ba835_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -35,10 +36,10 @@ def upgrade(): # ### end Alembic commands ### connection = op.get_bind() - connection.execute("alter table ockovani_registrace add column pred_zavorou boolean default false") - connection.execute("alter table ockovani_registrace add column za_zavorou boolean default false") - connection.execute("alter table ockovani_registrace drop CONSTRAINT ockovani_registrace_part_pkey") - connection.execute("alter table ockovani_registrace add CONSTRAINT ockovani_registrace_part_pkey PRIMARY KEY (datum, ockovaci_misto_id, vekova_skupina, povolani, stat, rezervace, datum_rezervace, import_id, ockovani, pred_zavorou, za_zavorou)") + connection.execute(text("alter table ockovani_registrace add column pred_zavorou boolean default false")) + connection.execute(text("alter table ockovani_registrace add column za_zavorou boolean default false")) + connection.execute(text("alter table ockovani_registrace drop CONSTRAINT ockovani_registrace_part_pkey")) + connection.execute(text("alter table ockovani_registrace add CONSTRAINT ockovani_registrace_part_pkey PRIMARY KEY (datum, ockovaci_misto_id, vekova_skupina, povolani, stat, rezervace, datum_rezervace, import_id, ockovani, pred_zavorou, za_zavorou)")) def downgrade(): @@ -58,7 +59,7 @@ def downgrade(): # ### end Alembic commands ### connection = op.get_bind() - connection.execute("alter table ockovani_registrace drop column pred_zavorou") - connection.execute("alter table ockovani_registrace drop column za_zavorou") - connection.execute("alter table ockovani_registrace drop CONSTRAINT ockovani_registrace_part_pkey") - connection.execute("alter table ockovani_registrace add CONSTRAINT ockovani_registrace_part_pkey PRIMARY KEY (datum, ockovaci_misto_id, vekova_skupina, povolani, stat, rezervace, datum_rezervace, import_id, ockovani)") + connection.execute(text("alter table ockovani_registrace drop column pred_zavorou")) + connection.execute(text("alter table ockovani_registrace drop column za_zavorou")) + connection.execute(text("alter table ockovani_registrace drop CONSTRAINT ockovani_registrace_part_pkey")) + connection.execute(text("alter table ockovani_registrace add CONSTRAINT ockovani_registrace_part_pkey PRIMARY KEY (datum, ockovaci_misto_id, vekova_skupina, povolani, stat, rezervace, datum_rezervace, import_id, ockovani)")) diff --git a/migrations/versions/c703c687dcbf_.py b/migrations/versions/c703c687dcbf_.py index 583984d6..70400b2c 100644 --- a/migrations/versions/c703c687dcbf_.py +++ b/migrations/versions/c703c687dcbf_.py @@ -8,6 +8,7 @@ import pandas as pd from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -34,13 +35,13 @@ def upgrade(): ) connection = op.get_bind() - connection.execute("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('Pfizer', 'Comirnaty', 2)") - connection.execute("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('Moderna', 'COVID-19 Vaccine Moderna', 2)") - connection.execute("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('AstraZeneca', 'VAXZEVRIA', 2)") - connection.execute("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('Janssen', 'COVID-19 Vaccine Janssen', 1)") - connection.execute("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('Novavax', 'Novavax', 2)") - connection.execute("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('CureVac', 'CureVac', 2)") - connection.execute("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('GSK', 'GSK', 2)") + connection.execute(text("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('Pfizer', 'Comirnaty', 2)")) + connection.execute(text("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('Moderna', 'COVID-19 Vaccine Moderna', 2)")) + connection.execute(text("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('AstraZeneca', 'VAXZEVRIA', 2)")) + connection.execute(text("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('Janssen', 'COVID-19 Vaccine Janssen', 1)")) + connection.execute(text("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('Novavax', 'Novavax', 2)")) + connection.execute(text("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('CureVac', 'CureVac', 2)")) + connection.execute(text("INSERT INTO vakciny (vyrobce, vakcina, davky) VALUES('GSK', 'GSK', 2)")) op.create_foreign_key(None, 'ockovani_distribuce', 'vakciny', ['vyrobce'], ['vyrobce']) op.create_foreign_key(None, 'ockovani_lide', 'vakciny', ['vakcina'], ['vakcina']) diff --git a/migrations/versions/cb94fb60af8b_.py b/migrations/versions/cb94fb60af8b_.py index fc9a8a10..913b319c 100644 --- a/migrations/versions/cb94fb60af8b_.py +++ b/migrations/versions/cb94fb60af8b_.py @@ -6,7 +6,7 @@ """ from alembic import op -import sqlalchemy as sa +from sqlalchemy import text import csv import urllib from app.models import ZdravotnickeStredisko @@ -23,7 +23,7 @@ def upgrade(): # ### end Alembic commands ### connection = op.get_bind() - connection.execute("truncate table zdravotnicke_stredisko") + connection.execute(text("truncate table zdravotnicke_stredisko")) url = "https://nrpzs.uzis.cz/res/file/export/export-2021-10.csv" response = urllib.request.urlopen(url) diff --git a/migrations/versions/d0ff680fad37_.py b/migrations/versions/d0ff680fad37_.py index 8b0a42f4..a5005f5c 100644 --- a/migrations/versions/d0ff680fad37_.py +++ b/migrations/versions/d0ff680fad37_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -18,7 +19,7 @@ def upgrade(): connection = op.get_bind() - connection.execute("update praktici_kapacity set adresa=split_part(adresa, ',', 1)") + connection.execute(text("update praktici_kapacity set adresa=split_part(adresa, ',', 1)")) def downgrade(): diff --git a/migrations/versions/dd9f6e3ad7c1_.py b/migrations/versions/dd9f6e3ad7c1_.py index 42d678e4..5dedde4d 100644 --- a/migrations/versions/dd9f6e3ad7c1_.py +++ b/migrations/versions/dd9f6e3ad7c1_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -24,8 +25,8 @@ def upgrade(): sa.PrimaryKeyConstraint('bezregistrace_id', 'cas_mereni') ) connection = op.get_bind() - connection.execute("truncate table ockovani_lide") - connection.execute("update vakciny set vakcina='Spikevax' where vyrobce='Moderna'") + connection.execute(text("truncate table ockovani_lide")) + connection.execute(text("update vakciny set vakcina='Spikevax' where vyrobce='Moderna'")) def downgrade(): diff --git a/migrations/versions/eb41821d2f4c_.py b/migrations/versions/eb41821d2f4c_.py index af1c2f84..62d48440 100644 --- a/migrations/versions/eb41821d2f4c_.py +++ b/migrations/versions/eb41821d2f4c_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -19,21 +20,21 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### connection = op.get_bind() - connection.execute("truncate table ockovani_rezervace") - connection.execute("alter table ockovani_rezervace drop CONSTRAINT if exists ockovani_rezervace_pkey") - connection.execute("alter table ockovani_rezervace drop CONSTRAINT if exists ockovani_rezervace_import_id_fkey") + connection.execute(text("truncate table ockovani_rezervace")) + connection.execute(text("alter table ockovani_rezervace drop CONSTRAINT if exists ockovani_rezervace_pkey")) + connection.execute(text("alter table ockovani_rezervace drop CONSTRAINT if exists ockovani_rezervace_import_id_fkey")) op.drop_column('ockovani_rezervace', 'import_id') - connection.execute("alter table ockovani_rezervace add CONSTRAINT ockovani_rezervace_pkey PRIMARY KEY (datum, ockovaci_misto_id, kalendar_ockovani)") + connection.execute(text("alter table ockovani_rezervace add CONSTRAINT ockovani_rezervace_pkey PRIMARY KEY (datum, ockovaci_misto_id, kalendar_ockovani)")) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### connection = op.get_bind() - connection.execute("truncate table ockovani_rezervace") - connection.execute("alter table ockovani_rezervace drop CONSTRAINT if exists ockovani_rezervace_pkey") + connection.execute(text("truncate table ockovani_rezervace")) + connection.execute(text("alter table ockovani_rezervace drop CONSTRAINT if exists ockovani_rezervace_pkey")) op.add_column('ockovani_rezervace', sa.Column('import_id', sa.INTEGER(), autoincrement=False, nullable=False)) op.create_foreign_key('ockovani_rezervace_import_id_fkey', 'ockovani_rezervace', 'importy', ['import_id'], ['id'], ondelete='CASCADE') - connection.execute("alter table ockovani_rezervace add CONSTRAINT ockovani_rezervace_pkey PRIMARY KEY (datum, ockovaci_misto_id, kalendar_ockovani, import_id)") + connection.execute(text("alter table ockovani_rezervace add CONSTRAINT ockovani_rezervace_pkey PRIMARY KEY (datum, ockovaci_misto_id, kalendar_ockovani, import_id)")) # ### end Alembic commands ### diff --git a/migrations/versions/f238461c71d7_.py b/migrations/versions/f238461c71d7_.py index 1394fddc..f8b9ab21 100644 --- a/migrations/versions/f238461c71d7_.py +++ b/migrations/versions/f238461c71d7_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -92,9 +93,9 @@ def upgrade(): op.drop_column('okresy_metriky', 'typ_metriky') op.drop_column('okresy_metriky', 'hodnota_int') connection = op.get_bind() - connection.execute("truncate table ockovaci_mista_metriky") - connection.execute("truncate table okresy_metriky") - connection.execute("truncate table kraje_metriky") + connection.execute(text("truncate table ockovaci_mista_metriky")) + connection.execute(text("truncate table okresy_metriky")) + connection.execute(text("truncate table kraje_metriky")) op.add_column('kraje_metriky', sa.Column('kraj_id', sa.Unicode(), nullable=False)) op.drop_constraint('kraje_metriky_id_fkey', 'kraje_metriky', type_='foreignkey') op.create_foreign_key(None, 'kraje_metriky', 'kraje', ['kraj_id'], ['id']) diff --git a/migrations/versions/f636a63c48d6_.py b/migrations/versions/f636a63c48d6_.py index e0396726..464ec6a7 100644 --- a/migrations/versions/f636a63c48d6_.py +++ b/migrations/versions/f636a63c48d6_.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import text # revision identifiers, used by Alembic. @@ -57,8 +58,8 @@ def upgrade(): op.drop_table('srovnani_ockovani') connection = op.get_bind() - connection.execute("truncate table ockovani_lide") - connection.execute("update vakciny set vakcina='SPIKEVAX' where vyrobce='Moderna'") + connection.execute(text("truncate table ockovani_lide")) + connection.execute(text("update vakciny set vakcina='SPIKEVAX' where vyrobce='Moderna'")) # ### end Alembic commands ### From cebf5beecd4664664e571e19cbbd2c7702ef0223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Stan=C4=9Bk?= Date: Thu, 18 Apr 2024 12:39:06 +0200 Subject: [PATCH 08/11] add gunicorn dep --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 376d06af..b03cf0de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ alembic>=1.5.0 urllib3>=1.26.0 openpyxl>=3.0.0 pandas>=2.1.0 +gunicorn From b04a0bdeb37c132427de24d498e5e3fc92187dcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Stan=C4=9Bk?= Date: Thu, 18 Apr 2024 15:16:03 +0200 Subject: [PATCH 09/11] update actions --- .github/workflows/check-disk-space.yml | 2 -- .github/workflows/clean-db.yml | 2 -- .github/workflows/deploy-acceptance.yml | 6 ++---- .github/workflows/deploy-production.yml | 6 ++---- .github/workflows/post-tweet.yml | 2 -- .github/workflows/sync-prd-acc-db.yml | 2 -- .github/workflows/test.yml | 2 -- .github/workflows/update-data-daily.yml | 2 -- .github/workflows/update-data-hourly.yml | 2 -- .github/workflows/update-data-monthly.yml | 2 -- .github/workflows/update-web.yml | 2 -- scripts/cron | 4 ++-- 12 files changed, 6 insertions(+), 28 deletions(-) diff --git a/.github/workflows/check-disk-space.yml b/.github/workflows/check-disk-space.yml index 1fae3290..23563197 100644 --- a/.github/workflows/check-disk-space.yml +++ b/.github/workflows/check-disk-space.yml @@ -16,8 +16,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/clean-db.yml b/.github/workflows/clean-db.yml index b7d2ba41..a460bfae 100644 --- a/.github/workflows/clean-db.yml +++ b/.github/workflows/clean-db.yml @@ -18,8 +18,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/deploy-acceptance.yml b/.github/workflows/deploy-acceptance.yml index f507ffe5..edf72f65 100644 --- a/.github/workflows/deploy-acceptance.yml +++ b/.github/workflows/deploy-acceptance.yml @@ -23,9 +23,7 @@ jobs: SSH_USER: ${{ secrets.SSH_USER }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - uses: actions/checkout@v3 - - id: ip - uses: haythem/public-ip@v1.3 + - uses: actions/checkout@v4 - name: Prepare config file uses: falnyr/replace-env-vars-action@v1.3.2 env: @@ -61,7 +59,7 @@ jobs: - name: Unpack release and clean tmp run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "tar -xvf ${TMP_DIR}/release.tar -C ${RELEASE_DIR} && rm ${TMP_DIR}/*" - name: Create venv and install requirements - run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "cd ${RELEASE_DIR} && ~/Python-3.10.0/python -m venv venv && source venv/bin/activate && pip install -r requirements.txt --no-cache-dir" + run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "cd ${RELEASE_DIR} && python3 -m venv venv && source venv/bin/activate && pip install -r requirements.txt --no-cache-dir" - name: Swap releases run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "rm -f ${APP_DIR} && ln -s ${RELEASE_DIR} ${APP_DIR} && find ${RELEASES_DIR} -maxdepth 1 -mindepth 1 -not -path ${RELEASE_DIR} -exec rm -rf {} \;" - name: Execute migrations diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 2107fd29..1130b2fa 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -21,9 +21,7 @@ jobs: SSH_USER: ${{ secrets.SSH_USER }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - uses: actions/checkout@v3 - - id: ip - uses: haythem/public-ip@v1.3 + - uses: actions/checkout@v4 - name: Prepare config file uses: falnyr/replace-env-vars-action@v1.3.2 env: @@ -59,7 +57,7 @@ jobs: - name: Unpack release and clean tmp run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "tar -xvf ${TMP_DIR}/release.tar -C ${RELEASE_DIR} && rm ${TMP_DIR}/*" - name: Create venv and install requirements - run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "cd ${RELEASE_DIR} && ~/Python-3.10.0/python -m venv venv && source venv/bin/activate && pip install -r requirements.txt --no-cache-dir" + run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "cd ${RELEASE_DIR} && python3 -m venv venv && source venv/bin/activate && pip install -r requirements.txt --no-cache-dir" - name: Swap releases run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "rm -f ${APP_DIR} && ln -s ${RELEASE_DIR} ${APP_DIR} && find ${RELEASES_DIR} -maxdepth 1 -mindepth 1 -not -path ${RELEASE_DIR} -exec rm -rf {} \;" - name: Execute migrations diff --git a/.github/workflows/post-tweet.yml b/.github/workflows/post-tweet.yml index f795d8da..48b4e258 100644 --- a/.github/workflows/post-tweet.yml +++ b/.github/workflows/post-tweet.yml @@ -17,8 +17,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/sync-prd-acc-db.yml b/.github/workflows/sync-prd-acc-db.yml index d1b419b3..c4b40889 100644 --- a/.github/workflows/sync-prd-acc-db.yml +++ b/.github/workflows/sync-prd-acc-db.yml @@ -17,8 +17,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b080841a..34632d4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,8 +21,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/update-data-daily.yml b/.github/workflows/update-data-daily.yml index 72a134e9..a81bc9db 100644 --- a/.github/workflows/update-data-daily.yml +++ b/.github/workflows/update-data-daily.yml @@ -17,8 +17,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/update-data-hourly.yml b/.github/workflows/update-data-hourly.yml index d058a4dd..4312fc50 100644 --- a/.github/workflows/update-data-hourly.yml +++ b/.github/workflows/update-data-hourly.yml @@ -17,8 +17,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/update-data-monthly.yml b/.github/workflows/update-data-monthly.yml index e73ae6da..e3e0a6f4 100644 --- a/.github/workflows/update-data-monthly.yml +++ b/.github/workflows/update-data-monthly.yml @@ -17,8 +17,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/update-web.yml b/.github/workflows/update-web.yml index 45fc087c..d8a049fe 100644 --- a/.github/workflows/update-web.yml +++ b/.github/workflows/update-web.yml @@ -21,8 +21,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/scripts/cron b/scripts/cron index e4a59ec2..b93ba34e 100644 --- a/scripts/cron +++ b/scripts/cron @@ -4,5 +4,5 @@ # 38 7-22 * * * bash /home/ockovani/prd/app/scripts/execute_action.sh 15481698 # monthly fetcher 00 5 5 * * bash /home/ockovani/prd/app/scripts/execute_action.sh 16635318 -# post tweet -35 7 * * * bash /home/ockovani/prd/app/scripts/execute_action.sh 6797956 +# post tweet - disabled +# 35 7 * * * bash /home/ockovani/prd/app/scripts/execute_action.sh 6797956 From f35e4415ab5ea44c7f882d1667f8a4748a384f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Stan=C4=9Bk?= Date: Thu, 18 Apr 2024 15:40:26 +0200 Subject: [PATCH 10/11] alert button fix --- app/templates/base.html | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/templates/base.html b/app/templates/base.html index 6f88f9e2..f84bee6a 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -99,10 +99,8 @@