diff --git a/.github/workflows/check-disk-space.yml b/.github/workflows/check-disk-space.yml index 1fae3290..23563197 100644 --- a/.github/workflows/check-disk-space.yml +++ b/.github/workflows/check-disk-space.yml @@ -16,8 +16,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/clean-db.yml b/.github/workflows/clean-db.yml index b7d2ba41..a460bfae 100644 --- a/.github/workflows/clean-db.yml +++ b/.github/workflows/clean-db.yml @@ -18,8 +18,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/deploy-acceptance.yml b/.github/workflows/deploy-acceptance.yml index f507ffe5..edf72f65 100644 --- a/.github/workflows/deploy-acceptance.yml +++ b/.github/workflows/deploy-acceptance.yml @@ -23,9 +23,7 @@ jobs: SSH_USER: ${{ secrets.SSH_USER }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - uses: actions/checkout@v3 - - id: ip - uses: haythem/public-ip@v1.3 + - uses: actions/checkout@v4 - name: Prepare config file uses: falnyr/replace-env-vars-action@v1.3.2 env: @@ -61,7 +59,7 @@ jobs: - name: Unpack release and clean tmp run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "tar -xvf ${TMP_DIR}/release.tar -C ${RELEASE_DIR} && rm ${TMP_DIR}/*" - name: Create venv and install requirements - run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "cd ${RELEASE_DIR} && ~/Python-3.10.0/python -m venv venv && source venv/bin/activate && pip install -r requirements.txt --no-cache-dir" + run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "cd ${RELEASE_DIR} && python3 -m venv venv && source venv/bin/activate && pip install -r requirements.txt --no-cache-dir" - name: Swap releases run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "rm -f ${APP_DIR} && ln -s ${RELEASE_DIR} ${APP_DIR} && find ${RELEASES_DIR} -maxdepth 1 -mindepth 1 -not -path ${RELEASE_DIR} -exec rm -rf {} \;" - name: Execute migrations diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 2107fd29..1130b2fa 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -21,9 +21,7 @@ jobs: SSH_USER: ${{ secrets.SSH_USER }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - uses: actions/checkout@v3 - - id: ip - uses: haythem/public-ip@v1.3 + - uses: actions/checkout@v4 - name: Prepare config file uses: falnyr/replace-env-vars-action@v1.3.2 env: @@ -59,7 +57,7 @@ jobs: - name: Unpack release and clean tmp run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "tar -xvf ${TMP_DIR}/release.tar -C ${RELEASE_DIR} && rm ${TMP_DIR}/*" - name: Create venv and install requirements - run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "cd ${RELEASE_DIR} && ~/Python-3.10.0/python -m venv venv && source venv/bin/activate && pip install -r requirements.txt --no-cache-dir" + run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "cd ${RELEASE_DIR} && python3 -m venv venv && source venv/bin/activate && pip install -r requirements.txt --no-cache-dir" - name: Swap releases run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "rm -f ${APP_DIR} && ln -s ${RELEASE_DIR} ${APP_DIR} && find ${RELEASES_DIR} -maxdepth 1 -mindepth 1 -not -path ${RELEASE_DIR} -exec rm -rf {} \;" - name: Execute migrations diff --git a/.github/workflows/post-tweet.yml b/.github/workflows/post-tweet.yml index f795d8da..48b4e258 100644 --- a/.github/workflows/post-tweet.yml +++ b/.github/workflows/post-tweet.yml @@ -17,8 +17,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/sync-prd-acc-db.yml b/.github/workflows/sync-prd-acc-db.yml index d1b419b3..c4b40889 100644 --- a/.github/workflows/sync-prd-acc-db.yml +++ b/.github/workflows/sync-prd-acc-db.yml @@ -17,8 +17,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b080841a..34632d4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,8 +21,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/update-data-daily.yml b/.github/workflows/update-data-daily.yml index 72a134e9..b410dc21 100644 --- a/.github/workflows/update-data-daily.yml +++ b/.github/workflows/update-data-daily.yml @@ -16,9 +16,10 @@ jobs: SSH_KEY: ${{ secrets.SSH_KEY }} SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_RESOURCE_GROUP: ${{ secrets.AZURE_RESOURCE_GROUP }} + AZURE_VM_NAME: ${{ secrets.AZURE_VM_NAME }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ @@ -26,9 +27,27 @@ jobs: eval `ssh-agent -s` echo "${SSH_KEY}" | tr -d '\r' | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV + - name: Azure login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Azure scale up VM + uses: azure/cli@v2 + with: + inlineScript: | + az vm deallocate --resource-group ${AZURE_RESOURCE_GROUP} --name ${AZURE_VM_NAME} + az vm resize --resource-group ${AZURE_RESOURCE_GROUP} --name ${AZURE_VM_NAME} --size Standard_D4s_v3 + az vm start --resource-group ${AZURE_RESOURCE_GROUP} --name ${AZURE_VM_NAME} - name: Execute fetcher run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} -o ServerAliveInterval=120 "cd ${APP_DIR} && source venv/bin/activate && flask fetch-data all" - name: Compute metrics run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} -o ServerAliveInterval=120 "cd ${APP_DIR} && source venv/bin/activate && flask compute-metrics all" - name: Verify data run: ssh ${SSH_USER_WEB}@${SSH_HOST} -p ${SSH_PORT} "cd ${APP_DIR} && source venv/bin/activate && flask verify-data" + - name: Azure scale down VM + uses: azure/cli@v2 + with: + inlineScript: | + az vm deallocate --resource-group ${AZURE_RESOURCE_GROUP} --name ${AZURE_VM_NAME} + az vm resize --resource-group ${AZURE_RESOURCE_GROUP} --name ${AZURE_VM_NAME} --size Standard_B1s + az vm start --resource-group ${AZURE_RESOURCE_GROUP} --name ${AZURE_VM_NAME} diff --git a/.github/workflows/update-data-hourly.yml b/.github/workflows/update-data-hourly.yml index d058a4dd..4312fc50 100644 --- a/.github/workflows/update-data-hourly.yml +++ b/.github/workflows/update-data-hourly.yml @@ -17,8 +17,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/update-data-monthly.yml b/.github/workflows/update-data-monthly.yml index e73ae6da..e3e0a6f4 100644 --- a/.github/workflows/update-data-monthly.yml +++ b/.github/workflows/update-data-monthly.yml @@ -17,8 +17,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/.github/workflows/update-web.yml b/.github/workflows/update-web.yml index 45fc087c..d8a049fe 100644 --- a/.github/workflows/update-web.yml +++ b/.github/workflows/update-web.yml @@ -21,8 +21,6 @@ jobs: SSH_PORT: ${{ secrets.SSH_PORT }} SSH_USER_WEB: ${{ secrets.SSH_USER_WEB }} steps: - - id: ip - uses: haythem/public-ip@v1.3 - name: Setup SSH key run: | mkdir -p ~/.ssh/ diff --git a/app/etl/center_metrics_etl.py b/app/etl/center_metrics_etl.py index a88dfed4..2c458d7f 100644 --- a/app/etl/center_metrics_etl.py +++ b/app/etl/center_metrics_etl.py @@ -44,9 +44,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for each vaccination center.""" registrations = db.session.query( OckovaciMisto.id, func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).outerjoin(OckovaniRegistrace, and_(OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id, OckovaniRegistrace.import_id == self._import_id)) \ .group_by(OckovaciMisto.id) \ .all() @@ -68,15 +68,15 @@ def _compute_reservations(self): reservations = db.session.query( OckovaciMisto.id, func.coalesce(func.sum(OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), 0).label('rezervace_celkem'), - func.coalesce(func.sum(case([(OckovaniRezervace.datum >= self._date, OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici_1"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici_2"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita)], else_=0)), 0).label("rezervace_cekajici_3"), - func.coalesce(func.sum(case([(OckovaniRezervace.datum == self._date, OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita_1"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita_2"), - func.coalesce(func.sum(case([(and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita)], else_=0)), 0).label("rezervace_kapacita_3"), - func.min(case([(and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1', OckovaniRezervace.volna_kapacita > 0), OckovaniRezervace.datum)], else_=None)).label("rezervace_nejblizsi_volno") + func.coalesce(func.sum(case((OckovaniRezervace.datum >= self._date, OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici_1"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici_2"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita - OckovaniRezervace.volna_kapacita), else_=0)), 0).label("rezervace_cekajici_3"), + func.coalesce(func.sum(case((OckovaniRezervace.datum == self._date, OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V1'), OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita_1"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'V2'), OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita_2"), + func.coalesce(func.sum(case((and_(OckovaniRezervace.datum == self._date, OckovaniRezervace.kalendar_ockovani == 'VN'), OckovaniRezervace.maximalni_kapacita), else_=0)), 0).label("rezervace_kapacita_3"), + func.min(case((and_(OckovaniRezervace.datum >= self._date, OckovaniRezervace.kalendar_ockovani == 'V1', OckovaniRezervace.volna_kapacita > 0), OckovaniRezervace.datum), else_=None)).label("rezervace_nejblizsi_volno") ).outerjoin(OckovaniRezervace, OckovaciMisto.id == OckovaniRezervace.ockovaci_misto_id) \ .group_by(OckovaciMisto.id) \ .all() @@ -102,10 +102,10 @@ def _compute_reservations(self): def _compute_vaccinated(self): """Computes metrics based on vaccinated people dataset for each vaccination center.""" vaccinated = db.session.query(OckovaciMisto.id, func.coalesce(func.sum(OckovaniLide.pocet), 0).label('ockovani_pocet_davek'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4')) \ + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4')) \ .outerjoin(OckovaniLide, and_(OckovaciMisto.nrpzs_kod == OckovaniLide.zarizeni_kod, OckovaniLide.datum < self._date)) \ .join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .filter(or_(and_(OckovaciMisto.status == True, OckovaciMisto.nrpzs_kod.in_(queries.unique_nrpzs_active_subquery())), @@ -130,9 +130,9 @@ def _compute_distributed(self): """Computes metrics based on distributed vaccines dataset for each vaccination center.""" distributed = db.session.query( OckovaciMisto.id, ( - func.coalesce(func.sum(case([(and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Příjem'), OckovaniDistribuce.pocet_davek)], else_=0)), 0) - + func.coalesce(func.sum(case([(and_(OckovaciMisto.id == OckovaniDistribuce.cilove_ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek)], else_=0)), 0) - - func.coalesce(func.sum(case([(and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek)], else_=0)), 0)) + func.coalesce(func.sum(case((and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Příjem'), OckovaniDistribuce.pocet_davek), else_=0)), 0) + + func.coalesce(func.sum(case((and_(OckovaciMisto.id == OckovaniDistribuce.cilove_ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek), else_=0)), 0) + - func.coalesce(func.sum(case((and_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaniDistribuce.akce == 'Výdej'), OckovaniDistribuce.pocet_davek), else_=0)), 0)) .label('vakciny_prijate_pocet') ).outerjoin(OckovaniDistribuce, and_( or_(OckovaciMisto.id == OckovaniDistribuce.ockovaci_misto_id, OckovaciMisto.id == OckovaniDistribuce.cilove_ockovaci_misto_id), @@ -230,8 +230,8 @@ def _compute_derived(self): success_ratio_7 = db.session.query( OckovaciMisto.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(7)) \ @@ -247,8 +247,8 @@ def _compute_derived(self): success_ratio_14 = db.session.query( OckovaciMisto.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(14)) \ @@ -264,8 +264,8 @@ def _compute_derived(self): success_ratio_30 = db.session.query( OckovaciMisto.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(30)) \ diff --git a/app/etl/cr_metrics_etl.py b/app/etl/cr_metrics_etl.py index 06502aab..316099a9 100644 --- a/app/etl/cr_metrics_etl.py +++ b/app/etl/cr_metrics_etl.py @@ -45,7 +45,7 @@ def _compute_population(self): """Computes metrics based on population for cr.""" population = db.session.query( func.sum(Populace.pocet).label('pocet_obyvatel_celkem'), - func.sum(case([(Populace.vek >= 18, Populace.pocet)], else_=0)).label('pocet_obyvatel_dospeli') + func.sum(case((Populace.vek >= 18, Populace.pocet), else_=0)).label('pocet_obyvatel_dospeli') ).filter(Populace.orp_kod == 'CZ0') \ .one() @@ -61,9 +61,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for cr.""" registrations = db.session.query( func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .one() @@ -111,10 +111,10 @@ def _compute_vaccinated(self): """Computes metrics based on vaccinated people dataset for cr.""" vaccinated = db.session.query( func.coalesce(func.sum(OckovaniLide.pocet), 0).label('ockovani_pocet_davek'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4') + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4') ).join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .filter(OckovaniLide.datum < self._date) \ .one() @@ -133,7 +133,7 @@ def _compute_vaccinated(self): def _compute_distributed(self): """Computes metrics based on distributed vaccines dataset for cr.""" distributed = db.session.query( - func.sum(case([(OckovaniDistribuce.akce == 'Příjem', OckovaniDistribuce.pocet_davek)], else_=0)).label('vakciny_prijate_pocet') + func.sum(case((OckovaniDistribuce.akce == 'Příjem', OckovaniDistribuce.pocet_davek), else_=0)).label('vakciny_prijate_pocet') ).filter(OckovaniDistribuce.datum < self._date) \ .one() @@ -189,8 +189,8 @@ def _compute_derived(self): )) success_ratio_7 = db.session.query( - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(7)) \ .one() @@ -201,8 +201,8 @@ def _compute_derived(self): )) success_ratio_14 = db.session.query( - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(14)) \ .one() @@ -213,8 +213,8 @@ def _compute_derived(self): )) success_ratio_30 = db.session.query( - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).filter(OckovaniRegistrace.import_id == self._import_id) \ .filter(OckovaniRegistrace.datum >= self._date - timedelta(30)) \ .one() diff --git a/app/etl/kraj_metrics_etl.py b/app/etl/kraj_metrics_etl.py index 2ac8e26a..2729312e 100644 --- a/app/etl/kraj_metrics_etl.py +++ b/app/etl/kraj_metrics_etl.py @@ -45,7 +45,7 @@ def _compute_population(self): """Computes metrics based on population for each kraj.""" population = db.session.query( Kraj.id, func.sum(Populace.pocet).label('pocet_obyvatel_celkem'), - func.sum(case([(Populace.vek >= 18, Populace.pocet)], else_=0)).label('pocet_obyvatel_dospeli') + func.sum(case((Populace.vek >= 18, Populace.pocet), else_=0)).label('pocet_obyvatel_dospeli') ).join(Populace, Populace.orp_kod == Kraj.id) \ .group_by(Kraj.id) @@ -63,9 +63,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for each kraj.""" registrations = db.session.query( Okres.kraj_id, func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).join(OckovaciMisto, OckovaciMisto.okres_id == Okres.id) \ .outerjoin(OckovaniRegistrace, and_(OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id, OckovaniRegistrace.import_id == self._import_id)) \ .group_by(Okres.kraj_id) \ @@ -125,10 +125,10 @@ def _compute_vaccinated(self): """Computes metrics based on vaccinated people dataset for each kraj.""" vaccinated = db.session.query( Kraj.id, func.coalesce(func.sum(OckovaniLide.pocet), 0).label('ockovani_pocet_davek'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4') + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4') ).outerjoin(OckovaniLide, and_(OckovaniLide.kraj_nuts_kod == Kraj.id, OckovaniLide.datum < self._date)) \ .join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .group_by(Kraj.id) \ @@ -147,10 +147,10 @@ def _compute_vaccinated(self): vaccinated_bydl = db.session.query( Kraj.id, - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 1, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_castecne_bydl'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_plne_bydl'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 3, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_3_bydl'), - func.coalesce(func.sum(case([(OckovaniLide.poradi_davky == 4, OckovaniLide.pocet)], else_=0)), 0).label('ockovani_pocet_4_bydl') + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 1, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_castecne_bydl'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == Vakcina.davky, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_plne_bydl'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 3, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_3_bydl'), + func.coalesce(func.sum(case((OckovaniLide.poradi_davky == 4, OckovaniLide.pocet), else_=0)), 0).label('ockovani_pocet_4_bydl') ).outerjoin(OckovaniLide, and_(OckovaniLide.kraj_bydl_nuts == Kraj.id, OckovaniLide.datum < self._date)) \ .join(Vakcina, Vakcina.vakcina == OckovaniLide.vakcina) \ .group_by(Kraj.id) \ @@ -276,8 +276,8 @@ def _compute_derived(self): success_ratio_7 = db.session.query( Kraj.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).join(Okres, Okres.kraj_id == Kraj.id) \ .join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ @@ -295,8 +295,8 @@ def _compute_derived(self): success_ratio_14 = db.session.query( Kraj.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).join(Okres, Okres.kraj_id == Kraj.id) \ .join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ @@ -314,8 +314,8 @@ def _compute_derived(self): success_ratio_30 = db.session.query( Kraj.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).join(Okres, Okres.kraj_id == Kraj.id) \ .join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ diff --git a/app/etl/okres_metrics_etl.py b/app/etl/okres_metrics_etl.py index 82e69963..e7154438 100644 --- a/app/etl/okres_metrics_etl.py +++ b/app/etl/okres_metrics_etl.py @@ -43,7 +43,7 @@ def _compute_population(self): """Computes metrics based on population for each okres.""" population = db.session.query( Okres.id, func.sum(Populace.pocet).label('pocet_obyvatel_celkem'), - func.sum(case([(Populace.vek >= 18, Populace.pocet)], else_=0)).label('pocet_obyvatel_dospeli') + func.sum(case((Populace.vek >= 18, Populace.pocet), else_=0)).label('pocet_obyvatel_dospeli') ).join(Populace, Populace.orp_kod == Okres.id) \ .group_by(Okres.id) @@ -61,9 +61,9 @@ def _compute_registrations(self): """Computes metrics based on registrations dataset for each okres.""" registrations = db.session.query( OckovaciMisto.okres_id, func.coalesce(func.sum(OckovaniRegistrace.pocet), 0).label('registrace_celkem'), - func.coalesce(func.sum(case([((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_fronta"), - func.coalesce(func.sum(case([((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet)], else_=0)), 0).label("registrace_pred_zavorou"), - func.coalesce(func.sum(case([(OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet)], else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') + func.coalesce(func.sum(case(((OckovaniRegistrace.rezervace == False) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_fronta"), + func.coalesce(func.sum(case(((OckovaniRegistrace.pred_zavorou == True) & (OckovaniRegistrace.ockovani < 1), OckovaniRegistrace.pocet), else_=0)), 0).label("registrace_pred_zavorou"), + func.coalesce(func.sum(case((OckovaniRegistrace.datum_rezervace >= self._date - timedelta(7), OckovaniRegistrace.pocet), else_=0)) / 7.0, 0).label('registrace_rezervace_prumer') ).outerjoin(OckovaniRegistrace, and_(OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id, OckovaniRegistrace.import_id == self._import_id)) \ .filter(OckovaciMisto.okres_id != None) \ .group_by(OckovaciMisto.okres_id) \ @@ -222,8 +222,8 @@ def _compute_derived(self): success_ratio_7 = db.session.query( Okres.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_tydenni_uspesnost') ).join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ @@ -240,8 +240,8 @@ def _compute_derived(self): success_ratio_14 = db.session.query( Okres.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_14denni_uspesnost') ).join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ @@ -258,8 +258,8 @@ def _compute_derived(self): success_ratio_30 = db.session.query( Okres.id, - (1.0 * func.coalesce(func.sum(case([(OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet)], else_=0)), 0) - / case([(func.sum(OckovaniRegistrace.pocet) == 0, None)], else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') + (1.0 * func.coalesce(func.sum(case((OckovaniRegistrace.rezervace == True, OckovaniRegistrace.pocet), else_=0)), 0) + / case((func.sum(OckovaniRegistrace.pocet) == 0, None), else_=func.sum(OckovaniRegistrace.pocet))).label('registrace_30denni_uspesnost') ).join(OckovaciMisto, (OckovaciMisto.okres_id == Okres.id)) \ .join(OckovaniRegistrace, OckovaciMisto.id == OckovaniRegistrace.ockovaci_misto_id) \ .filter(OckovaniRegistrace.import_id == self._import_id) \ diff --git a/app/fetcher/deaths_fetcher.py b/app/fetcher/deaths_fetcher.py index a7bd4ebe..40a968e4 100644 --- a/app/fetcher/deaths_fetcher.py +++ b/app/fetcher/deaths_fetcher.py @@ -21,7 +21,8 @@ def fetch(self, import_id: int) -> None: df = df[['datum', 'vek', 'kraj_nuts_kod']] df['kraj_nuts_kod'] = df['kraj_nuts_kod'].fillna('-') - + df = df.dropna(subset=['vek']) + df['vek'] = df['vek'].astype('int') df = df.groupby(df.columns.tolist(), dropna=False).size().reset_index(name='pocet') self._truncate() diff --git a/app/fetcher/fetcher.py b/app/fetcher/fetcher.py index ec2e841b..8ce7e7a2 100644 --- a/app/fetcher/fetcher.py +++ b/app/fetcher/fetcher.py @@ -3,6 +3,7 @@ from email import utils as eut from io import StringIO from typing import Optional +from sqlalchemy import text import requests @@ -30,7 +31,9 @@ def fetch(self, import_id: int) -> None: pass def _truncate(self) -> None: - db.engine.execute('truncate table {}'.format(self._table)) + with db.engine.connect() as conn: + result = conn.execute(text('TRUNCATE table {}'.format(self._table))) + conn.commit() def _log_download_finished(self): app.logger.info(f"Fetcher '{type(self).__name__}' source file downloaded.") diff --git a/app/fetcher/fetcher_launcher.py b/app/fetcher/fetcher_launcher.py index 8d0db2fb..3e74fe30 100644 --- a/app/fetcher/fetcher_launcher.py +++ b/app/fetcher/fetcher_launcher.py @@ -85,9 +85,9 @@ def _init_fetchers(self, dataset: str) -> None: # inits requested fetchers if dataset == 'all': # self._fetchers.append(CentersFetcher()) # replaced by CentersApiFetcher - self._fetchers.append(CentersApiFetcher()) - self._fetchers.append(OpeningHoursFetcher()) - self._fetchers.append(HealthFacilitiesFetcher()) + # self._fetchers.append(CentersApiFetcher()) + # self._fetchers.append(OpeningHoursFetcher()) + # self._fetchers.append(HealthFacilitiesFetcher()) # self._fetchers.append(DistributedFetcher()) # not updated since 2022-11-29 # self._fetchers.append(UsedFetcher()) # not updated since 2022-02-16 self._fetchers.append(RegistrationsFetcher()) @@ -264,5 +264,6 @@ def _set_import_finished(self) -> None: if __name__ == '__main__': - fetcher = FetcherLauncher() - fetcher.fetch(sys.argv[1]) + with app.app_context(): + fetcher = FetcherLauncher() + fetcher.fetch(sys.argv[1]) diff --git a/app/fetcher/reservations_fetcher.py b/app/fetcher/reservations_fetcher.py index 84162ed2..43edc7a0 100644 --- a/app/fetcher/reservations_fetcher.py +++ b/app/fetcher/reservations_fetcher.py @@ -18,7 +18,7 @@ def __init__(self): def fetch(self, import_id: int) -> None: df = pd.read_csv(self._url) - df = df.drop(['ockovaci_misto_nazev', 'kraj_nuts_kod', 'kraj_nazev'], axis=1) + df = df.drop(['ockovaci_misto_nazev', 'kraj_nuts_kod', 'kraj_nazev', 'id'], axis=1) df = df.groupby(['datum', 'ockovaci_misto_id', 'kalendar_ockovani'], dropna=False).sum().reset_index() diff --git a/app/fetcher/tests_fetcher.py b/app/fetcher/tests_fetcher.py index 98e845ec..75d17c17 100644 --- a/app/fetcher/tests_fetcher.py +++ b/app/fetcher/tests_fetcher.py @@ -17,7 +17,23 @@ def __init__(self): def fetch(self, import_id: int) -> None: df = pd.read_csv(self._url) - + df['incidence_pozitivni'] = df['incidence_pozitivni'].fillna(-1).astype(int) + df['pozit_typologie_test_indik_diagnosticka'] = df['pozit_typologie_test_indik_diagnosticka'].fillna(-1).astype(int) + df['pozit_typologie_test_indik_epidemiologicka'] = df['pozit_typologie_test_indik_epidemiologicka'].fillna(-1).astype(int) + df['pozit_typologie_test_indik_preventivni'] = df['pozit_typologie_test_indik_preventivni'].fillna( + -1).astype(int) + df['pozit_typologie_test_indik_ostatni'] = df['pozit_typologie_test_indik_ostatni'].fillna(-1).astype( + int) + df['PCR_pozit_sympt'] = df['PCR_pozit_sympt'].fillna(-1).astype( + int) + df['AG_pozit_symp'] = df['AG_pozit_symp'].fillna(-1).astype( + int) + df['AG_pozit_asymp_PCR_conf'] = df['AG_pozit_asymp_PCR_conf'].fillna(-1).astype( + int) + df['PCR_pozit_asymp'] = df['PCR_pozit_asymp'].fillna(-1).astype( + int) + df['PCR_pozit_sympt'] = df['PCR_pozit_sympt'].fillna(-1).astype( + int) df = df.drop(columns=['id']) df = df.rename(columns=str.lower) diff --git a/app/queries.py b/app/queries.py index 689f31d8..0a5785ab 100644 --- a/app/queries.py +++ b/app/queries.py @@ -192,7 +192,8 @@ def find_free_vaccines_available(nrpzs_kod=None, okres_id=None, kraj_id=None): PrakticiKapacity.kontakt_email, PrakticiKapacity.expirace, PrakticiKapacity.poznamka, PrakticiKapacity.kraj, ZdravotnickeStredisko.nrpzs_kod, ZdravotnickeStredisko.latitude, ZdravotnickeStredisko.longitude) \ - .outerjoin(ZdravotnickeStredisko, ZdravotnickeStredisko.zdravotnicke_zarizeni_kod == PrakticiKapacity.zdravotnicke_zarizeni_kod) \ + .outerjoin(ZdravotnickeStredisko, + ZdravotnickeStredisko.zdravotnicke_zarizeni_kod == PrakticiKapacity.zdravotnicke_zarizeni_kod) \ .filter(or_(func.left(PrakticiKapacity.zdravotnicke_zarizeni_kod, 11) == nrpzs_kod, nrpzs_kod is None)) \ .filter(or_(ZdravotnickeStredisko.okres_kod == okres_id, okres_id is None)) \ .filter(or_(ZdravotnickeStredisko.kraj_kod == kraj_id, kraj_id is None)) \ @@ -477,19 +478,23 @@ def count_registrations(filter_column, filter_value): df['fronta_pocet'] = df[['pocet']].where(df['rezervace'] == False).fillna(0).astype('int') df['fronta_cekani'] = (df['dnes'] - df['datum']).astype('timedelta64[ns]').dt.days df['fronta_pocet_x_cekani'] = df['fronta_pocet'] * df['fronta_cekani'] - df['s_terminem_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['ockovani'] < 1)).fillna(0).astype('int') + df['s_terminem_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['ockovani'] < 1)).fillna(0).astype( + 'int') df['registrace_7'] = df[['pocet']].where(df['datum'] >= get_import_date() - timedelta(7)) - df['registrace_7_rez'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(7))) + df['registrace_7_rez'] = df[['pocet']].where( + (df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(7))) # df['registrace_14'] = df[['pocet']].where(df['datum'] >= get_import_date() - timedelta(14)) # df['registrace_14_rez'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(14))) # df['registrace_30'] = df[['pocet']].where(df['datum'] >= get_import_date() - timedelta(30)) # df['registrace_30_rez'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum'] >= get_import_date() - timedelta(30))) df['rezervace_7_cekani'] = (df['datum_rezervace_fix'] - df['datum']).astype('timedelta64[ns]').dt.days - df['rezervace_7_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(7))) + df['rezervace_7_pocet'] = df[['pocet']].where( + (df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(7))) df['rezervace_7_pocet_x_cekani'] = df['rezervace_7_cekani'] * df['rezervace_7_pocet'] - df['rezervace_14_pocet'] = df[['pocet']].where((df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(14))) + df['rezervace_14_pocet'] = df[['pocet']].where( + (df['rezervace'] == True) & (df['datum_rezervace_fix'] >= get_import_date() - timedelta(14))) - df = df.groupby(['vekova_skupina', 'povolani']).sum() + df = df.groupby(['vekova_skupina', 'povolani'], group_keys=False, level=0).sum() df['uspesnost_7'] = ((df['registrace_7_rez'] / df['registrace_7']) * 100).replace({np.nan: None}) # df['uspesnost_14'] = ((df['registrace_14_rez'] / df['registrace_14']) * 100).replace({np.nan: None}) @@ -501,9 +506,12 @@ def count_registrations(filter_column, filter_value): # df['registrace_30'] = df['registrace_30'].astype('int') # df['registrace_30_rez'] = df['registrace_30_rez'].astype('int') df['fronta_prumer_cekani'] = ((df['fronta_pocet_x_cekani'] / df['fronta_pocet']) / 7).replace({np.nan: None}) - df['rezervace_prumer_cekani'] = ((df['rezervace_7_pocet_x_cekani'] / df['rezervace_7_pocet']) / 7).replace({np.nan: None}) + df['rezervace_prumer_cekani'] = ((df['rezervace_7_pocet_x_cekani'] / df['rezervace_7_pocet']) / 7).replace( + {np.nan: None}) - df = df[(df['fronta_pocet'] > 0) | df['fronta_prumer_cekani'].notnull() | df['rezervace_prumer_cekani'].notnull() | df['uspesnost_7'].notnull()] + df = df[ + (df['fronta_pocet'] > 0) | df['fronta_prumer_cekani'].notnull() | df['rezervace_prumer_cekani'].notnull() | df[ + 'uspesnost_7'].notnull()] return df.reset_index().sort_values(by=['vekova_skupina', 'povolani']) @@ -822,7 +830,8 @@ def count_free_slots(center_id=None): def count_vaccinated_week(): - return db.session.query(column('datum'), column('pocet_1'), column('pocet_2'), column('pocet_3'), column('pocet_4'), column('pocet_celkem')).from_statement(text( + return db.session.query(column('datum'), column('pocet_1'), column('pocet_2'), column('pocet_3'), column('pocet_4'), + column('pocet_celkem')).from_statement(text( f""" select datum, sum(case when poradi_davky = 1 then pocet else 0 end) pocet_1, @@ -1009,7 +1018,8 @@ def count_vaccinated_unvaccinated_comparison_age(): .replace({np.nan: 0}) for g in ['plne', 'posilujici']: - df_norm[d + '_' + g + '_ratio'] = (df_norm[d + '_bez_norm'] / df_norm[d + '_' + g + '_norm']).replace({np.inf: np.nan}) + df_norm[d + '_' + g + '_ratio'] = (df_norm[d + '_bez_norm'] / df_norm[d + '_' + g + '_norm']).replace( + {np.inf: np.nan}) df_norm.loc[df_norm['populace_' + g + '_zastoupeni'] < 0.05, d + '_' + g + '_ratio'] = np.nan df_norm[d + '_' + g + '_ratio'] = df_norm[d + '_' + g + '_ratio'].replace({np.nan: None}) @@ -1297,9 +1307,8 @@ def get_infected_graph_data(): {'0-11': '0-17', '12-15': '0-17', '16-17': '0-17', '18-24': '18-29', '25-29': '18-29', '30-34': '30-39', '35-39': '30-39', '40-44': '40-49', '45-49': '40-49', '50-54': '50-59', '55-59': '50-59', '60-64': '60-69', '65-69': '60-69', '70-74': '70-79', '75-79': '70-79'}) - df = df.groupby(['vekova_skupina_grp', 'datum']).sum() - df_sum = df.rolling(7).sum() + df_sum = df[['pocet_vek', 'pocet_nakazeni']].rolling(7).sum() df_sum['pocet_nakazeni_norm'] = ((df_sum['pocet_nakazeni'] / df_sum['pocet_vek']) * 100000) @@ -1350,7 +1359,7 @@ def get_deaths_graph_data(): '65-69': '60-69', '70-74': '70-79', '75-79': '70-79'}) df = df.groupby(['vekova_skupina_grp', 'datum']).sum() - df_sum = df.rolling(7).sum() + df_sum = df[['pocet_vek', 'pocet_umrti']].rolling(7).sum() df_sum['pocet_umrti_norm'] = ((df_sum['pocet_umrti'] / df_sum['pocet_vek']) * 100000) @@ -1375,11 +1384,15 @@ def get_tests_graph_data(): df = df.set_index('datum') df_sum = df.rolling(7).sum() - df_sum['pozitivita_diagnosticka'] = (df_sum['pozit_typologie_test_indik_diagnosticka'] / df_sum['typologie_test_indik_diagnosticka']) * 100 - df_sum['pozitivita_epidemiologicka'] = (df_sum['pozit_typologie_test_indik_epidemiologicka'] / df_sum['typologie_test_indik_epidemiologicka']) * 100 - df_sum['pozitivita_preventivni'] = (df_sum['pozit_typologie_test_indik_preventivni'] / df_sum['typologie_test_indik_preventivni']) * 100 + df_sum['pozitivita_diagnosticka'] = (df_sum['pozit_typologie_test_indik_diagnosticka'] / df_sum[ + 'typologie_test_indik_diagnosticka']) * 100 + df_sum['pozitivita_epidemiologicka'] = (df_sum['pozit_typologie_test_indik_epidemiologicka'] / df_sum[ + 'typologie_test_indik_epidemiologicka']) * 100 + df_sum['pozitivita_preventivni'] = (df_sum['pozit_typologie_test_indik_preventivni'] / df_sum[ + 'typologie_test_indik_preventivni']) * 100 - df = pd.merge(df_sum[['pozitivita_diagnosticka', 'pozitivita_epidemiologicka', 'pozitivita_preventivni']], df, left_index=True, right_index=True) + df = pd.merge(df_sum[['pozitivita_diagnosticka', 'pozitivita_epidemiologicka', 'pozitivita_preventivni']], df, + left_index=True, right_index=True) df = df[df.index >= df.index.min() + timedelta(7)] @@ -1457,18 +1470,26 @@ def get_tests_orp_graph_data(): def get_hospital_capacities_graph_data(): - capacities = pd.read_sql_query(f"select * from kapacity_nemocnic where datum < '{get_import_date()}'", db.engine).fillna(0) - capacities['luzka_standard_kyslik_kapacita_volna_covid_pozitivni'] += capacities['inf_luzka_kyslik_kapacita_volna_covid_pozitivni'] - capacities['luzka_standard_kyslik_kapacita_volna_covid_negativni'] += capacities['inf_luzka_kyslik_kapacita_volna_covid_negativni'] + capacities = pd.read_sql_query(f"select * from kapacity_nemocnic where datum < '{get_import_date()}'", + db.engine).fillna(0) + capacities['luzka_standard_kyslik_kapacita_volna_covid_pozitivni'] += capacities[ + 'inf_luzka_kyslik_kapacita_volna_covid_pozitivni'] + capacities['luzka_standard_kyslik_kapacita_volna_covid_negativni'] += capacities[ + 'inf_luzka_kyslik_kapacita_volna_covid_negativni'] capacities['luzka_standard_kyslik_kapacita_celkem'] += capacities['inf_luzka_kyslik_kapacita_celkem'] - capacities['luzka_hfno_cpap_kapacita_volna_covid_pozitivni'] += capacities['inf_luzka_hfno_kapacita_volna_covid_pozitivni'] - capacities['luzka_hfno_cpap_kapacita_volna_covid_negativni'] += capacities['inf_luzka_hfno_kapacita_volna_covid_negativni'] + capacities['luzka_hfno_cpap_kapacita_volna_covid_pozitivni'] += capacities[ + 'inf_luzka_hfno_kapacita_volna_covid_pozitivni'] + capacities['luzka_hfno_cpap_kapacita_volna_covid_negativni'] += capacities[ + 'inf_luzka_hfno_kapacita_volna_covid_negativni'] capacities['luzka_hfno_cpap_kapacita_celkem'] += capacities['inf_luzka_hfno_kapacita_celkem'] - capacities['luzka_upv_niv_kapacita_volna_covid_pozitivni'] += capacities['inf_luzka_upv_kapacita_volna_covid_pozitivni'] - capacities['luzka_upv_niv_kapacita_volna_covid_negativni'] += capacities['inf_luzka_upv_kapacita_volna_covid_negativni'] + capacities['luzka_upv_niv_kapacita_volna_covid_pozitivni'] += capacities[ + 'inf_luzka_upv_kapacita_volna_covid_pozitivni'] + capacities['luzka_upv_niv_kapacita_volna_covid_negativni'] += capacities[ + 'inf_luzka_upv_kapacita_volna_covid_negativni'] capacities['luzka_upv_niv_kapacita_celkem'] += capacities['inf_luzka_upv_kapacita_celkem'] - capacities_21 = pd.read_sql_query("select * from kapacity_nemocnic_21 where datum >= '2021-03-30'", db.engine).fillna(0) + capacities_21 = pd.read_sql_query("select * from kapacity_nemocnic_21 where datum >= '2021-03-30'", + db.engine).fillna(0) capacities_20 = pd.read_sql_query("select * from kapacity_nemocnic_20", db.engine).fillna(0) diff --git a/app/static/img/sec4good_mungo_logo_tmave.png b/app/static/img/sec4good_mungo_logo_tmave.png new file mode 100644 index 00000000..3ddad821 Binary files /dev/null and b/app/static/img/sec4good_mungo_logo_tmave.png differ diff --git a/app/templates/base.html b/app/templates/base.html index da84f4ad..f84bee6a 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -99,10 +99,8 @@
diff --git a/app/templates/index.html b/app/templates/index.html index 31738812..2e8abbd8 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -115,6 +115,9 @@

COVID-19 data o očkování

Aktuality a důležité informace