From 993e7036b2c642b7151460553403c2cdd69ade49 Mon Sep 17 00:00:00 2001 From: etj Date: Thu, 17 Mar 2022 13:00:08 +0100 Subject: [PATCH] #278 Create DB migrations --- README.md | 8 +- .../dcatapit/migration/dcatapit_pkg/README | 1 + .../migration/dcatapit_pkg/alembic.ini | 79 ++++++++++++++++++ .../dcatapit/migration/dcatapit_pkg/env.py | 81 +++++++++++++++++++ .../migration/dcatapit_pkg/script.py.mako | 24 ++++++ .../d54a365195ea_remove_indexes_in_vocab.py | 31 +++++++ 6 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 ckanext/dcatapit/migration/dcatapit_pkg/README create mode 100644 ckanext/dcatapit/migration/dcatapit_pkg/alembic.ini create mode 100644 ckanext/dcatapit/migration/dcatapit_pkg/env.py create mode 100644 ckanext/dcatapit/migration/dcatapit_pkg/script.py.mako create mode 100644 ckanext/dcatapit/migration/dcatapit_pkg/versions/d54a365195ea_remove_indexes_in_vocab.py diff --git a/README.md b/README.md index f22d388..52fbace 100644 --- a/README.md +++ b/README.md @@ -429,7 +429,11 @@ In order to update the existing translations proceed as follow: 2. Run the migration script: - ckan -c CONFIG_FILE dcatapit migrate-200 + ckan -c $CONFIG_FILE dcatapit migrate-200 + +3. Update the DB: + + ckan -c $CONFIG_FILE db upgrade -p dcatapit_pkg 4. Reindex the datasets @@ -443,6 +447,8 @@ subthemes, and this content would conflict with the dcat one. The migration will move the content from the `theme` extra field to the `themes_aggregate` field, while the logic will provide on-the-fly valid content for the `theme` field so that `ckanext-dcat` will not complain. +The `db upgrade` will remove some harmful constraints in the vocabulary model. + ### Migration from 1.0.0 to 1.1.0 In order to update an old installation (from 1.0.0 to 1.1.0 version): diff --git a/ckanext/dcatapit/migration/dcatapit_pkg/README b/ckanext/dcatapit/migration/dcatapit_pkg/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/ckanext/dcatapit/migration/dcatapit_pkg/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/ckanext/dcatapit/migration/dcatapit_pkg/alembic.ini b/ckanext/dcatapit/migration/dcatapit_pkg/alembic.ini new file mode 100644 index 0000000..1b73dce --- /dev/null +++ b/ckanext/dcatapit/migration/dcatapit_pkg/alembic.ini @@ -0,0 +1,79 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to /mnt/ssd2/homeext/geosol/prj/ckan/29_dcat/ckanext-dcatapit/ckanext/dcatapit/migration/dcatapit_pkg/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat /mnt/ssd2/homeext/geosol/prj/ckan/29_dcat/ckanext-dcatapit/ckanext/dcatapit/migration/dcatapit_pkg/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,dcatapit + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_dcatapit] +level = INFO +handlers = console +qualname = ckanext.dcatapit + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/ckanext/dcatapit/migration/dcatapit_pkg/env.py b/ckanext/dcatapit/migration/dcatapit_pkg/env.py new file mode 100644 index 0000000..0093682 --- /dev/null +++ b/ckanext/dcatapit/migration/dcatapit_pkg/env.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +import os + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +name = os.path.basename(os.path.dirname(__file__)) + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + + url = config.get_main_option(u"sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True, + version_table=u'{}_alembic_version'.format(name) + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix=u'sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table=u'{}_alembic_version'.format(name) + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/ckanext/dcatapit/migration/dcatapit_pkg/script.py.mako b/ckanext/dcatapit/migration/dcatapit_pkg/script.py.mako new file mode 100644 index 0000000..5c26170 --- /dev/null +++ b/ckanext/dcatapit/migration/dcatapit_pkg/script.py.mako @@ -0,0 +1,24 @@ + """${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/ckanext/dcatapit/migration/dcatapit_pkg/versions/d54a365195ea_remove_indexes_in_vocab.py b/ckanext/dcatapit/migration/dcatapit_pkg/versions/d54a365195ea_remove_indexes_in_vocab.py new file mode 100644 index 0000000..ab6c027 --- /dev/null +++ b/ckanext/dcatapit/migration/dcatapit_pkg/versions/d54a365195ea_remove_indexes_in_vocab.py @@ -0,0 +1,31 @@ +"""Remove indexes in vocab + +Revision ID: d54a365195ea +Revises: cfd1d82a53fe +Create Date: 2022-03-17 12:06:10.611576 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'd54a365195ea' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # Using op.execute in order not to raise error in case the index/constraint is not there + + # op.drop_index('ix_dcatapit_vocabulary_text', 'dcatapit_vocabulary') + op.execute('DROP INDEX IF EXISTS ix_dcatapit_vocabulary_text') + + # op.drop_constraint('dcatapit_subtheme_path_key', 'dcatapit_subtheme') + op.execute('ALTER TABLE dcatapit_subtheme DROP CONSTRAINT IF EXISTS dcatapit_subtheme_path_key') + + +def downgrade(): + op.create_index('ix_dcatapit_vocabulary_text', 'dcatapit_vocabulary', ['text']) + op.create_unique_constraint('dcatapit_subtheme_path_key', 'dcatapit_subtheme', ['path']) +