Skip to content

Commit

Permalink
Merge pull request #28 from Amsterdam/release/3.0.0
Browse files Browse the repository at this point in the history
Release/3.0.0
  • Loading branch information
kramer65 authored May 14, 2020
2 parents fadc272 + e35c94f commit 05facff
Show file tree
Hide file tree
Showing 15 changed files with 83 additions and 81 deletions.
4 changes: 2 additions & 2 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!groovy
def PROJECT_NAME = "stadsarchief"
def PROJECT_NAME = "iiif-metadata-server"
def SLACK_CHANNEL = '#opdrachten-deployments'
def PLAYBOOK = 'deploy-stadsarchief.yml'
def PLAYBOOK = 'deploy-iiif-metadata-server.yml'
def SLACK_MESSAGE = [
"title_link": BUILD_URL,
"fields": [
Expand Down
14 changes: 7 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ env: ## Print current env
env | sort

import_bag: ## Populate database with Bag data
${dc} exec database update-table.sh bag bag_verblijfsobject public stadsarchief
${dc} exec database update-table.sh bag bag_ligplaats public stadsarchief
${dc} exec database update-table.sh bag bag_standplaats public stadsarchief
${dc} exec database update-table.sh bag bag_nummeraanduiding public stadsarchief
${dc} exec database update-table.sh bag bag_pand public stadsarchief
${dc} exec database update-table.sh bag bag_verblijfsobjectpandrelatie public stadsarchief
${dc} exec database update-table.sh bag bag_openbareruimte public stadsarchief
${dc} exec database update-table.sh bag bag_verblijfsobject public iiif_metadata_server
${dc} exec database update-table.sh bag bag_ligplaats public iiif_metadata_server
${dc} exec database update-table.sh bag bag_standplaats public iiif_metadata_server
${dc} exec database update-table.sh bag bag_nummeraanduiding public iiif_metadata_server
${dc} exec database update-table.sh bag bag_pand public iiif_metadata_server
${dc} exec database update-table.sh bag bag_verblijfsobjectpandrelatie public iiif_metadata_server
${dc} exec database update-table.sh bag bag_openbareruimte public iiif_metadata_server
12 changes: 6 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Stadsarchief
# iiif-metadata-server

Provide API and import for datasets from stadsarchief such as bouwdossiers
Provide API and import for datasets from iiif-metadata-server such as bouwdossiers


# Pre_wabo and wabo dossiers
Expand Down Expand Up @@ -70,22 +70,22 @@ Disable login requirement for local development

Test API

`http://localhost:8000/stadsarchief/bouwdossier`
`http://localhost:8000/iiif-metadata/bouwdossier`

or

`http://localhost:8000/stadsarchief/docs/swagger`
`http://localhost:8000/iiif-metadata/docs/swagger`



# OpenAPI

Test API login in acceptance with SWAGGER :

`https://acc.api.data.amsterdam.nl/api/swagger/?url=/stadsarchief/docs/swagger.json`
`https://acc.api.data.amsterdam.nl/api/swagger/?url=/iiif-metadata/docs/swagger.json`


# Import database from acceptance


docker-compose exec database update-db.sh stadsarchief <your username>
docker-compose exec database update-db.sh iiif-metadata-server <your username>
16 changes: 8 additions & 8 deletions deploy/import/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@ services:
database:
image: amsterdam/postgres11
environment:
POSTGRES_DB: stadsarchief
POSTGRES_USER: stadsarchief
POSTGRES_DB: iiif_metadata_server
POSTGRES_USER: iiif_metadata_server
POSTGRES_PASSWORD: insecure
volumes:
- ./backups:/tmp/backups

importer:
image: docker-registry.data.amsterdam.nl/datapunt/stadsarchief:${RELEASE}
image: docker-registry.data.amsterdam.nl/datapunt/iiif-metadata-server:${RELEASE}
# build: ../..
links:
- database
Expand All @@ -19,15 +19,15 @@ services:
BOUWDOSSIERS_OBJECTSTORE_PASSWORD: ${BOUWDOSSIERS_OBJECTSTORE_PASSWORD}
DATABASE_HOST: database
DATABASE_PORT: 5432
DATABASE_NAME: stadsarchief
DATABASE_USER: stadsarchief
DATABASE_NAME: iiif_metadata_server
DATABASE_USER: iiif_metadata_server
DATABASE_PASSWORD: insecure
MIN_BOUWDOSSIERS_COUNT:
command: >
bash -c "/deploy/import/docker-import-db.sh"
empty_db:
image: docker-registry.data.amsterdam.nl/datapunt/stadsarchief:${RELEASE}
image: docker-registry.data.amsterdam.nl/datapunt/iiif-metadata-server:${RELEASE}
# build: ../..
links:
- database
Expand All @@ -36,8 +36,8 @@ services:
BOUWDOSSIERS_OBJECTSTORE_PASSWORD: ${BOUWDOSSIERS_OBJECTSTORE_PASSWORD}
DATABASE_HOST: database
DATABASE_PORT: 5432
DATABASE_NAME: stadsarchief
DATABASE_USER: stadsarchief
DATABASE_NAME: iiif_metadata_server
DATABASE_USER: iiif_metadata_server
DATABASE_PASSWORD: insecure
command: >
bash -c "/deploy/import/docker-empty-db.sh"
4 changes: 2 additions & 2 deletions deploy/import/empty-database.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ set -u
DIR="$(dirname $0)"

dc() {
docker-compose -p stadsarchief -f ${DIR}/docker-compose.yml $*
docker-compose -p iiif-metadata-server -f ${DIR}/docker-compose.yml $*
}

echo "Removing any previous backups"
Expand All @@ -26,6 +26,6 @@ echo "Emptying db"
dc run --rm empty_db

echo "Running backups"
dc exec -T database backup-db.sh stadsarchief
dc exec -T database backup-db.sh iiif_metadata_server
dc down -v
echo "Done"
22 changes: 11 additions & 11 deletions deploy/import/import.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ set -u
DIR="$(dirname $0)"

dc() {
docker-compose -p stadsarchief -f ${DIR}/docker-compose.yml $*
docker-compose -p iiif-metadata-server -f ${DIR}/docker-compose.yml $*
}

echo "Removing any previous backups"
Expand All @@ -25,25 +25,25 @@ dc run importer /deploy/docker-migrate.sh


# load latest bag into database
echo "Load latest verblijfsobjecten, ligplaatsen, standplaatsen, nummeraanduidingen en panden in stadsarchief database"
echo "Load latest verblijfsobjecten, ligplaatsen, standplaatsen, nummeraanduidingen en panden in iiif-metadata-server database"

# dc exec -T database update-db.sh atlas
dc exec -T database update-table.sh bag bag_verblijfsobject public stadsarchief
dc exec -T database update-table.sh bag bag_ligplaats public stadsarchief
dc exec -T database update-table.sh bag bag_standplaats public stadsarchief
dc exec -T database update-table.sh bag bag_nummeraanduiding public stadsarchief
dc exec -T database update-table.sh bag bag_pand public stadsarchief
dc exec -T database update-table.sh bag bag_verblijfsobjectpandrelatie public stadsarchief
dc exec -T database update-table.sh bag bag_openbareruimte public stadsarchief
dc exec -T database update-table.sh bag bag_verblijfsobject public iiif_metadata_server
dc exec -T database update-table.sh bag bag_ligplaats public iiif_metadata_server
dc exec -T database update-table.sh bag bag_standplaats public iiif_metadata_server
dc exec -T database update-table.sh bag bag_nummeraanduiding public iiif_metadata_server
dc exec -T database update-table.sh bag bag_pand public iiif_metadata_server
dc exec -T database update-table.sh bag bag_verblijfsobjectpandrelatie public iiif_metadata_server
dc exec -T database update-table.sh bag bag_openbareruimte public iiif_metadata_server


echo "Importing data"
dc run --rm importer

echo "Drop bag tables used for importing"
docker-compose -p stadsarchief -f ${DIR}/docker-compose.yml exec -T database psql -U stadsarchief -c 'DROP TABLE bag_openbareruimte, bag_verblijfsobjectpandrelatie, bag_pand, bag_nummeraanduiding, bag_standplaats, bag_ligplaats, bag_verblijfsobject' stadsarchief
docker-compose -p iiif-metadata-server -f ${DIR}/docker-compose.yml exec -T database psql -U iiif_metadata_server -c 'DROP TABLE bag_openbareruimte, bag_verblijfsobjectpandrelatie, bag_pand, bag_nummeraanduiding, bag_standplaats, bag_ligplaats, bag_verblijfsobject' iiif_metadata_server

echo "Running backups"
dc exec -T database backup-db.sh stadsarchief
dc exec -T database backup-db.sh iiif_metadata_server
dc down -v
echo "Done"
10 changes: 5 additions & 5 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ services:
ports:
- "5432:5432"
environment:
POSTGRES_DB: stadsarchief
POSTGRES_USER: stadsarchief
POSTGRES_DB: iiif_metadata_server
POSTGRES_USER: iiif_metadata_server
POSTGRES_PASSWORD: insecure
volumes:
- "~/.ssh/datapunt.key:/root/.ssh/datapunt.key"
Expand All @@ -15,7 +15,7 @@ services:
build:
context: .
target: app
image: docker-registry.data.amsterdam.nl/datapunt/stadsarchief:${VERSION:-latest}
image: docker-registry.data.amsterdam.nl/datapunt/iiif-metadata-server:${VERSION:-latest}
ports:
- 8000:8000
environment:
Expand Down Expand Up @@ -58,8 +58,8 @@ services:
environment:
- DATAPUNT_API_URL=${DATAPUNT_API_URL:-https://api.data.amsterdam.nl/}
- SECRET_KEY=insecure
- DATABASE_NAME=stadsarchief
- DATABASE_USER=stadsarchief
- DATABASE_NAME=iiif_metadata_server
- DATABASE_USER=iiif_metadata_server
- DATABASE_PASSWORD=insecure
entrypoint: /deploy/wait-for-it.sh database:5432 --

Expand Down
6 changes: 3 additions & 3 deletions src/bouwdossiers/batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@

log = logging.getLogger(__name__)

# This project now only uses the STADSDEEL codes received from the stadsarchief xml files.
# This project now only uses the STADSDEEL codes received from the metadata xml files.
# The decision was made to avoid confusion especially because the STADSDEEL
# codes are used in the document names.

# The below mapping was used earlier to map the STADSDEEl codes received from the stadsarchief xml files
# The below mapping was used earlier to map the STADSDEEl codes received from the metadata xml files
# To the codes in bag api. It is kept for reference.

# Stadsarchief stadsdeel codes
# stadsdeel codes
# MAP_STADSDEEL_XML_CODE = {
# 'SA': 'Centrum',
# 'SU': 'Oost',
Expand Down
2 changes: 1 addition & 1 deletion src/bouwdossiers/management/commands/run_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def import_dossiers(self, options):
import_pre_wabo_dossiers(max_files_count)

def handle(self, *args, **options):
log.info('Stadsarchief import started')
log.info('Metadata import started')

if not options['skipgetfiles']:
log.info('Get files from objectstore')
Expand Down
4 changes: 2 additions & 2 deletions src/bouwdossiers/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class BouwDossier(models.Model):
id = models.AutoField(primary_key=True)
importfile = models.ForeignKey(ImportFile, related_name='bouwdossiers', on_delete=CASCADE)
dossiernr = models.IntegerField(null=False)
stadsdeel = models.CharField(max_length=3, db_index=True) # stadsarchief stadsdeel
stadsdeel = models.CharField(max_length=3, db_index=True)
titel = models.CharField(max_length=512, null=False, db_index=True)
datering = models.DateField(null=True)
dossier_type = models.CharField(max_length=64, null=True)
Expand Down Expand Up @@ -98,7 +98,7 @@ class Adres(models.Model):
huisnummer_van = models.IntegerField(null=True)
huisnummer_tot = models.IntegerField(null=True)
openbareruimte_id = models.CharField(max_length=16, db_index=True, null=True) # landelijk_id
stadsdeel = models.CharField(max_length=3, db_index=True) # stadsarchief stadsdeel
stadsdeel = models.CharField(max_length=3, db_index=True)
nummeraanduidingen = ArrayField(models.CharField(max_length=16, null=False), blank=True)
nummeraanduidingen_label = ArrayField(models.CharField(max_length=256, null=False), blank=True)
panden = ArrayField(models.CharField(max_length=16, null=False), blank=True)
Expand Down
Loading

0 comments on commit 05facff

Please sign in to comment.