Merge pull request #57 from Opetushallitus/feat/pid-service-replacement

Generate URNs in aoe-web-backend

aoe-infra/bin/infra.ts

@@ -455,7 +455,6 @@ if (environmentName === 'dev' || environmentName === 'qa' || environmentName ===
       Secrets.secrets.SESSION_SECRET,
       Secrets.secrets.CLIENT_SECRET,
       Secrets.secrets.JWT_SECRET,
-      Secrets.secrets.PID_API_KEY,
       Secrets.secrets.PROXY_URI,
       Secrets.secrets.CLIENT_ID
     ],

aoe-infra/environments/dev.json

@@ -74,10 +74,9 @@
             "memory_limit": "1024",
             "min_count": 1,
             "max_count": 1,
-            "image_tag": "ga-267",
+            "image_tag": "ga-281",
             "allow_ecs_exec": true,
             "env_vars": {
-                "PID_SERVICE_URL": "http://localhost",
                 "NODE_ENV": "production",
                 "LOG_LEVEL": "error",
                 "PORT_LISTEN": "8080",
@@ -163,8 +162,8 @@
                 "KAFKA_ENABLED": "1",
                 "LOGIN_ENABLED": "1",
 
-                "PID_SERVICE_RUN_SCHEDULED": "0",
-                "PID_SERVICE_ENABLED": "0",
+                "PID_SERVICE_RUN_SCHEDULED": "1",
+                "PID_SERVICE_ENABLED": "1",
 
                 "STREAM_ENABLED": "1",
                 "STREAM_FILESIZE_MIN": "100000",

aoe-infra/environments/prod.json

@@ -74,10 +74,9 @@
             "memory_limit": "4096",
             "min_count": 1,
             "max_count": 1,
-            "image_tag": "ga-276",
+            "image_tag": "ga-281",
             "allow_ecs_exec": true,
             "env_vars": {
-                "PID_SERVICE_URL": "http://localhost",
                 "NODE_ENV": "production",
                 "LOG_LEVEL": "error",
                 "PORT_LISTEN": "8080",
@@ -163,8 +162,8 @@
                 "KAFKA_ENABLED": "1",
                 "LOGIN_ENABLED": "1",
 
-                "PID_SERVICE_RUN_SCHEDULED": "0",
-                "PID_SERVICE_ENABLED": "0",
+                "PID_SERVICE_RUN_SCHEDULED": "1",
+                "PID_SERVICE_ENABLED": "1",
 
                 "STREAM_ENABLED": "1",
                 "STREAM_FILESIZE_MIN": "100000",

aoe-infra/environments/qa.json

@@ -74,10 +74,9 @@
             "memory_limit": "1024",
             "min_count": 1,
             "max_count": 1,
-            "image_tag": "ga-267",
+            "image_tag": "ga-281",
             "allow_ecs_exec": true,
             "env_vars": {
-                "PID_SERVICE_URL": "http://localhost",
                 "NODE_ENV": "production",
                 "LOG_LEVEL": "error",
                 "PORT_LISTEN": "8080",
@@ -163,8 +162,8 @@
                 "KAFKA_ENABLED": "1",
                 "LOGIN_ENABLED": "1",
 
-                "PID_SERVICE_RUN_SCHEDULED": "0",
-                "PID_SERVICE_ENABLED": "0",
+                "PID_SERVICE_RUN_SCHEDULED": "1",
+                "PID_SERVICE_ENABLED": "1",
 
                 "STREAM_ENABLED": "1",
                 "STREAM_FILESIZE_MIN": "100000",

aoe-infra/lib/secrets-manager-stack.ts

@@ -33,7 +33,6 @@ export class SecretManagerStack extends cdk.Stack {
         SESSION_SECRET: { envVarName: 'SESSION_SECRET', path: '/service/web-backend/SESSION_SECRET', secretKey: 'secretkey' },
         CLIENT_SECRET: { envVarName: 'CLIENT_SECRET', path: '/service/web-backend/CLIENT_SECRET', secretKey: 'secretkey' },
         JWT_SECRET: { envVarName: 'JWT_SECRET', path: '/service/web-backend/JWT_SECRET', secretKey: 'secretkey' },
-        PID_API_KEY: { envVarName: 'PID_API_KEY', path: '/service/web-backend/PID_API_KEY', secretKey: 'secretkey' },
         ANALYTICS_PG_PASS: {envVarName: 'SPRING_DATASOURCE_PRIMARY_PASSWORD', path: '/auroradbs/web-backend/dev/reporter', secretKey: 'password' },
         ANALYTICS_DOCDB_PASSWORD: {envVarName: 'MONGODB_PRIMARY_PASSWORD', path: '/service/data-analytics/DOCDB_PASS', secretKey: 'secretkey' },
         ANALYTICS_TRUST_STORE_PASSWORD: {envVarName: 'TRUST_STORE_PASS', path: '/service/data-analytics/TRUST_STORE_PASS', secretKey: 'secretkey' },

aoe-infra/scripts/bastion_userdata.sh

@@ -6,7 +6,7 @@ chmod 770 /data
 
 echo -e "[mongodb-org-5.0] \nname=MongoDB Repository\nbaseurl=https://repo.mongodb.org/yum/amazon/2/mongodb-org/5.0/x86_64/\ngpgcheck=1 \nenabled=1 \ngpgkey=https://www.mongodb.org/static/pgp/server-5.0.asc" | sudo tee /etc/yum.repos.d/mongodb-org-5.0.repo
 
-rpm --import https://www.mongodb.org/static/pgp/server-6.0.asc
+sudo rpm --import https://www.mongodb.org/static/pgp/server-6.0.asc
 
 cat <<EOF | sudo tee /etc/yum.repos.d/mongodb-org-6.0.repo
 [mongodb-org-6.0]

aoe-web-backend/.env.template

@@ -108,8 +108,6 @@ CONVERSION_TO_PDF_ENABLED=1
 ## PID Service
 PID_SERVICE_RUN_SCHEDULED=0
 PID_SERVICE_ENABLED=0
-PID_API_KEY=
-PID_SERVICE_URL=
 
 ## PosgreSQL Database
 POSTGRESQL_HOST=

aoe-web-backend/src/config/index.ts

@@ -45,8 +45,6 @@ process.env.STREAM_REDIRECT_URI || missingEnvs.push('STREAM_REDIRECT_URI');
 process.env.STREAM_STATUS_HOST || missingEnvs.push('STREAM_STATUS_HOST');
 process.env.STREAM_STATUS_PATH || missingEnvs.push('STREAM_STATUS_PATH');
 process.env.STREAM_STATUS_HOST_HTTPS_ENABLED || missingEnvs.push('STREAM_STATUS_HOST_HTTPS_ENABLED');
-process.env.PID_API_KEY || missingEnvs.push('PID_API_KEY');
-process.env.PID_SERVICE_URL || missingEnvs.push('PID_SERVICE_URL');
 process.env.PG_USER || missingEnvs.push('PG_USER');
 process.env.PG_PASS || missingEnvs.push('PG_PASS');
 
@@ -120,8 +118,6 @@ export default {
   // AOE server and service component general purpose configurations.
   SERVER_CONFIG_OPTIONS: {
     oaipmhAnalyticsURL: process.env.SERVER_CONFIG_OAIPMH_ANALYTICS_URL as string,
-    pidApiKey: process.env.PID_API_KEY as string,
-    pidServiceURL: process.env.PID_SERVICE_URL as string,
   } as const,
 
   // Session management conventions to handle session initialization and persistence.

aoe-web-backend/src/domain/aoeModels.ts

@@ -19,6 +19,32 @@ export const commonSettings: ModelOptions = {
   timestamps: false,
 };
 
+export const Urn = sequelize.define<UrnModel>(
+  'urn',
+  {
+    id: {
+      field: 'id',
+      type: DataTypes.INTEGER,
+      primaryKey: true,
+      autoIncrement: true,
+    },
+    material_url: {
+      field: 'material_url',
+      type: DataTypes.STRING,
+      allowNull: false,
+      unique: true,
+    },
+  },
+  {
+    indexes: [
+      {
+        unique: true,
+        fields: ['material_url'],
+      },
+    ],
+  } && (commonSettings as ModelOptions),
+);
+
 export const AOEUser = <AOEUserType>sequelize.define(
   'aoeuser',
   {

aoe-web-backend/src/services/pidResolutionService.ts

@@ -1,35 +1,35 @@
-import axios, { AxiosRequestConfig } from 'axios';
 import { updateEduMaterialVersionURN } from '@query/apiQueries';
 import { getEdumaterialVersionsWithoutURN } from '@query/pidQueries';
 import { getEduMaterialVersionURL } from './urlService';
 import winstonLogger from '@util/winstonLogger';
-import { IRegisterPID } from '@aoe/services/pidResolutionService';
-import config from '@/config';
+import { Urn } from '@domain/aoeModels';
 
 /**
  * Request for PID registration using URN type.
  * @param url string Resource URL for PID registration.
  */
-export const registerPID = async (url: string): Promise<any> => {
-  try {
-    const pidRegistrationParams: IRegisterPID = {
-      url: url as string,
-      type: 'URN',
-      persist: '0',
-    };
-    const requestHeaders: Record<string, string> = {
-      'Content-Type': 'application/json',
-      apikey: config.SERVER_CONFIG_OPTIONS.pidApiKey,
-    };
-    const response: Record<string, unknown> = await axios.post(
-      config.SERVER_CONFIG_OPTIONS.pidServiceURL,
-      pidRegistrationParams as IRegisterPID,
-      { headers: requestHeaders } as AxiosRequestConfig,
-    );
-    return response.data;
-  } catch (error) {
-    winstonLogger.error('PID registration failed in registerPID(): ' + error);
+export const registerPID = async (url: string): Promise<string> => {
+  const record = await Urn.findOne({
+    where: { material_url: url },
+  });
+
+  if (record) {
+    winstonLogger.error(`URL ${url} already has urn generated`);
+    return null;
   }
+
+  const newId = await Urn.create({ material_url: url });
+  const internalId = newId.id;
+  const now = new Date();
+  const year = now.getFullYear();
+  const month = (now.getMonth() + 1).toString().padStart(2, '0');
+
+  // Previous URN generator generated formattedInternalId with 8 digits. To prevent possible duplicates increase it to 9
+  const formattedInternalId = internalId.toString().padStart(9, '0');
+  const formattedString = `${year}${month}${formattedInternalId}`;
+
+  const luhnChecksum = calculateLuhn(formattedString);
+  return `urn:nbn:fi:oerfi-${year}${month}${formattedInternalId}_${luhnChecksum}`;
 };
 
 /**
@@ -82,6 +82,27 @@ export const processEntriesWithoutPID = async (): Promise<void> => {
   }
 };
 
+const calculateLuhn = (number: string): number => {
+  let sum = 0;
+  let alternate = false;
+
+  for (let i = number.length - 1; i >= 0; i--) {
+    let n = parseInt(number[i], 10);
+
+    if (alternate) {
+      n *= 2;
+      if (n > 9) {
+        n -= 9;
+      }
+    }
+
+    sum += n;
+    alternate = !alternate;
+  }
+
+  return (10 - (sum % 10)) % 10;
+};
+
 export default {
   processEntriesWithoutPID,
   registerPID,

aoe-web-backend/types/aoe/index.d.ts

@@ -1,4 +1,4 @@
-import { BuildOptions, Model } from 'sequelize';
+import { BuildOptions, CreationOptional, InferAttributes, InferCreationAttributes, Model } from 'sequelize';
 
 /**
  * Global interface and type declarations for the data persistence with Sequelize.
@@ -141,4 +141,9 @@ declare global {
   type TemporaryRecordType = typeof Model & {
     new (values?: Record<string, unknown>, options?: BuildOptions): TemporaryRecord;
   };
+
+  interface UrnModel extends Model<InferAttributes<UrnModel>, InferCreationAttributes<UrnModel, { omit: 'id' }>> {
+    id: CreationOptional<number>;
+    material_url: string;
+  }
 }