Events v2: implement deduplication configuration (#8329)

packages/auth/tsconfig.json

@@ -12,7 +12,7 @@
     "sourceMap": true,
     "moduleResolution": "node",
     "rootDir": ".",
-    "lib": ["esnext.asynciterable", "es6", "es2017"],
+    "lib": ["esnext.asynciterable", "es6", "es2019"],
     "forceConsistentCasingInFileNames": true,
     "noImplicitReturns": true,
     "noImplicitThis": true,

packages/client/src/v2-events/features/events/fixtures.ts

@@ -241,5 +241,25 @@ export const tennisClubMembershipEvent = {
       },
       forms: [DEFAULT_FORM]
     }
+  ],
+  deduplication: [
+    {
+      id: 'STANDARD CHECK',
+      label: {
+        defaultMessage: 'Standard check',
+        description:
+          'This could be shown to the user in a reason for duplicate detected',
+        id: '...'
+      },
+      query: {
+        type: 'and',
+        clauses: [
+          {
+            fieldId: 'applicant.firstname',
+            type: 'strict'
+          }
+        ]
+      }
+    }
   ]
 } satisfies EventConfig

packages/client/src/v2-events/features/events/useEventConfiguration.ts

@@ -20,12 +20,6 @@ export function useEventConfigurations() {
   return config
 }
 
-export function getAllFields(configuration: EventConfig) {
-  return configuration.actions
-    .flatMap((action) => action.forms.filter((form) => form.active))
-    .flatMap((form) => form.pages.flatMap((page) => page.fields))
-}
-
 /**
  * Fetches configured events and finds a matching event
  * @param eventIdentifier e.g. 'birth', 'death', 'marriage' or any configured event

packages/client/src/v2-events/features/workqueues/EventOverview/EventOverview.tsx

@@ -14,14 +14,14 @@ import { useTypedParams } from 'react-router-typesafe-routes/dom'
 import {
   ActionDocument,
   EventIndex,
+  getAllFields,
   SummaryConfig
 } from '@opencrvs/commons/client'
 import { Content, ContentSize } from '@opencrvs/components/lib/Content'
 import { IconWithName } from '@client/v2-events/components/IconWithName'
 import { ROUTES } from '@client/v2-events/routes'
 
 import {
-  getAllFields,
   useEventConfiguration,
   useEventConfigurations
 } from '@client/v2-events/features/events/useEventConfiguration'

packages/commons/src/events/ActionInput.ts

@@ -9,8 +9,8 @@
  * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
  */
 
-import { ActionType } from './ActionConfig'
 import { z } from 'zod'
+import { ActionType } from './ActionConfig'
 import { FieldValue } from './FieldValue'
 
 const BaseActionInput = z.object({
@@ -39,7 +39,8 @@ export const RegisterActionInput = BaseActionInput.merge(
 
 export const ValidateActionInput = BaseActionInput.merge(
   z.object({
-    type: z.literal(ActionType.VALIDATE).default(ActionType.VALIDATE)
+    type: z.literal(ActionType.VALIDATE).default(ActionType.VALIDATE),
+    duplicates: z.array(z.string())
   })
 )
 

packages/commons/src/events/DeduplicationConfig.ts

@@ -0,0 +1,115 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * OpenCRVS is also distributed under the terms of the Civil Registration
+ * & Healthcare Disclaimer located at http://opencrvs.org/license.
+ *
+ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
+ */
+import { z } from 'zod'
+import { TranslationConfig } from './TranslationConfig'
+
+const FieldReference = z.string()
+
+const Matcher = z.object({
+  fieldId: z.string(),
+  options: z
+    .object({
+      boost: z.number().optional()
+    })
+    .optional()
+    .default({})
+})
+
+const FuzzyMatcher = Matcher.extend({
+  type: z.literal('fuzzy'),
+  options: z
+    .object({
+      /**
+       * Names of length 3 or less characters = 0 edits allowed
+       * Names of length 4 - 6 characters = 1 edit allowed
+       * Names of length >7 characters = 2 edits allowed
+       */
+      fuzziness: z
+        .union([z.string(), z.number()])
+        .optional()
+        .default('AUTO:4,7'),
+      boost: z.number().optional().default(1)
+    })
+    .optional()
+    .default({})
+})
+
+const StrictMatcher = Matcher.extend({
+  type: z.literal('strict'),
+  options: z
+    .object({
+      boost: z.number().optional().default(1)
+    })
+    .optional()
+    .default({})
+})
+
+const DateRangeMatcher = Matcher.extend({
+  type: z.literal('dateRange'),
+  options: z.object({
+    days: z.number(),
+    origin: FieldReference,
+    boost: z.number().optional().default(1)
+  })
+})
+
+const DateDistanceMatcher = Matcher.extend({
+  type: z.literal('dateDistance'),
+  options: z.object({
+    days: z.number(),
+    origin: FieldReference,
+    boost: z.number().optional().default(1)
+  })
+})
+
+export type And = {
+  type: 'and'
+  clauses: any[]
+}
+
+const And: z.ZodType<And> = z.object({
+  type: z.literal('and'),
+  clauses: z.lazy(() => Clause.array())
+})
+
+export type Or = {
+  type: 'or'
+  clauses: any[]
+}
+
+const Or: z.ZodType<Or> = z.object({
+  type: z.literal('or'),
+  clauses: z.lazy(() => Clause.array())
+})
+
+export type Clause =
+  | And
+  | Or
+  | z.infer<typeof FuzzyMatcher>
+  | z.infer<typeof StrictMatcher>
+  | z.infer<typeof DateRangeMatcher>
+
+export const Clause = z.union([
+  And,
+  Or,
+  FuzzyMatcher,
+  StrictMatcher,
+  DateRangeMatcher,
+  DateDistanceMatcher
+])
+
+export const DeduplicationConfig = z.object({
+  id: z.string(),
+  label: TranslationConfig,
+  query: Clause
+})
+
+export type DeduplicationConfig = z.infer<typeof DeduplicationConfig>

packages/commons/src/events/EventConfig.ts

@@ -14,6 +14,7 @@ import { TranslationConfig } from './TranslationConfig'
 import { SummaryConfig, SummaryConfigInput } from './SummaryConfig'
 import { WorkqueueConfig } from './WorkqueueConfig'
 import { FormConfig, FormConfigInput } from './FormConfig'
+import { DeduplicationConfig } from './DeduplicationConfig'
 
 /**
  * Description of event features defined by the country. Includes configuration for process steps and forms involved.
@@ -29,7 +30,8 @@ export const EventConfig = z.object({
   summary: SummaryConfig,
   label: TranslationConfig,
   actions: z.array(ActionConfig),
-  workqueues: z.array(WorkqueueConfig)
+  workqueues: z.array(WorkqueueConfig),
+  deduplication: z.array(DeduplicationConfig).optional().default([])
 })
 
 export const EventConfigInput = EventConfig.extend({

packages/commons/src/events/index.ts

@@ -25,3 +25,4 @@ export * from './FieldValue'
 export * from './state'
 export * from './utils'
 export * from './defineConfig'
+export * from './DeduplicationConfig'

packages/commons/src/events/utils.ts

@@ -13,7 +13,7 @@ import { TranslationConfig } from './TranslationConfig'
 
 import { EventMetadataKeys, eventMetadataLabelMap } from './EventMetadata'
 import { flattenDeep } from 'lodash'
-import { EventConfigInput } from './EventConfig'
+import { EventConfig, EventConfigInput } from './EventConfig'
 import { SummaryConfigInput } from './SummaryConfig'
 import { WorkqueueConfigInput } from './WorkqueueConfig'
 import { FieldType } from './FieldConfig'
@@ -100,3 +100,9 @@ export const resolveFieldLabels = ({
     })
   }
 }
+
+export function getAllFields(configuration: EventConfig) {
+  return configuration.actions
+    .flatMap((action) => action.forms.filter((form) => form.active))
+    .flatMap((form) => form.pages.flatMap((page) => page.fields))
+}

packages/commons/tsconfig.esm.json

@@ -11,7 +11,7 @@
     "moduleResolution": "node",
     "rootDir": "src",
     "declaration": true,
-    "lib": ["esnext.asynciterable", "es2015", "es6", "es2017"],
+    "lib": ["esnext.asynciterable", "es2015", "es6", "es2019"],
     "forceConsistentCasingInFileNames": true,
     "noImplicitReturns": true,
     "noImplicitThis": true,

packages/commons/tsconfig.json

@@ -10,7 +10,7 @@
     "sourceMap": true,
     "rootDir": "src",
     "declaration": true,
-    "lib": ["esnext.asynciterable", "es2015", "es6", "es2017"],
+    "lib": ["esnext.asynciterable", "es2015", "es6", "es2019"],
     "forceConsistentCasingInFileNames": true,
     "noImplicitReturns": true,
     "noImplicitThis": true,

packages/components/tsconfig.json

@@ -4,7 +4,7 @@
     "outDir": "lib/",
     "module": "ESNext",
     "target": "es6",
-    "lib": ["es6", "dom", "es2017"],
+    "lib": ["es6", "dom", "es2019"],
     "sourceMap": true,
     "allowJs": false,
     "declaration": true,

packages/config/tsconfig.json

@@ -11,7 +11,7 @@
     "sourceMap": true,
     "moduleResolution": "node16",
     "outDir": "build/dist",
-    "lib": ["esnext.asynciterable", "es6", "es2017"],
+    "lib": ["esnext.asynciterable", "es6", "es2019"],
     "forceConsistentCasingInFileNames": true,
     "noImplicitReturns": true,
     "noImplicitThis": true,

packages/documents/tsconfig.json

@@ -10,7 +10,7 @@
     "skipLibCheck": true,
     "moduleResolution": "node",
     "rootDir": ".",
-    "lib": ["esnext.asynciterable", "es6", "es2017"],
+    "lib": ["esnext.asynciterable", "es6", "es2019"],
     "forceConsistentCasingInFileNames": true,
     "noImplicitReturns": true,
     "noImplicitThis": true,

packages/events/package.json

@@ -20,6 +20,7 @@
     "@opencrvs/commons": "^1.7.0",
     "@trpc/server": "^11.0.0-rc.532",
     "app-module-path": "^2.2.0",
+    "date-fns": "^4.1.0",
     "envalid": "^8.0.0",
     "jsonwebtoken": "^9.0.0",
     "mongodb": "6.9.0",
@@ -29,9 +30,9 @@
   },
   "devDependencies": {
     "@testcontainers/elasticsearch": "^10.15.0",
+    "@types/jsonwebtoken": "^9.0.0",
     "@typescript-eslint/eslint-plugin": "^4.5.0",
     "@typescript-eslint/parser": "^4.5.0",
-    "@types/jsonwebtoken": "^9.0.0",
     "cross-env": "^7.0.0",
     "eslint": "^7.11.0",
     "eslint-config-prettier": "^9.0.0",

packages/events/src/router/router.ts

@@ -21,7 +21,8 @@ import {
   deleteEvent,
   EventInputWithId,
   getEventById,
-  patchEvent
+  patchEvent,
+  validate
 } from '@events/service/events'
 import { presignFilesInEvent } from '@events/service/files'
 import {
@@ -30,7 +31,10 @@ import {
   setLocations
 } from '@events/service/locations/locations'
 import { Context, middleware } from './middleware/middleware'
-import { getIndexedEvents } from '@events/service/indexing/indexing'
+import {
+  ensureIndexExists,
+  getIndexedEvents
+} from '@events/service/indexing/indexing'
 import { EventConfig, getUUID } from '@opencrvs/commons'
 import {
   DeclareActionInput,
@@ -78,14 +82,24 @@ export const appRouter = router({
   }),
   event: router({
     create: publicProcedure.input(EventInput).mutation(async (options) => {
-      const config = await getEventConfigurations(options.ctx.token)
-      const eventIds = config.map((c) => c.id)
+      const configurations = await getEventConfigurations(options.ctx.token)
+      const eventIds = configurations.map((c) => c.id)
 
       validateEventType({
         eventTypes: eventIds,
         eventInputType: options.input.type
       })
 
+      const configuration = configurations.find(
+        (c) => c.id === options.input.type
+      )
+
+      if (!configuration) {
+        throw new Error(`Event configuration not found ${options.input.type}`)
+      }
+
+      await ensureIndexExists(options.input.type, configuration)
+
       return createEvent({
         eventInput: options.input,
         createdBy: options.ctx.user.id,
@@ -136,9 +150,9 @@ export const appRouter = router({
         })
       }),
       validate: publicProcedure
-        .input(ValidateActionInput)
+        .input(ValidateActionInput.omit({ duplicates: true }))
         .mutation((options) => {
-          return addAction(options.input, {
+          return validate(options.input, {
             eventId: options.input.eventId,
             createdBy: options.ctx.user.id,
             createdAtLocation: options.ctx.user.primaryOfficeId,