Support isolated folder for monorepo

examples/api/example-skipIsolated.ts

@@ -0,0 +1,14 @@
+import {detectClones} from "jscpd";
+
+(async () => {
+  const clones = await detectClones({
+    path: [
+      __dirname + '/../fixtures'
+    ],
+    skipIsolated: [
+      ['packages/businessA', 'packages/businessB', 'packages/businessC'],
+    ],
+    silent: true
+  });
+  console.log(clones);
+})()

fixtures/lib1/file2.c

@@ -0,0 +1,29 @@
+/*
+ * Copy the size of snapshot frame "sn" to frame "fr".  Do the same for all
+ * following frames and children.
+ * Returns a pointer to the old current window, or NULL.
+ */
+static win_T *restore_snapshot_rec(frame_T *sn, frame_T *fr)
+{
+  win_T       *wp = NULL;
+  win_T       *wp2;
+
+  fr->fr_height = sn->fr_height;
+  fr->fr_width = sn->fr_width;
+  if (fr->fr_layout == FR_LEAF) {
+    frame_new_height(fr, fr->fr_height, FALSE, FALSE);
+    frame_new_width(fr, fr->fr_width, FALSE, FALSE);
+    wp = sn->fr_win;
+  }
+  win_T       *wp = NULL;
+  win_T       *wp2;
+
+  fr->fr_height = sn->fr_height;
+  fr->fr_width = sn->fr_width;
+  if (fr->fr_layout == FR_LEAF) {
+    frame_new_height(fr, fr->fr_height, FALSE, FALSE);
+    frame_new_width(fr, fr->fr_width, FALSE, FALSE);
+    wp = sn->fr_win;
+  }
+  return wp;
+}

fixtures/lib1/file_1.js

@@ -0,0 +1,55 @@
+/**
+*12312
+*/
+function utf8_encode ( str_data ) {
+    // Encodes an ISO-8859-1 string to UTF-8
+    //
+    // +   original by: Webtoolkit.info (http://www.webtoolkit.info/)
+    str_data = str_data.replace(/\r\n/g,"\n");
+    var utftext = "";
+
+    for (var n = 0; n < str_data.length; n++) {
+        var c = str_data.charCodeAt(n);
+        if (c < 128) {
+            utftext += String.fromCharCode(c);
+        } else if((c > 127) && (c < 2048)) {
+            utftext += String.fromCharCode((c >> 6) | 192);
+            utftext += String.fromCharCode((c & 63) | 128);
+        } else {
+            utftext += String.fromCharCode((c >> 12) | 224);
+            utftext += String.fromCharCode(((c >> 6) & 63) | 128);
+            utftext += String.fromCharCode((c & 63) | 128);
+        }
+    }
+
+    return utftext;
+}
+
+module.exports = function (store) {
+    function getset (name, value) {
+        var node = vars.store;
+        var keys = name.split('.');
+        keys.slice(0,-1).forEach(function (k) {
+            if (node[k] === undefined) node[k] = {};
+            node = node[k]
+        });
+        var key = keys[keys.length - 1];
+        if (arguments.length == 1) {
+            return node[key];
+        }
+        else {
+            return node[key] = value;
+        }
+    }
+
+    var vars = {
+        get : function (name) {
+            return getset(name);
+        },
+        set : function (name, value) {
+            return getset(name, value);
+        },
+        store : store || {},
+    };
+    return vars;
+};

fixtures/lib2/file_2.js

@@ -0,0 +1,56 @@
+module.exports = function (store) {
+    function getset (name, value) {
+        var node = vars.store;
+        var keys = name.split('.');
+        keys.slice(0,-1).forEach(function (k) {
+            if (node[k] === undefined) node[k] = {};
+            node = node[k]
+        });
+        var key = keys[keys.length - 1];
+        if (arguments.length == 1) {
+            return node[key];
+        }
+        else {
+            return node[key] = value;
+        }
+    }
+
+    var vars = {
+        get : function (name) {
+            return getset(name);
+        },
+        set : function (name, value) {
+            return getset(name, value);
+        },
+        store : store || {},
+    };
+    return vars;
+};
+module.exports = function (store) {
+    function getset (name, value) {
+        var node = vars.store;
+        var keys = name.split('.');
+        keys.slice(0,-1).forEach(function (k) {
+            if (node[k] === undefined) node[k] = {};
+            node = node[k]
+        });
+        var key = keys[keys.length - 1];
+        if (arguments.length == 1) {
+            return node[key];
+        }
+        else {
+            return node[key] = value;
+        }
+    }
+
+    var vars = {
+        get : function (name) {
+            return getset(name);
+        },
+        set : function (name, value) {
+            return getset(name, value);
+        },
+        store : store || {},
+    };
+    return vars;
+};

packages/core/src/interfaces/options.interface.ts

@@ -27,6 +27,7 @@ export interface IOptions {
 	absolute?: boolean;
 	noSymlinks?: boolean;
 	skipLocal?: boolean;
+  skipIsolated?: string[][];
 	ignoreCase?: boolean;
 	gitignore?: boolean;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any

packages/finder/src/in-files-detector.ts

@@ -13,7 +13,7 @@ import {
 } from '@jscpd/core';
 import {getFormatByFile} from '@jscpd/tokenizer';
 import {EntryWithContent, IHook, IReporter} from './interfaces';
-import {SkipLocalValidator} from './validators';
+import {SkipLocalValidator, SkipIsolatedValidator} from './validators';
 
 export class InFilesDetector {
 
@@ -55,6 +55,10 @@ export class InFilesDetector {
       validators.push(new SkipLocalValidator());
     }
 
+    if (options.skipIsolated) {
+      validators.push(new SkipIsolatedValidator());
+    }
+
     const detector = new Detector(this.tokenizer, store, validators, options);
 
     this.subscribes.forEach((listener: ISubscriber) => {

packages/finder/src/validators/index.ts

@@ -1 +1,2 @@
 export * from './skip-local.validator';
+export * from './skip-isolated.validator';

packages/finder/src/validators/skip-isolated.validator.ts

@@ -0,0 +1,43 @@
+import {getOption, IClone, ICloneValidator, IOptions, IValidationResult} from '@jscpd/core';
+import {isAbsolute, relative} from "path";
+
+export class SkipIsolatedValidator implements ICloneValidator {
+  isRelativeMemoMap = new Map();
+
+
+  validate(clone: IClone, options: IOptions): IValidationResult {
+    const status = !this.shouldSkipClone(clone, options);
+    return {
+      status,
+      clone,
+      message: [
+        `Sources of duplication located in isolated folder (${clone.duplicationA.sourceId}, ${clone.duplicationB.sourceId})`
+      ]
+    };
+  }
+
+  public shouldSkipClone(clone: IClone, options: IOptions): boolean {
+    const skipIsolatedPathList: string[][] = getOption('skipIsolated', options);
+    return skipIsolatedPathList.some(
+      (dirList) => {
+        const relA = dirList.find(dir => this.isRelativeMemo(clone.duplicationA.sourceId, dir));
+        const relB = dirList.find(dir => this.isRelativeMemo(clone.duplicationB.sourceId, dir));
+        return relA && relB && relA !== relB
+      }
+    );
+  }
+
+  private isRelativeMemo(file: string, dir: string) {
+    const memoKey = `${file},${dir}`;
+    if (this.isRelativeMemoMap.has(memoKey)) return this.isRelativeMemoMap.get(memoKey);
+    const isRel = SkipIsolatedValidator.isRelative(file, dir)
+    this.isRelativeMemoMap.set(memoKey, isRel);
+    return isRel;
+  }
+
+  private static isRelative(file: string, path: string): boolean {
+    const rel = relative(path, file);
+    return rel !== '' && !rel.startsWith('..') && !isAbsolute(rel);
+  }
+
+}

packages/jscpd/README.md

@@ -92,9 +92,9 @@ Minimal block size of code in tokens. The block of code less than `min-tokens` w
  - Cli options: `--min-tokens`, `-k`
  - Type: **number**
  - Default: **50**
- 
+
  *This option is called ``minTokens`` in the config file.*
- 
+
 ### Min Lines
 
 Minimal block size of code in lines. The block of code less than `min-lines` will be skipped.
@@ -243,6 +243,31 @@ will detect clones in separate folders only, clones from same folder will be ski
  - Type: **boolean**
  - Default: **false**
 
+### Skip Isolated
+Use for skip detect duplications in isolated folder.
+
+Example:
+```bash
+jscpd . --skipLocal "packages/businessA|packages/businessB,libs/businessA|libs|businessB"
+```
+clones from the isolated folder will be skipped.
+
+```text
+monorepo/
+├─ .monorepo.config.json
+├─ node_modules/
+├─ packages/
+│  ├─ businessA/ // for TEAM A only
+│  ├─ businessB/ // for TEAM B only
+├─ globals/
+├─ infra /
+```
+
+- Cli options: `--skipIsolated`
+- Type: **string[][]**
+
+
+
 ### Formats Extensions
 Define the list of formats with file extensions. Available over [150 formats](../../supported_formats.md).
 

packages/jscpd/__tests__/options.test.ts

@@ -4,6 +4,7 @@ import {IClone} from '@jscpd/core';
 import {jscpd, detectClones} from '../src';
 import {bold, yellow} from 'colors/safe';
 import sinon = require('sinon');
+import * as fs from "fs";
 
 const pathToFixtures = __dirname + '/../../../fixtures';
 
@@ -126,6 +127,49 @@ describe('jscpd options', () => {
 		});
 	});
 
+  describe('skip isolated', () => {
+    const folder1Path = pathToFixtures + '/folder1';
+    const folder2Path = pathToFixtures + '/folder2';
+    const lib1Path = pathToFixtures + '/lib1';
+    const lib2Path = pathToFixtures + '/lib2';
+
+    it('should not skip clone if it is located in isolated folder without --skipIsolated option', async () => {
+      const clones: IClone[] = await jscpd([
+        '', '',
+        folder1Path,
+        folder2Path,
+        lib1Path,
+        lib2Path,
+      ]);
+      // lib2 file_2.js lib2 file_2.js
+      // lib1 file_1.js lib2 file_2.js
+      // lib1 file2.c lib1 file2.c
+      // folder2 file_2.js lib2 file_2.js
+      // folder1 file_1.js lib1 file_1.js
+      // folder1 file2.c lib1 file2.c
+      expect(clones.length).to.equal(6);
+    });
+
+    it('should skip clone if it is located in isolated folder with --skipIsolated option', async () => {
+      const clones: IClone[] = await jscpd([
+        '', '',
+        folder1Path,
+        folder2Path,
+        lib1Path,
+        lib2Path,
+        '--skipIsolated',
+        // folder1Path is isolated with folder2Path, lib1Path is isolated with lib2Path
+        `${folder1Path}|${folder2Path},${lib1Path}|${lib2Path}`
+      ]);
+      // lib2 file_2.js lib2 file_2.js
+      // lib1 file2.c lib1 file2.c
+      // folder2 file_2.js lib2 file_2.js
+      // folder1 file_1.js lib1 file_1.js
+      // folder1 file2.c lib1 file2.c
+      expect(clones.length).to.equal(5);
+    });
+  });
+
 	describe('silent', () => {
     it('should not print more information about detection process', async () => {
       await jscpd(['', '', fileWithClones, '--silent']);

packages/jscpd/src/init/cli.ts

@@ -50,6 +50,7 @@ export function initCli(packageJson, argv: string[]): Command {
 		.option('-v, --verbose', 'show full information during detection process')
 		.option('--list', 'show list of total supported formats')
 		.option('--skipLocal', 'skip duplicates in local folders, just detect cross folders duplications')
+		.option('--skipIsolated [string]', 'skip duplicates cross from isolated folders for monorepo (e.g. packages/a|packages/b,lib/a|lib/b)')
     .option('--exitCode [number]', 'exit code to use when code duplications are detected')
 
 	cli.parse(argv);

packages/jscpd/src/options.ts

@@ -27,6 +27,7 @@ const convertCliToOptions = (cli: Command): Partial<IOptions> => {
     absolute: cli.absolute,
     noSymlinks: cli.noSymlinks,
     skipLocal: cli.skipLocal,
+    skipIsolated: cli.skipIsolated?.split(',')?.map(s => s.split('|')),
     ignoreCase: cli.ignoreCase,
     gitignore: cli.gitignore,
     exitCode: cli.exitCode,