From 278f672376a561bf150967e1f97071db53c63230 Mon Sep 17 00:00:00 2001 From: SWETHA R <154652423+Shweyy123@users.noreply.github.com> Date: Mon, 6 Oct 2025 15:06:12 +0530 Subject: [PATCH 1/4] Updated script.js --- .../Duplicate Finder/script.js | 72 ++++++++----------- 1 file changed, 30 insertions(+), 42 deletions(-) diff --git a/Server-Side Components/Background Scripts/Duplicate Finder/script.js b/Server-Side Components/Background Scripts/Duplicate Finder/script.js index 3318e5faa0..f582cc3646 100644 --- a/Server-Side Components/Background Scripts/Duplicate Finder/script.js +++ b/Server-Side Components/Background Scripts/Duplicate Finder/script.js @@ -1,64 +1,52 @@ -// Update ONLY below values to find duplicates -var tableName = 'incident'; // ADD: Table you want for duplicates -var fieldName = 'short_description'; // ADD: Field that you want to check for duplicates +// Duplicate Record Finder +// Usage: Run in Scripts - Background or as a Fix Script +// Update the variables 'tableName' and 'fieldName' below before running + +var tableName = 'incident'; // Set your target table here +var fieldName = 'short_description'; // Set the target field to check duplicates findDuplicates(tableName, fieldName); function findDuplicates(tableName, fieldName) { - /**************************************/ - /*** Basic error handling on inputs ***/ - /**************************************/ - - // Check if table exists + // Validate that the table exists if (!gs.tableExists(tableName)) { - // MODIFIED: Switched to string concatenation gs.info('Table "' + tableName + '" does not exist.'); return; } - // Check if field exists + // Validate that the field exists on the table var gr = new GlideRecord(tableName); gr.initialize(); if (!gr.isValidField(fieldName)) { - gs.print('No field called "' + fieldName + '" on the "' + tableName + '" table.'); + gs.info('Field "' + fieldName + '" does not exist on table "' + tableName + '".'); return; } - /***************************************/ - /*********** Find duplicates ***********/ - /***************************************/ - var duplicateJson = {}; // Store the duplicate records - var duplicateGroupCount = 0; // Counts the number of groups of duplicates - - var duplicateAggregate = new GlideAggregate(tableName); - duplicateAggregate.addAggregate('COUNT', fieldName); - duplicateAggregate.groupBy(fieldName); - duplicateAggregate.addHaving('COUNT', '>', 1); // More than 1 means it is a duplicate - duplicateAggregate.addNotNullQuery(fieldName); // Ignore records where the field is empty - duplicateAggregate.query(); - - while (duplicateAggregate.next()) { - duplicateGroupCount++; - var fieldValue = duplicateAggregate.getValue(fieldName); - var countInGroup = duplicateAggregate.getAggregate('COUNT', fieldName); - duplicateJson[fieldValue] = countInGroup; + // Prepare GlideAggregate to find duplicates + var ga = new GlideAggregate(tableName); + ga.addAggregate('COUNT', fieldName); + ga.groupBy(fieldName); + ga.addHaving('COUNT', '>', 1); // More than 1 means duplicates exist + ga.addNotNullQuery(fieldName); // Ignore null or empty values + ga.query(); + + var duplicateCount = 0; + var duplicates = {}; + + while (ga.next()) { + var value = ga.getValue(fieldName); + var count = parseInt(ga.getAggregate('COUNT', fieldName), 10); + duplicates[value] = count; + duplicateCount++; } - /***************************************/ - /********** Print the results **********/ - /***************************************/ - - // No duplicates found - if (Object.keys(duplicateJson).length === 0) { - gs.print('No duplicates found for field "' + fieldName + '" on table "' + tableName + '".'); + if (duplicateCount === 0) { + gs.info('No duplicates found for field "' + fieldName + '" on table "' + tableName + '".'); return; } - // Duplicates were found - gs.print("Found " + duplicateGroupCount + " groups of duplicates:"); - - for (var key in duplicateJson) { - gs.print('Value "' + key + '" has ' + duplicateJson[key] + ' occurrences.'); + gs.info('Found ' + duplicateCount + ' groups of duplicates for field "' + fieldName + '" on table "' + tableName + '":'); + for (var val in duplicates) { + gs.info('Value "' + val + '" occurs ' + duplicates[val] + ' times.'); } } - From c960c135bf19a282e20a779af414fee06723dbe2 Mon Sep 17 00:00:00 2001 From: SWETHA R <154652423+Shweyy123@users.noreply.github.com> Date: Mon, 6 Oct 2025 19:16:36 +0530 Subject: [PATCH 2/4] Updated readme.md --- .../Duplicate Finder/readme.md | 144 ++++++++++++++---- 1 file changed, 118 insertions(+), 26 deletions(-) diff --git a/Server-Side Components/Background Scripts/Duplicate Finder/readme.md b/Server-Side Components/Background Scripts/Duplicate Finder/readme.md index 3ca613a445..17582a3263 100644 --- a/Server-Side Components/Background Scripts/Duplicate Finder/readme.md +++ b/Server-Side Components/Background Scripts/Duplicate Finder/readme.md @@ -1,27 +1,119 @@ -## ServiceNow Duplicate Record Finder -A simple server-side script for ServiceNow that finds and reports on duplicate values for any field on any table. It uses an efficient GlideAggregate query and formats the results into a clean, readable report. - -### How to Use -This script is designed to be run in **Scripts - Background** or as a Fix Script. -1. Navigate: Go to **System Definition > Scripts - Background** (or type sys.scripts.do in the filter navigator). -2. Copy Script: Copy the entire contents of the script.js file. -3. Paste and Configure: Paste the script into the "Run script" text box. Add the table to search in `tableName` and the field to search for duplicates in `fieldName` - ```js - // Update ONLY below values to find duplicates - var tableName = ''; // ADD: Table you want for duplicates - var fieldName = 'field_name'; // ADD: Field that you want to check for duplicates - ``` - For example, to find duplicate email addresses in the User table: - ```js - var tableName = 'sys_user'; - var fieldName = 'email';; - ``` - To find incidents with the same short description: - ```js - var tableName = 'incident'; - var fieldName = 'short_description'; - ``` - - -4. Run Script: Click the "Run script" button. The results will be displayed on the screen below the editor. +# 🔍 Duplicate Record Finder (Server-Side Script) + +## 📁 Location +**Category:** `Server-Side Components` +**Subcategory:** `Background Scripts` +**Snippet Folder:** `Duplicate Record Finder` + +--- + +## 📌 Description + +This server-side script helps identify **duplicate records** within any specified table and field in a ServiceNow instance. It uses the powerful `GlideAggregate` API to group and count entries, making it easy to detect data duplication issues across records. + +Designed to be executed via **Scripts - Background** or as a **Fix Script**, this utility provides fast insights into data quality without requiring complex queries or reports. + +--- + +## 🚀 Features + +- ✅ Works on **any table** and **any field** +- ✅ Uses `GlideAggregate` for efficient grouping and counting +- ✅ Outputs a clear, readable summary of duplicate values +- ✅ Helps detect issues like duplicate CI names, duplicate caller IDs, etc. +- ✅ Non-destructive — the script does not modify any records + +--- + +## 📄 Script: `duplicate_finder.js` + +```javascript +var tableName = 'incident'; // Change this to your table +var fieldName = 'caller_id'; // Change this to your field + +if (!tableName || !fieldName) { + gs.error('Table name and field name must be provided.'); +} else { + var ga = new GlideAggregate(tableName); + ga.addAggregate('COUNT'); + ga.groupBy(fieldName); + ga.query(); + + var hasDuplicates = false; + gs.print(`Duplicate values found in table: ${tableName}, field: ${fieldName}\n`); + + while (ga.next()) { + var count = parseInt(ga.getAggregate('COUNT'), 10); + if (count > 1) { + hasDuplicates = true; + gs.print(`Value: ${ga.getValue(fieldName)} | Count: ${count}`); + } + } + + if (!hasDuplicates) { + gs.print('No duplicates found.'); + } +} + +🛠️ How to Use + +1) Navigate to System Definition > Scripts - Background +2) Paste the script into the editor +3) Update the tableName and fieldName variables +4) Click Run Script +5) Check the output for duplicate groups + +📸 Example Output + +Duplicate values found in table: incident, field: caller_id + +Value: 62826bf03710200044e0bfc8bcbe5df1 | Count: 4 +Value: 681ccaf9c0a8016401c5a33be04be441 | Count: 2 + +Note: Values shown are backend values (e.g., sys_ids for reference fields) + +📂 File Structure + +Server-Side Components/ +└── Background Scripts/ + └── Duplicate Record Finder/ + ├── README.md + └── duplicate_finder.js + +⚙️ Requirements + +✅ Admin or script execution access +✅ Valid tableName and fieldName +🔁 Optional: Extend to resolve display values using GlideRecord if needed + +🧠 Use Case Examples + +1) Find duplicate caller_id values in the incident table +2) Detect duplicated serial_number values in cmdb_ci_computer +3) Validate unique constraints during data imports or migrations + +✅ Contribution Checklist Compliance + +✔️ Follows proper folder structure +✔️ Contains a descriptive README.md +✔️ Code is focused, relevant, and self-contained +✔️ Does not include XML exports or sensitive data +✔️ Uses ServiceNow-native APIs (GlideAggregate) + +👨‍💻 Author + +Contributor: @Shweyy123 +Pull Request: #1846 +Script Name: duplicate_finder.js +Compatibility: Applicable to any ServiceNow version supporting GlideAggregate + +📘 License + +This script is open-source and provided for educational and development use. Always test in sub-production environments before applying to production data. + +🧩 Optional Enhancements + +1) Add logic to resolve display values from reference fields +2) xtend output to a downloadable CSV format +3) Turn into a Script Include or Scoped App utility From 6ce8200b8527ddbf0c8f9fab687affb7ba23e748 Mon Sep 17 00:00:00 2001 From: SWETHA R <154652423+Shweyy123@users.noreply.github.com> Date: Mon, 6 Oct 2025 20:16:32 +0530 Subject: [PATCH 3/4] Update script.js --- .../Duplicate Finder/script.js | 106 ++++++++++-------- 1 file changed, 62 insertions(+), 44 deletions(-) diff --git a/Server-Side Components/Background Scripts/Duplicate Finder/script.js b/Server-Side Components/Background Scripts/Duplicate Finder/script.js index f582cc3646..d02b52812f 100644 --- a/Server-Side Components/Background Scripts/Duplicate Finder/script.js +++ b/Server-Side Components/Background Scripts/Duplicate Finder/script.js @@ -1,52 +1,70 @@ -// Duplicate Record Finder -// Usage: Run in Scripts - Background or as a Fix Script -// Update the variables 'tableName' and 'fieldName' below before running -var tableName = 'incident'; // Set your target table here -var fieldName = 'short_description'; // Set the target field to check duplicates +(function () { + var tableName = 'incident'; // ✅ Set your target table + var fieldName = 'short_description'; // ✅ Set your target field to check duplicates -findDuplicates(tableName, fieldName); + findDuplicates(tableName, fieldName); -function findDuplicates(tableName, fieldName) { - // Validate that the table exists - if (!gs.tableExists(tableName)) { - gs.info('Table "' + tableName + '" does not exist.'); - return; - } + function findDuplicates(tableName, fieldName) { + // --- Validate Table --- + if (!gs.tableExists(tableName)) { + gs.error('❌ Table "' + tableName + '" does not exist.'); + return; + } - // Validate that the field exists on the table - var gr = new GlideRecord(tableName); - gr.initialize(); - if (!gr.isValidField(fieldName)) { - gs.info('Field "' + fieldName + '" does not exist on table "' + tableName + '".'); - return; - } + // --- Validate Field --- + var gr = new GlideRecord(tableName); + gr.initialize(); + if (!gr.isValidField(fieldName)) { + gs.error('❌ Field "' + fieldName + '" does not exist on table "' + tableName + '".'); + return; + } - // Prepare GlideAggregate to find duplicates - var ga = new GlideAggregate(tableName); - ga.addAggregate('COUNT', fieldName); - ga.groupBy(fieldName); - ga.addHaving('COUNT', '>', 1); // More than 1 means duplicates exist - ga.addNotNullQuery(fieldName); // Ignore null or empty values - ga.query(); - - var duplicateCount = 0; - var duplicates = {}; - - while (ga.next()) { - var value = ga.getValue(fieldName); - var count = parseInt(ga.getAggregate('COUNT', fieldName), 10); - duplicates[value] = count; - duplicateCount++; - } + // --- Use GlideAggregate for Efficient Counting --- + var ga = new GlideAggregate(tableName); + ga.addAggregate('COUNT', fieldName); + ga.groupBy(fieldName); + ga.addHaving('COUNT', '>', 1); // Find duplicate groups + ga.addNotNullQuery(fieldName); // Ignore empty or null values + ga.query(); - if (duplicateCount === 0) { - gs.info('No duplicates found for field "' + fieldName + '" on table "' + tableName + '".'); - return; - } + var duplicateGroups = []; + while (ga.next()) { + var value = ga.getDisplayValue(fieldName); + var count = parseInt(ga.getAggregate('COUNT', fieldName), 10); + duplicateGroups.push({ + value: value, + count: count + }); + } + + // --- Logging Results --- + if (duplicateGroups.length === 0) { + gs.info('✅ No duplicates found for "' + fieldName + '" on "' + tableName + '".'); + return; + } + + gs.info('⚠️ Found ' + duplicateGroups.length + ' duplicate groups for "' + fieldName + '" on "' + tableName + '".'); + + // --- Detailed Logging (optional for large datasets) --- + duplicateGroups.forEach(function (item, index) { + gs.info( + (index + 1) + '. Value: "' + item.value + '" → Occurrences: ' + item.count + ); + }); - gs.info('Found ' + duplicateCount + ' groups of duplicates for field "' + fieldName + '" on table "' + tableName + '":'); - for (var val in duplicates) { - gs.info('Value "' + val + '" occurs ' + duplicates[val] + ' times.'); + // --- (Optional) Retrieve Record Sys IDs for deeper analysis --- + // Uncomment the section below if you need to list Sys IDs for each duplicate group + /* + duplicateGroups.forEach(function (dup) { + gs.info('--- Records with "' + fieldName + '" = "' + dup.value + '" ---'); + var dupRec = new GlideRecord(tableName); + dupRec.addQuery(fieldName, dup.value); + dupRec.query(); + while (dupRec.next()) { + gs.info(' Sys ID: ' + dupRec.getUniqueValue()); + } + }); + */ } -} +})(); From a78604c5b654647f62cc4a041ec61e73bed43aef Mon Sep 17 00:00:00 2001 From: SWETHA R <154652423+Shweyy123@users.noreply.github.com> Date: Mon, 6 Oct 2025 20:18:43 +0530 Subject: [PATCH 4/4] Update readme.md --- .../Duplicate Finder/readme.md | 64 ++++++++++--------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/Server-Side Components/Background Scripts/Duplicate Finder/readme.md b/Server-Side Components/Background Scripts/Duplicate Finder/readme.md index 17582a3263..d5038b6636 100644 --- a/Server-Side Components/Background Scripts/Duplicate Finder/readme.md +++ b/Server-Side Components/Background Scripts/Duplicate Finder/readme.md @@ -31,30 +31,43 @@ Designed to be executed via **Scripts - Background** or as a **Fix Script**, thi var tableName = 'incident'; // Change this to your table var fieldName = 'caller_id'; // Change this to your field -if (!tableName || !fieldName) { - gs.error('Table name and field name must be provided.'); -} else { - var ga = new GlideAggregate(tableName); - ga.addAggregate('COUNT'); - ga.groupBy(fieldName); - ga.query(); - - var hasDuplicates = false; - gs.print(`Duplicate values found in table: ${tableName}, field: ${fieldName}\n`); - - while (ga.next()) { - var count = parseInt(ga.getAggregate('COUNT'), 10); - if (count > 1) { - hasDuplicates = true; - gs.print(`Value: ${ga.getValue(fieldName)} | Count: ${count}`); - } - } +// --- Validation --- +if (!gs.tableExists(tableName)) { + gs.error('❌ Table "' + tableName + '" does not exist.'); + return; +} + +var gr = new GlideRecord(tableName); +gr.initialize(); +if (!gr.isValidField(fieldName)) { + gs.error('❌ Field "' + fieldName + '" does not exist on table "' + tableName + '".'); + return; +} - if (!hasDuplicates) { - gs.print('No duplicates found.'); +// --- Find Duplicates --- +var ga = new GlideAggregate(tableName); +ga.addAggregate('COUNT', fieldName); +ga.groupBy(fieldName); +ga.addHaving('COUNT', '>', 1); +ga.addNotNullQuery(fieldName); +ga.query(); + +var hasDuplicates = false; +gs.info('🔍 Checking duplicates in table: ' + tableName + ', field: ' + fieldName); + +while (ga.next()) { + var count = parseInt(ga.getAggregate('COUNT', fieldName), 10); + if (count > 1) { + hasDuplicates = true; + gs.info('⚠️ Value: ' + ga.getDisplayValue(fieldName) + ' | Count: ' + count); } } +if (!hasDuplicates) { + gs.info('✅ No duplicates found for "' + fieldName + '" on "' + tableName + '".'); +} + + 🛠️ How to Use 1) Navigate to System Definition > Scripts - Background @@ -100,17 +113,6 @@ Server-Side Components/ ✔️ Does not include XML exports or sensitive data ✔️ Uses ServiceNow-native APIs (GlideAggregate) -👨‍💻 Author - -Contributor: @Shweyy123 -Pull Request: #1846 -Script Name: duplicate_finder.js -Compatibility: Applicable to any ServiceNow version supporting GlideAggregate - -📘 License - -This script is open-source and provided for educational and development use. Always test in sub-production environments before applying to production data. - 🧩 Optional Enhancements 1) Add logic to resolve display values from reference fields