find_duplicate_files
find_duplicate_files
Finds duplicate files by comparing MD5 hashes. Scans a directory recursively and groups files with identical content. Use when freeing disk space by removing redundant copies.
Metadata
Name
find_duplicate_files
Updated
2 weeks ago
Source
Risk
Low
Requires consent
false
Affected scope
user
Code
/**
* mcp/skills/findDuplicateFiles.ts — find_duplicate_files skill
*
* Finds duplicate files by comparing MD5 hashes. Scans a directory
* recursively and groups files with identical content. Use when freeing
* disk space by removing redundant copies.
*
* Platform strategy
* -----------------
* Both Pure Node.js crypto.createHash('md5') with fs.createReadStream —
* cross-platform, no child_process needed.
*
* Smoke test
* npx tsx -r dotenv/config mcp/skills/findDuplicateFiles.ts
*/
import * as fs from "fs";
import * as fsp from "fs/promises";
import * as os from "os";
import * as nodePath from "path";
import * as crypto from "crypto";
import { z } from "zod";
import { expandTilde } from "./_shared/expandTilde";
import { Semaphore } from "./_shared/semaphore";
const _statSem = process.platform === "win32" ? new Semaphore(32) : null;
const _readdirSem = process.platform === "win32" ? new Semaphore(16) : null;
// -- Meta ---------------------------------------------------------------------
export const meta = {
name: "find_duplicate_files",
description:
"Finds duplicate files by comparing MD5 hashes. Scans a directory " +
"recursively and groups files with identical content. " +
"Use when freeing disk space by removing redundant copies.",
riskLevel: "low",
destructive: false,
requiresConsent: false,
supportsDryRun: false,
affectedScope: ["user"],
auditRequired: false,
tccCategories: ["FullDiskAccess"],
timeoutMs: 180_000,
schema: {
path: z
.string()
.optional()
.describe("Directory to scan. Defaults to home directory"),
minSizeMb: z
.number()
.optional()
.describe("Minimum file size in MB to consider. Default: 5"),
extensions: z
.array(z.string())
.optional()
.describe("File extensions to check e.g. ['.jpg','.pdf']. Omit for all files"),
topDeletableLimit: z
.number()
.int()
.positive()
.optional()
.describe(
"When set, the tool returns ONLY a pre-computed `topDeletables: " +
"[{path, sizeBytes}]` array of the N largest deletable duplicate " +
"files (one keeper per group is preserved; the rest are deletable, " +
"ranked by per-file size descending). `duplicateGroups` and " +
"`totalWastedBytes` are omitted to prevent downstream substitution " +
"from accidentally surfacing scan-wide aggregates as if they were " +
"the actionable slice. Use this when the caller wants a bounded " +
"view ready to feed into delete_files (e.g. disk-cleanup's 5-cap)."
),
},
} as const;
// -- Types --------------------------------------------------------------------
interface DuplicateFile {
path: string;
name: string;
}
interface DuplicateGroup {
hash: string;
sizeMb: number;
files: DuplicateFile[];
}
// -- Constants ----------------------------------------------------------------
const MAX_DEPTH = 10;
const SKIP_DIRS = new Set([
"node_modules", ".git", ".npm", ".yarn", ".cache",
"__pycache__", ".venv", "venv",
".Trash", ".Trashes",
"$Recycle.Bin", "System Volume Information",
...(process.platform === "win32" ? ["Packages", "D3DSCache"] : []),
]);
const HASH_CONCURRENCY = process.platform === "win32" ? 4 : 16;
const PARTIAL_HASH_BYTES = 64 * 1024;
// -- Helpers ------------------------------------------------------------------
function hashFile(filePath: string, end?: number): Promise<string> {
return new Promise((resolve, reject) => {
const hash = crypto.createHash("md5");
const stream = end !== undefined
? fs.createReadStream(filePath, { end })
: fs.createReadStream(filePath);
stream.on("data", (chunk) => hash.update(chunk));
stream.on("end", () => resolve(hash.digest("hex")));
stream.on("error", reject);
});
}
interface WalkStats {
dirsVisited: number;
dirsPermissionDenied: number;
deadlineHit: boolean;
}
function isPermissionError(err: unknown): boolean {
const code = (err as { code?: string })?.code;
return code === "EPERM" || code === "EACCES";
}
async function walk(
dir: string,
minBytes: number,
extensions: Set<string> | null,
acc: { path: string; size: number }[],
depth: number,
stats: WalkStats,
deadlineMs: number,
): Promise<void> {
if (depth > MAX_DEPTH) return;
if (Date.now() >= deadlineMs) { stats.deadlineHit = true; return; }
stats.dirsVisited++;
if (_readdirSem) await _readdirSem.acquire();
let entries: import("fs").Dirent[];
try {
entries = await fsp.readdir(dir, { withFileTypes: true });
} catch (err) {
if (isPermissionError(err)) stats.dirsPermissionDenied++;
return;
} finally {
_readdirSem?.release();
}
await Promise.allSettled(
entries.map(async (e) => {
if (stats.deadlineHit) return;
if (e.name.startsWith(".") && depth > 0) return;
const full = nodePath.join(dir, e.name);
if (e.isDirectory()) {
if (SKIP_DIRS.has(e.name)) return;
await walk(full, minBytes, extensions, acc, depth + 1, stats, deadlineMs);
} else if (e.isFile()) {
if (extensions && !extensions.has(nodePath.extname(e.name).toLowerCase())) return;
if (_statSem) await _statSem.acquire();
try {
const stat = await fsp.stat(full);
if (stat.size >= minBytes) {
acc.push({ path: full, size: stat.size });
}
} catch {
} finally { _statSem?.release(); }
}
}),
);
}
async function run(
{
path: inputPath,
minSizeMb = 5,
extensions,
topDeletableLimit,
}: {
path?: string;
minSizeMb?: number;
extensions?: string[];
topDeletableLimit?: number;
} = {},
) {
const home = os.homedir();
const scanPath = nodePath.resolve(expandTilde(inputPath || home) ?? home);
const rel = nodePath.relative(home, scanPath);
if (rel.startsWith("..") || nodePath.isAbsolute(rel)) {
throw new Error(
`[find_duplicate_files] Path must be within home directory (${home}): ${scanPath}`,
);
}
try {
await fsp.access(scanPath);
} catch {
throw new Error(`[find_duplicate_files] Path not accessible: ${scanPath}`);
}
const minBytes = Math.max(0, minSizeMb * 1_000_000);
const extSet = extensions && extensions.length > 0
? new Set(extensions.map((e) => (e.startsWith(".") ? e : `.${e}`).toLowerCase()))
: null;
const ceilingMs = Date.now() + 60_000;
const internalDeadlineMs = Date.now() + Math.floor(ceilingMs - Date.now() * 0.9);
const files: { path: string; size: number }[] = [];
const walkStats: WalkStats = { dirsVisited: 0, dirsPermissionDenied: 0, deadlineHit: false };
await walk(scanPath, minBytes, extSet, files, 0, walkStats, internalDeadlineMs);
const bySize = new Map<number, typeof files>();
for (const f of files) {
const group = bySize.get(f.size) ?? [];
group.push(f);
bySize.set(f.size, group);
}
const sizeCollisions = [...bySize.values()].filter((g) => g.length > 1).flat();
let hashDeadlineHit = false;
async function runHashWorkers(
pool: typeof files,
hashFn: (path: string) => Promise<string>,
): Promise<Map<string, typeof files>> {
const result = new Map<string, typeof files>();
let cursor = 0;
const workers: Promise<void>[] = [];
const worker = async () => {
while (cursor < pool.length) {
if (Date.now() >= internalDeadlineMs) { hashDeadlineHit = true; return; }
const f = pool[cursor++];
try {
const h = await hashFn(f.path);
const group = result.get(h) ?? [];
group.push(f);
result.set(h, group);
} catch { }
}
};
for (let i = 0; i < HASH_CONCURRENCY; i++) workers.push(worker());
await Promise.all(workers);
return result;
}
const byPartial = await runHashWorkers(
sizeCollisions,
(p) => hashFile(p, PARTIAL_HASH_BYTES - 1),
);
const fullCandidates = [...byPartial.values()].filter((g) => g.length > 1).flat();
const byHash = await runHashWorkers(fullCandidates, (p) => hashFile(p));
const duplicateGroups: DuplicateGroup[] = [];
let totalWastedBytes = 0;
for (const [hash, group] of byHash.entries()) {
if (group.length < 2) continue;
const sizeMb = Math.round((group[0].size / 1_000_000) * 100) / 100;
totalWastedBytes += (group.length - 1) * group[0].size;
duplicateGroups.push({
hash,
sizeMb,
files: group.map((f) => ({ path: f.path, name: nodePath.basename(f.path) })),
});
}
duplicateGroups.sort((a, b) => {
const wastedA = (a.files.length - 1) * a.sizeMb;
const wastedB = (b.files.length - 1) * b.sizeMb;
return wastedB - wastedA;
});
let warning: string | undefined;
if (walkStats.deadlineHit || hashDeadlineHit) {
warning =
"Duplicate scan stopped at the per-tool deadline. Results cover only the " +
"files scanned so far — there may be more duplicates in untraversed subtrees.";
} else if (
walkStats.dirsVisited > 0 &&
walkStats.dirsPermissionDenied / walkStats.dirsVisited > 0.2
) {
warning =
`Duplicate scan is incomplete: ${walkStats.dirsPermissionDenied} of ` +
`${walkStats.dirsVisited} directories could not be read (likely missing ` +
`Full Disk Access). Open System Settings → Privacy & Security → ` +
`Full Disk Access, enable AI Support Agent, then quit and relaunch.`;
}
if (topDeletableLimit !== undefined) {
const pool: { path: string; sizeBytes: number }[] = [];
for (const group of duplicateGroups) {
const sizeBytes = Math.round(group.sizeMb * 1_000_000);
for (let i = 1; i < group.files.length; i++) {
pool.push({ path: group.files[i].path, sizeBytes });
}
}
pool.sort((a, b) => b.sizeBytes - a.sizeBytes);
const topDeletables = pool.slice(0, topDeletableLimit);
return {
scannedPath: scanPath,
scannedFiles: files.length,
topDeletables,
partial: walkStats.deadlineHit || hashDeadlineHit,
...(warning ? { warning } : {}),
};
}
return {
scannedPath: scanPath,
scannedFiles: files.length,
duplicateGroups,
totalWastedBytes,
partial: walkStats.deadlineHit || hashDeadlineHit,
...(warning ? { warning } : {}),
};
}
// -- CLI smoke test -----------------------------------------------------------
if (false) {
run({})
.then(r => console.log(JSON.stringify(r, null, 2)))
.catch((err: Error) => { console.error(err.message); process.exit(1); });
}