From 8d246e2c1b6c2b9ac3c89fb63cd3cfe0ed08b25e Mon Sep 17 00:00:00 2001 From: lot Date: Mon, 14 Aug 2023 22:14:09 +0800 Subject: [PATCH] generate file hash --- .gitignore | 1 + localFiles.mjs | 111 ++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 12 +++++ package.json | 8 +--- 4 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 localFiles.mjs diff --git a/.gitignore b/.gitignore index b1587a4..dbc4ada 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.generate-file-hash.json # Logs logs *.log diff --git a/localFiles.mjs b/localFiles.mjs new file mode 100644 index 0000000..bbc53d4 --- /dev/null +++ b/localFiles.mjs @@ -0,0 +1,111 @@ +import readdir from "readdir-enhanced"; +import fs from "fs"; +import crypto from "crypto"; +import multimatch from "multimatch"; + +function applyExcludeFilter(stat, excludeFilters) { + // match exclude, return immediatley + if (excludeFilters.length > 0) { + // todo this could be a performance problem... + const pathWithFolderSlash = stat.path + (stat.isDirectory() ? "/" : ""); + const excludeMatch = multimatch(pathWithFolderSlash, excludeFilters, { matchBase: true, dot: true }); + + if (excludeMatch.length > 0) { + return false; + } + } + + return true; +} + +/** + * @param algorithm : "md5" | "sha1" | "sha256" | "sha512" + */ +async function fileHash(filename, algorithm) { + return new Promise((resolve, reject) => { + // Algorithm depends on availability of OpenSSL on platform + // Another algorithms: "sha1", "md5", "sha256", "sha512" ... + let shasum = crypto.createHash(algorithm); + try { + let s = fs.createReadStream(filename); + s.on("data", function (data) { + shasum.update(data) + }); + + s.on("error", function (error) { + reject(error); + }); + + // making digest + s.on("end", function () { + const hash = shasum.digest("hex") + return resolve(hash); + }); + } + catch (error) { + return reject("calc fail"); + } + }); +} + +function createLocalState(localFiles, args) { + console.log(`Creating local state at ${args["local-dir"]}${args["state-name"]}`); + fs.writeFileSync(`${args["local-dir"]}${args["state-name"]}`, JSON.stringify(localFiles, undefined, 4), { encoding: "utf8" }); + console.log("Local state created"); +} +async function getLocalFiles(args) { + const files = await readdir.async(args["local-dir"], { deep: true, stats: true, sep: "/", filter: (stat) => applyExcludeFilter(stat, args.exclude) }); + let records = []; + + for (let stat of files) { + if (stat.isDirectory()) { + records.push({ + type: "folder", + name: stat.path, + size: undefined + }); + + continue; + } + + if (stat.isFile()) { + records.push({ + type: "file", + name: stat.path, + size: stat.size, + hash: await fileHash(args["local-dir"] + stat.path, "sha256") + }); + + continue; + } + + if (stat.isSymbolicLink()) { + console.warn("This script is currently unable to handle symbolic links - please add a feature request if you need this"); + } + } + + return { + description: 'Don\'t delete', + version: '1.0.0', + generatedTime: new Date().getTime(), + data: records + }; +} + +try { + const filename = ".generate-file-hash.json"; + const _args = { + 'local-dir': './', + 'state-name': filename, + 'exclude': [ + '**/node_modules/**', + '**/.git*/**', + '**/.git*/**', + filename, + ] + }; + const localFiles = await getLocalFiles(_args); + createLocalState(localFiles, _args); + } catch (error) { + console.log(error); +} diff --git a/package-lock.json b/package-lock.json index d4fad40..9d14b94 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "basic-ftp": "^5.0.2", + "crypto": "^1.0.1", "lodash": "^4.17.21", "multimatch": "^5.0.0", "pretty-bytes": "^5.6.0", @@ -147,6 +148,12 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, + "node_modules/crypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", + "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==", + "deprecated": "This package is no longer supported. It's now a built-in Node module. If you've depended on crypto, you should switch to the one that's built-in." + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -438,6 +445,11 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, + "crypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", + "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==" + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", diff --git a/package.json b/package.json index 1b9a5a9..12e7ee5 100644 --- a/package.json +++ b/package.json @@ -8,13 +8,9 @@ }, "author": "Lei OT", "license": "MIT", + "type": "module", "dependencies": { - "basic-ftp": "^5.0.2", - "lodash": "^4.17.21", "multimatch": "^5.0.0", - "pretty-bytes": "^5.6.0", - "pretty-ms": "^7.0.1", - "readdir-enhanced": "^6.0.4", - "yargs": "^17.7.1" + "readdir-enhanced": "^6.0.4" } }