const crypto = require('crypto');
|
const path = require('path');
|
|
const lodash = require('lodash');
|
|
const bulkFsTask = require('./util/bulk-fs-task');
|
const pluginCompat = require('./util/plugin-compat');
|
const promisify = require('./util/promisify');
|
const values = require('./util/Object.values');
|
const { parityCacheFromCache, pushParityWriteOps } = require('./util/parity');
|
|
class Md5Cache {
|
apply(compiler) {
|
let md5Cache = {};
|
let parityCache = {};
|
|
const fileMd5s = {};
|
const cachedMd5s = {};
|
let fileTimestamps = {};
|
const contextMd5s = {};
|
let contextTimestamps = {};
|
|
let md5CacheSerializer;
|
|
let latestStats = {};
|
let latestMd5s = {};
|
let unbuildMd5s = {};
|
|
let fileDependencies = [];
|
let contextDependencies = [];
|
|
let stat;
|
let readdir;
|
let readFile;
|
let mtime;
|
let md5;
|
let fileStamp;
|
let contextStamp;
|
let contextStamps;
|
|
function bindFS() {
|
stat = promisify(compiler.inputFileSystem.stat, {
|
context: compiler.inputFileSystem,
|
});
|
// stat = promisify(fs.stat, {context: fs});
|
|
readdir = promisify(compiler.inputFileSystem.readdir, {
|
context: compiler.inputFileSystem,
|
});
|
|
readFile = promisify(compiler.inputFileSystem.readFile, {
|
context: compiler.inputFileSystem,
|
});
|
|
mtime = file =>
|
stat(file)
|
.then(stat => +stat.mtime)
|
.catch(() => 0);
|
|
md5 = file =>
|
readFile(file)
|
.then(contents =>
|
crypto
|
.createHash('md5')
|
.update(contents, 'utf8')
|
.digest('hex'),
|
)
|
.catch(() => '');
|
|
fileStamp = (file, stats) => {
|
if (compiler.__hardSource_fileTimestamps[file]) {
|
return compiler.__hardSource_fileTimestamps[file];
|
} else {
|
if (!stats[file]) {
|
stats[file] = stat(file);
|
}
|
return stats[file].then(stat => {
|
const mtime = +stat.mtime;
|
compiler.__hardSource_fileTimestamps[file] = mtime;
|
return mtime;
|
});
|
}
|
};
|
|
contextStamp = (dir, stats) => {
|
const context = {};
|
|
let selfTime = 0;
|
|
function walk(dir) {
|
return readdir(dir)
|
.then(items =>
|
Promise.all(
|
items.map(item => {
|
const file = path.join(dir, item);
|
if (!stats[file]) {
|
stats[file] = stat(file);
|
}
|
return stats[file].then(
|
stat => {
|
if (stat.isDirectory()) {
|
return walk(path.join(dir, item)).then(items2 =>
|
items2.map(item2 => path.join(item, item2)),
|
);
|
}
|
if (+stat.mtime > selfTime) {
|
selfTime = +stat.mtime;
|
}
|
return item;
|
},
|
() => {
|
return;
|
},
|
);
|
}),
|
),
|
)
|
.catch(() => [])
|
.then(items =>
|
items
|
.reduce((carry, item) => carry.concat(item), [])
|
.filter(Boolean),
|
);
|
}
|
|
return walk(dir).then(items => {
|
items.sort();
|
const selfHash = crypto.createHash('md5');
|
items.forEach(item => {
|
selfHash.update(item);
|
});
|
context.mtime = selfTime;
|
context.hash = selfHash.digest('hex');
|
return context;
|
});
|
};
|
|
contextStamps = (contextDependencies, stats) => {
|
stats = stats || {};
|
const contexts = {};
|
contextDependencies.forEach(context => {
|
contexts[context] = { files: [], mtime: 0, hash: '' };
|
});
|
|
const compilerContextTs = compiler.contextTimestamps;
|
|
contextDependencies.forEach(contextPath => {
|
const _context = contextStamp(contextPath, stats);
|
if (!_context.then) {
|
contexts[contextPath] = _context;
|
} else {
|
contexts[contextPath] = _context.then(context => {
|
contexts[contextPath] = context;
|
return context;
|
});
|
}
|
});
|
return contexts;
|
};
|
}
|
|
if (compiler.inputFileSystem) {
|
bindFS();
|
} else {
|
pluginCompat.tap(
|
compiler,
|
'afterEnvironment',
|
'HardSource - Md5Cache',
|
bindFS,
|
);
|
}
|
|
pluginCompat.tap(
|
compiler,
|
'_hardSourceCreateSerializer',
|
'HardSource - Md5Cache',
|
(cacheSerializerFactory, cacheDirPath) => {
|
md5CacheSerializer = cacheSerializerFactory.create({
|
name: 'md5',
|
type: 'data',
|
autoParse: true,
|
cacheDirPath,
|
});
|
},
|
);
|
|
pluginCompat.tap(
|
compiler,
|
'_hardSourceResetCache',
|
'HardSource - Md5Cache',
|
() => {
|
md5Cache = {};
|
parityCache = {};
|
fileTimestamps = {};
|
contextTimestamps = {};
|
},
|
);
|
|
pluginCompat.tapPromise(
|
compiler,
|
'_hardSourceReadCache',
|
'HardSource - Md5Cache',
|
({ contextKeys, contextNormalPath }) =>
|
md5CacheSerializer
|
.read()
|
.then(_md5Cache => {
|
Object.keys(_md5Cache).forEach(key => {
|
if (key.startsWith('__hardSource_parityToken')) {
|
parityCache[key] = _md5Cache[key];
|
delete _md5Cache[key];
|
}
|
});
|
return _md5Cache;
|
})
|
.then(contextKeys(compiler, contextNormalPath))
|
.then(_md5Cache => {
|
Object.keys(_md5Cache).forEach(key => {
|
if (typeof _md5Cache[key] === 'string') {
|
_md5Cache[key] = JSON.parse(_md5Cache[key]);
|
}
|
|
if (_md5Cache[key] && _md5Cache[key].hash) {
|
cachedMd5s[key] = _md5Cache[key].hash;
|
}
|
});
|
md5Cache = _md5Cache;
|
})
|
.then(() => {
|
const dependencies = Object.keys(md5Cache);
|
fileDependencies = dependencies.filter(
|
file => md5Cache[file].isFile,
|
);
|
contextDependencies = dependencies.filter(
|
file => md5Cache[file].isDirectory,
|
);
|
}),
|
);
|
|
pluginCompat.tap(
|
compiler,
|
'_hardSourceParityCache',
|
'HardSource - Md5Cache',
|
parityRoot => {
|
parityCacheFromCache('Md5', parityRoot, parityCache);
|
},
|
);
|
|
pluginCompat.tapPromise(
|
compiler,
|
'_hardSourceVerifyCache',
|
'HardSource - Md5Cache',
|
() => {
|
latestStats = {};
|
latestMd5s = {};
|
unbuildMd5s = {};
|
|
const stats = {};
|
// var md5s = latestMd5s;
|
|
// Prepare objects to mark md5s to delete if they are not used.
|
for (const key in cachedMd5s) {
|
unbuildMd5s[key] = null;
|
}
|
|
return Promise.all([
|
(() => {
|
const compilerFileTs = (compiler.__hardSource_fileTimestamps = {});
|
const fileTs = (fileTimestamps = {});
|
|
return bulkFsTask(fileDependencies, (file, task) => {
|
if (compiler.__hardSource_fileTimestamps[file]) {
|
return compiler.__hardSource_fileTimestamps[file];
|
} else {
|
compiler.inputFileSystem.stat(
|
file,
|
task((err, value) => {
|
if (err) {
|
return 0;
|
}
|
|
const mtime = +value.mtime;
|
compiler.__hardSource_fileTimestamps[file] = mtime;
|
return mtime;
|
}),
|
);
|
}
|
}).then(mtimes => {
|
const bulk = lodash.zip(fileDependencies, mtimes);
|
return bulkFsTask(bulk, (item, task) => {
|
const file = item[0];
|
const mtime = item[1];
|
|
fileTs[file] = mtime || 0;
|
if (!compiler.__hardSource_fileTimestamps[file]) {
|
compiler.__hardSource_fileTimestamps[file] = mtime;
|
}
|
|
compiler.inputFileSystem.readFile(
|
file,
|
task(function(err, body) {
|
if (err) {
|
fileMd5s[file] = '';
|
return;
|
}
|
|
const hash = crypto
|
.createHash('md5')
|
.update(body, 'utf8')
|
.digest('hex');
|
|
fileMd5s[file] = hash;
|
}),
|
);
|
});
|
});
|
})(),
|
(() => {
|
compiler.contextTimestamps = compiler.contextTimestamps || {};
|
const contextTs = (contextTimestamps = {});
|
const contexts = contextStamps(contextDependencies, stats);
|
return Promise.all(values(contexts)).then(function() {
|
for (var contextPath in contexts) {
|
var context = contexts[contextPath];
|
|
if (!compiler.contextTimestamps[contextPath]) {
|
compiler.contextTimestamps[contextPath] = context.mtime;
|
}
|
contextTimestamps[contextPath] = context.mtime;
|
fileMd5s[contextPath] = context.hash;
|
}
|
});
|
})(),
|
]);
|
},
|
);
|
|
pluginCompat.tap(
|
compiler,
|
'compilation',
|
'HardSource - Md5Cache',
|
compilation => {
|
compilation.__hardSourceFileMd5s = fileMd5s;
|
compilation.__hardSourceCachedMd5s = cachedMd5s;
|
compilation.__hardSourceFileTimestamps = fileTimestamps;
|
},
|
);
|
|
pluginCompat.tapPromise(
|
compiler,
|
'_hardSourceWriteCache',
|
'HardSource - Md5Cache',
|
(compilation, { relateNormalPath, contextNormalPath }) => {
|
const moduleOps = [];
|
const dataOps = [];
|
const md5Ops = [];
|
const assetOps = [];
|
const moduleResolveOps = [];
|
const missingOps = [];
|
const resolverOps = [];
|
|
let buildingMd5s = {};
|
|
function buildMd5Ops(dependencies) {
|
dependencies.forEach(file => {
|
function updateMd5CacheItem(value) {
|
if (
|
!md5Cache[file] ||
|
(md5Cache[file] && md5Cache[file].hash !== value.hash)
|
) {
|
md5Cache[file] = value;
|
cachedMd5s[file] = value.hash;
|
|
md5Ops.push({
|
key: relateNormalPath(compiler, file),
|
value: value,
|
});
|
} else if (
|
!value.mtime &&
|
md5Cache[file] &&
|
md5Cache[file].mtime !== value.mtime
|
) {
|
md5Cache[file] = value;
|
cachedMd5s[file] = value.hash;
|
}
|
}
|
|
const building = buildingMd5s[file];
|
if (!building.then) {
|
updateMd5CacheItem(building);
|
} else {
|
buildingMd5s[file] = building.then(updateMd5CacheItem);
|
}
|
});
|
}
|
|
const fileDependencies = Array.from(compilation.fileDependencies).map(
|
file => contextNormalPath(compiler, file),
|
);
|
|
const MD5_TIME_PRECISION_BUFFER = 2000;
|
|
fileDependencies.forEach(file => {
|
if (buildingMd5s[file]) {
|
return;
|
}
|
delete unbuildMd5s[file];
|
|
if (fileMd5s[file]) {
|
buildingMd5s[file] = {
|
// Subtract a small buffer from now for file systems that record
|
// lower precision mtimes.
|
mtime: Date.now() - MD5_TIME_PRECISION_BUFFER,
|
hash: fileMd5s[file],
|
isFile: true,
|
isDirectory: false,
|
};
|
} else {
|
buildingMd5s[file] = md5(file).then(hash => ({
|
mtime: Date.now() - MD5_TIME_PRECISION_BUFFER,
|
hash,
|
isFile: true,
|
isDirectory: false,
|
}));
|
}
|
});
|
|
buildMd5Ops(fileDependencies);
|
|
const contextDependencies = Array.from(
|
compilation.contextDependencies,
|
).map(file => contextNormalPath(compiler, file));
|
|
const contexts = contextStamps(contextDependencies);
|
contextDependencies.forEach(file => {
|
if (buildingMd5s[file]) {
|
return;
|
}
|
delete unbuildMd5s[file];
|
|
let context = contexts[file];
|
if (!context.then) {
|
// Subtract a small buffer from now for file systems that record lower
|
// precision mtimes.
|
context.mtime = Date.now() - MD5_TIME_PRECISION_BUFFER;
|
context.isFile = false;
|
context.isDirectory = true;
|
} else {
|
context = context.then(context => {
|
context.mtime = Date.now() - MD5_TIME_PRECISION_BUFFER;
|
context.isFile = false;
|
context.isDirectory = true;
|
return context;
|
});
|
}
|
buildingMd5s[file] = context;
|
});
|
|
buildMd5Ops(contextDependencies);
|
|
const writeMd5Ops = Promise.all(
|
Object.keys(buildingMd5s).map(key => buildingMd5s[key]),
|
).then(() => {
|
if (!compilation.compiler.parentCompilation) {
|
for (const key in unbuildMd5s) {
|
md5Ops.push({
|
key: relateNormalPath(compiler, key),
|
value: unbuildMd5s[key],
|
});
|
}
|
}
|
|
pushParityWriteOps(compilation, md5Ops);
|
});
|
|
return writeMd5Ops.then(() => md5CacheSerializer.write(md5Ops));
|
},
|
);
|
}
|
}
|
|
module.exports = Md5Cache;
|
