index.js 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. import { stat, lstat, readdir, realpath } from 'node:fs/promises';
  2. import { Readable } from 'node:stream';
  3. import { resolve as presolve, relative as prelative, join as pjoin, sep as psep } from 'node:path';
  4. export const EntryTypes = {
  5. FILE_TYPE: 'files',
  6. DIR_TYPE: 'directories',
  7. FILE_DIR_TYPE: 'files_directories',
  8. EVERYTHING_TYPE: 'all',
  9. };
  10. const defaultOptions = {
  11. root: '.',
  12. fileFilter: (_entryInfo) => true,
  13. directoryFilter: (_entryInfo) => true,
  14. type: EntryTypes.FILE_TYPE,
  15. lstat: false,
  16. depth: 2147483648,
  17. alwaysStat: false,
  18. highWaterMark: 4096,
  19. };
  20. Object.freeze(defaultOptions);
  21. const RECURSIVE_ERROR_CODE = 'READDIRP_RECURSIVE_ERROR';
  22. const NORMAL_FLOW_ERRORS = new Set(['ENOENT', 'EPERM', 'EACCES', 'ELOOP', RECURSIVE_ERROR_CODE]);
  23. const ALL_TYPES = [
  24. EntryTypes.DIR_TYPE,
  25. EntryTypes.EVERYTHING_TYPE,
  26. EntryTypes.FILE_DIR_TYPE,
  27. EntryTypes.FILE_TYPE,
  28. ];
  29. const DIR_TYPES = new Set([
  30. EntryTypes.DIR_TYPE,
  31. EntryTypes.EVERYTHING_TYPE,
  32. EntryTypes.FILE_DIR_TYPE,
  33. ]);
  34. const FILE_TYPES = new Set([
  35. EntryTypes.EVERYTHING_TYPE,
  36. EntryTypes.FILE_DIR_TYPE,
  37. EntryTypes.FILE_TYPE,
  38. ]);
  39. const isNormalFlowError = (error) => NORMAL_FLOW_ERRORS.has(error.code);
  40. const wantBigintFsStats = process.platform === 'win32';
  41. const emptyFn = (_entryInfo) => true;
  42. const normalizeFilter = (filter) => {
  43. if (filter === undefined)
  44. return emptyFn;
  45. if (typeof filter === 'function')
  46. return filter;
  47. if (typeof filter === 'string') {
  48. const fl = filter.trim();
  49. return (entry) => entry.basename === fl;
  50. }
  51. if (Array.isArray(filter)) {
  52. const trItems = filter.map((item) => item.trim());
  53. return (entry) => trItems.some((f) => entry.basename === f);
  54. }
  55. return emptyFn;
  56. };
  57. /** Readable readdir stream, emitting new files as they're being listed. */
  58. export class ReaddirpStream extends Readable {
  59. constructor(options = {}) {
  60. super({
  61. objectMode: true,
  62. autoDestroy: true,
  63. highWaterMark: options.highWaterMark,
  64. });
  65. const opts = { ...defaultOptions, ...options };
  66. const { root, type } = opts;
  67. this._fileFilter = normalizeFilter(opts.fileFilter);
  68. this._directoryFilter = normalizeFilter(opts.directoryFilter);
  69. const statMethod = opts.lstat ? lstat : stat;
  70. // Use bigint stats if it's windows and stat() supports options (node 10+).
  71. if (wantBigintFsStats) {
  72. this._stat = (path) => statMethod(path, { bigint: true });
  73. }
  74. else {
  75. this._stat = statMethod;
  76. }
  77. this._maxDepth = opts.depth ?? defaultOptions.depth;
  78. this._wantsDir = type ? DIR_TYPES.has(type) : false;
  79. this._wantsFile = type ? FILE_TYPES.has(type) : false;
  80. this._wantsEverything = type === EntryTypes.EVERYTHING_TYPE;
  81. this._root = presolve(root);
  82. this._isDirent = !opts.alwaysStat;
  83. this._statsProp = this._isDirent ? 'dirent' : 'stats';
  84. this._rdOptions = { encoding: 'utf8', withFileTypes: this._isDirent };
  85. // Launch stream with one parent, the root dir.
  86. this.parents = [this._exploreDir(root, 1)];
  87. this.reading = false;
  88. this.parent = undefined;
  89. }
  90. async _read(batch) {
  91. if (this.reading)
  92. return;
  93. this.reading = true;
  94. try {
  95. while (!this.destroyed && batch > 0) {
  96. const par = this.parent;
  97. const fil = par && par.files;
  98. if (fil && fil.length > 0) {
  99. const { path, depth } = par;
  100. const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path));
  101. const awaited = await Promise.all(slice);
  102. for (const entry of awaited) {
  103. if (!entry)
  104. continue;
  105. if (this.destroyed)
  106. return;
  107. const entryType = await this._getEntryType(entry);
  108. if (entryType === 'directory' && this._directoryFilter(entry)) {
  109. if (depth <= this._maxDepth) {
  110. this.parents.push(this._exploreDir(entry.fullPath, depth + 1));
  111. }
  112. if (this._wantsDir) {
  113. this.push(entry);
  114. batch--;
  115. }
  116. }
  117. else if ((entryType === 'file' || this._includeAsFile(entry)) &&
  118. this._fileFilter(entry)) {
  119. if (this._wantsFile) {
  120. this.push(entry);
  121. batch--;
  122. }
  123. }
  124. }
  125. }
  126. else {
  127. const parent = this.parents.pop();
  128. if (!parent) {
  129. this.push(null);
  130. break;
  131. }
  132. this.parent = await parent;
  133. if (this.destroyed)
  134. return;
  135. }
  136. }
  137. }
  138. catch (error) {
  139. this.destroy(error);
  140. }
  141. finally {
  142. this.reading = false;
  143. }
  144. }
  145. async _exploreDir(path, depth) {
  146. let files;
  147. try {
  148. files = await readdir(path, this._rdOptions);
  149. }
  150. catch (error) {
  151. this._onError(error);
  152. }
  153. return { files, depth, path };
  154. }
  155. async _formatEntry(dirent, path) {
  156. let entry;
  157. const basename = this._isDirent ? dirent.name : dirent;
  158. try {
  159. const fullPath = presolve(pjoin(path, basename));
  160. entry = { path: prelative(this._root, fullPath), fullPath, basename };
  161. entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
  162. }
  163. catch (err) {
  164. this._onError(err);
  165. return;
  166. }
  167. return entry;
  168. }
  169. _onError(err) {
  170. if (isNormalFlowError(err) && !this.destroyed) {
  171. this.emit('warn', err);
  172. }
  173. else {
  174. this.destroy(err);
  175. }
  176. }
  177. async _getEntryType(entry) {
  178. // entry may be undefined, because a warning or an error were emitted
  179. // and the statsProp is undefined
  180. if (!entry && this._statsProp in entry) {
  181. return '';
  182. }
  183. const stats = entry[this._statsProp];
  184. if (stats.isFile())
  185. return 'file';
  186. if (stats.isDirectory())
  187. return 'directory';
  188. if (stats && stats.isSymbolicLink()) {
  189. const full = entry.fullPath;
  190. try {
  191. const entryRealPath = await realpath(full);
  192. const entryRealPathStats = await lstat(entryRealPath);
  193. if (entryRealPathStats.isFile()) {
  194. return 'file';
  195. }
  196. if (entryRealPathStats.isDirectory()) {
  197. const len = entryRealPath.length;
  198. if (full.startsWith(entryRealPath) && full.substr(len, 1) === psep) {
  199. const recursiveError = new Error(`Circular symlink detected: "${full}" points to "${entryRealPath}"`);
  200. // @ts-ignore
  201. recursiveError.code = RECURSIVE_ERROR_CODE;
  202. return this._onError(recursiveError);
  203. }
  204. return 'directory';
  205. }
  206. }
  207. catch (error) {
  208. this._onError(error);
  209. return '';
  210. }
  211. }
  212. }
  213. _includeAsFile(entry) {
  214. const stats = entry && entry[this._statsProp];
  215. return stats && this._wantsEverything && !stats.isDirectory();
  216. }
  217. }
  218. /**
  219. * Streaming version: Reads all files and directories in given root recursively.
  220. * Consumes ~constant small amount of RAM.
  221. * @param root Root directory
  222. * @param options Options to specify root (start directory), filters and recursion depth
  223. */
  224. export function readdirp(root, options = {}) {
  225. // @ts-ignore
  226. let type = options.entryType || options.type;
  227. if (type === 'both')
  228. type = EntryTypes.FILE_DIR_TYPE; // backwards-compatibility
  229. if (type)
  230. options.type = type;
  231. if (!root) {
  232. throw new Error('readdirp: root argument is required. Usage: readdirp(root, options)');
  233. }
  234. else if (typeof root !== 'string') {
  235. throw new TypeError('readdirp: root argument must be a string. Usage: readdirp(root, options)');
  236. }
  237. else if (type && !ALL_TYPES.includes(type)) {
  238. throw new Error(`readdirp: Invalid type passed. Use one of ${ALL_TYPES.join(', ')}`);
  239. }
  240. options.root = root;
  241. return new ReaddirpStream(options);
  242. }
  243. /**
  244. * Promise version: Reads all files and directories in given root recursively.
  245. * Compared to streaming version, will consume a lot of RAM e.g. when 1 million files are listed.
  246. * @returns array of paths and their entry infos
  247. */
  248. export function readdirpPromise(root, options = {}) {
  249. return new Promise((resolve, reject) => {
  250. const files = [];
  251. readdirp(root, options)
  252. .on('data', (entry) => files.push(entry))
  253. .on('end', () => resolve(files))
  254. .on('error', (error) => reject(error));
  255. });
  256. }
  257. export default readdirp;