index.js 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", { value: true });
  3. exports.ReaddirpStream = exports.EntryTypes = void 0;
  4. exports.readdirp = readdirp;
  5. exports.readdirpPromise = readdirpPromise;
  6. const promises_1 = require("node:fs/promises");
  7. const node_stream_1 = require("node:stream");
  8. const node_path_1 = require("node:path");
  9. exports.EntryTypes = {
  10. FILE_TYPE: 'files',
  11. DIR_TYPE: 'directories',
  12. FILE_DIR_TYPE: 'files_directories',
  13. EVERYTHING_TYPE: 'all',
  14. };
  15. const defaultOptions = {
  16. root: '.',
  17. fileFilter: (_entryInfo) => true,
  18. directoryFilter: (_entryInfo) => true,
  19. type: exports.EntryTypes.FILE_TYPE,
  20. lstat: false,
  21. depth: 2147483648,
  22. alwaysStat: false,
  23. highWaterMark: 4096,
  24. };
  25. Object.freeze(defaultOptions);
  26. const RECURSIVE_ERROR_CODE = 'READDIRP_RECURSIVE_ERROR';
  27. const NORMAL_FLOW_ERRORS = new Set(['ENOENT', 'EPERM', 'EACCES', 'ELOOP', RECURSIVE_ERROR_CODE]);
  28. const ALL_TYPES = [
  29. exports.EntryTypes.DIR_TYPE,
  30. exports.EntryTypes.EVERYTHING_TYPE,
  31. exports.EntryTypes.FILE_DIR_TYPE,
  32. exports.EntryTypes.FILE_TYPE,
  33. ];
  34. const DIR_TYPES = new Set([
  35. exports.EntryTypes.DIR_TYPE,
  36. exports.EntryTypes.EVERYTHING_TYPE,
  37. exports.EntryTypes.FILE_DIR_TYPE,
  38. ]);
  39. const FILE_TYPES = new Set([
  40. exports.EntryTypes.EVERYTHING_TYPE,
  41. exports.EntryTypes.FILE_DIR_TYPE,
  42. exports.EntryTypes.FILE_TYPE,
  43. ]);
  44. const isNormalFlowError = (error) => NORMAL_FLOW_ERRORS.has(error.code);
  45. const wantBigintFsStats = process.platform === 'win32';
  46. const emptyFn = (_entryInfo) => true;
  47. const normalizeFilter = (filter) => {
  48. if (filter === undefined)
  49. return emptyFn;
  50. if (typeof filter === 'function')
  51. return filter;
  52. if (typeof filter === 'string') {
  53. const fl = filter.trim();
  54. return (entry) => entry.basename === fl;
  55. }
  56. if (Array.isArray(filter)) {
  57. const trItems = filter.map((item) => item.trim());
  58. return (entry) => trItems.some((f) => entry.basename === f);
  59. }
  60. return emptyFn;
  61. };
  62. /** Readable readdir stream, emitting new files as they're being listed. */
  63. class ReaddirpStream extends node_stream_1.Readable {
  64. constructor(options = {}) {
  65. super({
  66. objectMode: true,
  67. autoDestroy: true,
  68. highWaterMark: options.highWaterMark,
  69. });
  70. const opts = { ...defaultOptions, ...options };
  71. const { root, type } = opts;
  72. this._fileFilter = normalizeFilter(opts.fileFilter);
  73. this._directoryFilter = normalizeFilter(opts.directoryFilter);
  74. const statMethod = opts.lstat ? promises_1.lstat : promises_1.stat;
  75. // Use bigint stats if it's windows and stat() supports options (node 10+).
  76. if (wantBigintFsStats) {
  77. this._stat = (path) => statMethod(path, { bigint: true });
  78. }
  79. else {
  80. this._stat = statMethod;
  81. }
  82. this._maxDepth = opts.depth ?? defaultOptions.depth;
  83. this._wantsDir = type ? DIR_TYPES.has(type) : false;
  84. this._wantsFile = type ? FILE_TYPES.has(type) : false;
  85. this._wantsEverything = type === exports.EntryTypes.EVERYTHING_TYPE;
  86. this._root = (0, node_path_1.resolve)(root);
  87. this._isDirent = !opts.alwaysStat;
  88. this._statsProp = this._isDirent ? 'dirent' : 'stats';
  89. this._rdOptions = { encoding: 'utf8', withFileTypes: this._isDirent };
  90. // Launch stream with one parent, the root dir.
  91. this.parents = [this._exploreDir(root, 1)];
  92. this.reading = false;
  93. this.parent = undefined;
  94. }
  95. async _read(batch) {
  96. if (this.reading)
  97. return;
  98. this.reading = true;
  99. try {
  100. while (!this.destroyed && batch > 0) {
  101. const par = this.parent;
  102. const fil = par && par.files;
  103. if (fil && fil.length > 0) {
  104. const { path, depth } = par;
  105. const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path));
  106. const awaited = await Promise.all(slice);
  107. for (const entry of awaited) {
  108. if (!entry)
  109. continue;
  110. if (this.destroyed)
  111. return;
  112. const entryType = await this._getEntryType(entry);
  113. if (entryType === 'directory' && this._directoryFilter(entry)) {
  114. if (depth <= this._maxDepth) {
  115. this.parents.push(this._exploreDir(entry.fullPath, depth + 1));
  116. }
  117. if (this._wantsDir) {
  118. this.push(entry);
  119. batch--;
  120. }
  121. }
  122. else if ((entryType === 'file' || this._includeAsFile(entry)) &&
  123. this._fileFilter(entry)) {
  124. if (this._wantsFile) {
  125. this.push(entry);
  126. batch--;
  127. }
  128. }
  129. }
  130. }
  131. else {
  132. const parent = this.parents.pop();
  133. if (!parent) {
  134. this.push(null);
  135. break;
  136. }
  137. this.parent = await parent;
  138. if (this.destroyed)
  139. return;
  140. }
  141. }
  142. }
  143. catch (error) {
  144. this.destroy(error);
  145. }
  146. finally {
  147. this.reading = false;
  148. }
  149. }
  150. async _exploreDir(path, depth) {
  151. let files;
  152. try {
  153. files = await (0, promises_1.readdir)(path, this._rdOptions);
  154. }
  155. catch (error) {
  156. this._onError(error);
  157. }
  158. return { files, depth, path };
  159. }
  160. async _formatEntry(dirent, path) {
  161. let entry;
  162. const basename = this._isDirent ? dirent.name : dirent;
  163. try {
  164. const fullPath = (0, node_path_1.resolve)((0, node_path_1.join)(path, basename));
  165. entry = { path: (0, node_path_1.relative)(this._root, fullPath), fullPath, basename };
  166. entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
  167. }
  168. catch (err) {
  169. this._onError(err);
  170. return;
  171. }
  172. return entry;
  173. }
  174. _onError(err) {
  175. if (isNormalFlowError(err) && !this.destroyed) {
  176. this.emit('warn', err);
  177. }
  178. else {
  179. this.destroy(err);
  180. }
  181. }
  182. async _getEntryType(entry) {
  183. // entry may be undefined, because a warning or an error were emitted
  184. // and the statsProp is undefined
  185. if (!entry && this._statsProp in entry) {
  186. return '';
  187. }
  188. const stats = entry[this._statsProp];
  189. if (stats.isFile())
  190. return 'file';
  191. if (stats.isDirectory())
  192. return 'directory';
  193. if (stats && stats.isSymbolicLink()) {
  194. const full = entry.fullPath;
  195. try {
  196. const entryRealPath = await (0, promises_1.realpath)(full);
  197. const entryRealPathStats = await (0, promises_1.lstat)(entryRealPath);
  198. if (entryRealPathStats.isFile()) {
  199. return 'file';
  200. }
  201. if (entryRealPathStats.isDirectory()) {
  202. const len = entryRealPath.length;
  203. if (full.startsWith(entryRealPath) && full.substr(len, 1) === node_path_1.sep) {
  204. const recursiveError = new Error(`Circular symlink detected: "${full}" points to "${entryRealPath}"`);
  205. // @ts-ignore
  206. recursiveError.code = RECURSIVE_ERROR_CODE;
  207. return this._onError(recursiveError);
  208. }
  209. return 'directory';
  210. }
  211. }
  212. catch (error) {
  213. this._onError(error);
  214. return '';
  215. }
  216. }
  217. }
  218. _includeAsFile(entry) {
  219. const stats = entry && entry[this._statsProp];
  220. return stats && this._wantsEverything && !stats.isDirectory();
  221. }
  222. }
  223. exports.ReaddirpStream = ReaddirpStream;
  224. /**
  225. * Streaming version: Reads all files and directories in given root recursively.
  226. * Consumes ~constant small amount of RAM.
  227. * @param root Root directory
  228. * @param options Options to specify root (start directory), filters and recursion depth
  229. */
  230. function readdirp(root, options = {}) {
  231. // @ts-ignore
  232. let type = options.entryType || options.type;
  233. if (type === 'both')
  234. type = exports.EntryTypes.FILE_DIR_TYPE; // backwards-compatibility
  235. if (type)
  236. options.type = type;
  237. if (!root) {
  238. throw new Error('readdirp: root argument is required. Usage: readdirp(root, options)');
  239. }
  240. else if (typeof root !== 'string') {
  241. throw new TypeError('readdirp: root argument must be a string. Usage: readdirp(root, options)');
  242. }
  243. else if (type && !ALL_TYPES.includes(type)) {
  244. throw new Error(`readdirp: Invalid type passed. Use one of ${ALL_TYPES.join(', ')}`);
  245. }
  246. options.root = root;
  247. return new ReaddirpStream(options);
  248. }
  249. /**
  250. * Promise version: Reads all files and directories in given root recursively.
  251. * Compared to streaming version, will consume a lot of RAM e.g. when 1 million files are listed.
  252. * @returns array of paths and their entry infos
  253. */
  254. function readdirpPromise(root, options = {}) {
  255. return new Promise((resolve, reject) => {
  256. const files = [];
  257. readdirp(root, options)
  258. .on('data', (entry) => files.push(entry))
  259. .on('end', () => resolve(files))
  260. .on('error', (error) => reject(error));
  261. });
  262. }
  263. exports.default = readdirp;