Add processInChunks function to process files in chunks

__tests__/utils.test.ts

@@ -1,6 +1,11 @@
 import fs from 'fs';
 import path from 'path';
-import { getFiles, isValidEncoding, replaceTextInFile } from '../src/utils';
+import {
+  getFiles,
+  isValidEncoding,
+  processInChunks,
+  replaceTextInFile,
+} from '../src/utils';
 
 describe('isValidEncoding', () => {
   it('should return true for valid encodings', () => {
@@ -29,6 +34,33 @@ describe('getFiles', () => {
   });
 });
 
+describe('processInChunks', () => {
+  it('should process an array in chunks', async () => {
+    const totalItems = 999;
+    const array = Array.from({ length: totalItems }, (_, i) => i);
+    const func = jest.fn(async (item: number) => {
+      item++;
+    });
+    const chunkSize = 100;
+
+    await processInChunks(array, func, chunkSize);
+
+    expect(func).toHaveBeenCalledTimes(totalItems);
+  });
+
+  it('should process an array with less items than chunk size', async () => {
+    const array = Array.from({ length: 50 }, (_, i) => i);
+    const func = jest.fn(async (item: number) => {
+      item++;
+    });
+    const chunkSize = 100;
+
+    await processInChunks(array, func, chunkSize);
+
+    expect(func).toHaveBeenCalledTimes(50);
+  });
+});
+
 describe('replaceTextInFile', () => {
   const testFilePath = path.join(__dirname, 'test-file.txt');
   const testFileContent = '{0}, world!';

dist/index.js

@@ -18,18 +18,23 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", ({ value: true }));
 const core_1 = __nccwpck_require__(2186);
 const utils_1 = __nccwpck_require__(918);
+// Entry point for the action
 function run() {
     return __awaiter(this, void 0, void 0, function* () {
         try {
+            // Get the input parameters
             const filesPattern = (0, core_1.getInput)('files');
             const searchText = (0, core_1.getInput)('search-text');
             const replaceText = (0, core_1.getInput)('replacement-text');
             const excludePattern = (0, core_1.getInput)('exclude');
             const inputEncoding = (0, core_1.getInput)('encoding');
+            // Validate the encoding
             if (!(0, utils_1.isValidEncoding)(inputEncoding)) {
                 throw new Error(`Invalid encoding: ${inputEncoding}`);
             }
+            // Get the file paths that match the files pattern and do not match the exclude pattern
             const filePaths = yield (0, utils_1.getFiles)(filesPattern, excludePattern);
+            // If no file paths were found, log a warning and exit
             if (filePaths.length === 0) {
                 (0, core_1.warning)(`No files found for the given pattern.`);
                 return;
@@ -37,11 +42,13 @@ function run() {
             (0, core_1.info)(`Found ${filePaths.length} files for the given pattern.`);
             (0, core_1.info)(`Replacing "${searchText}" with "${replaceText}".`);
             const encoding = inputEncoding;
-            const promises = filePaths.map((filePath) => __awaiter(this, void 0, void 0, function* () {
+            // Process the file paths in chunks, replacing the search text with the replace text in each file
+            // This is done to avoid opening too many files at once
+            const chunkSize = 10;
+            yield (0, utils_1.processInChunks)(filePaths, (filePath) => __awaiter(this, void 0, void 0, function* () {
                 (0, core_1.info)(`Replacing text in file ${filePath}`);
                 yield (0, utils_1.replaceTextInFile)(filePath, searchText, replaceText, encoding);
-            }));
-            yield Promise.all(promises);
+            }), chunkSize);
             (0, core_1.info)(`Done!`);
         }
         catch (err) {
@@ -79,7 +86,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", ({ value: true }));
-exports.replaceTextInFile = exports.getFiles = exports.isValidEncoding = void 0;
+exports.replaceTextInFile = exports.processInChunks = exports.getFiles = exports.isValidEncoding = void 0;
 const fs_1 = __importDefault(__nccwpck_require__(7147));
 const glob_1 = __nccwpck_require__(8211);
 const encodings = [
@@ -117,6 +124,27 @@ function getFiles(filesPattern, exclude) {
     });
 }
 exports.getFiles = getFiles;
+/**
+ * Processes an array in chunks, applying a given function to each item.
+ * @param array The array to process.
+ * @param func The function to apply to each item.
+ * @param chunkSize The number of items to process at a time.
+ * @returns A Promise that resolves when all items have been processed.
+ */
+function processInChunks(array, func, chunkSize) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // Split the array into chunks
+        const chunks = Array(Math.ceil(array.length / chunkSize))
+            .fill(0)
+            .map((_, index) => index * chunkSize)
+            .map(begin => array.slice(begin, begin + chunkSize));
+        // Process each chunk
+        for (const chunk of chunks) {
+            yield Promise.all(chunk.map(func));
+        }
+    });
+}
+exports.processInChunks = processInChunks;
 /**
  * Replaces all instances of the given text with the given value in the file.
  * @param filePath The path of the file to modify.

src/main.ts

@@ -3,22 +3,29 @@ import {
   Encoding,
   getFiles,
   isValidEncoding,
+  processInChunks,
   replaceTextInFile,
 } from './utils';
 
+// Entry point for the action
 async function run(): Promise<void> {
   try {
+    // Get the input parameters
     const filesPattern = getInput('files');
     const searchText = getInput('search-text');
     const replaceText = getInput('replacement-text');
     const excludePattern = getInput('exclude');
     const inputEncoding = getInput('encoding');
 
+    // Validate the encoding
     if (!isValidEncoding(inputEncoding)) {
       throw new Error(`Invalid encoding: ${inputEncoding}`);
     }
 
+    // Get the file paths that match the files pattern and do not match the exclude pattern
     const filePaths = await getFiles(filesPattern, excludePattern);
+
+    // If no file paths were found, log a warning and exit
     if (filePaths.length === 0) {
       warning(`No files found for the given pattern.`);
       return;
@@ -28,14 +35,19 @@ async function run(): Promise<void> {
     info(`Replacing "${searchText}" with "${replaceText}".`);
 
     const encoding = inputEncoding as Encoding;
-    const promises: Promise<void>[] = filePaths.map(
+
+    // Process the file paths in chunks, replacing the search text with the replace text in each file
+    // This is done to avoid opening too many files at once
+    const chunkSize = 10;
+    await processInChunks(
+      filePaths,
       async (filePath: string) => {
         info(`Replacing text in file ${filePath}`);
         await replaceTextInFile(filePath, searchText, replaceText, encoding);
       },
+      chunkSize,
     );
 
-    await Promise.all(promises);
     info(`Done!`);
   } catch (err) {
     if (err instanceof Error) {

src/utils.ts

@@ -39,6 +39,30 @@ export async function getFiles(
   }
 }
 
+/**
+ * Processes an array in chunks, applying a given function to each item.
+ * @param array The array to process.
+ * @param func The function to apply to each item.
+ * @param chunkSize The number of items to process at a time.
+ * @returns A Promise that resolves when all items have been processed.
+ */
+export async function processInChunks<T>(
+  array: T[],
+  func: (item: T) => Promise<void>,
+  chunkSize: number,
+): Promise<void> {
+  // Split the array into chunks
+  const chunks = Array(Math.ceil(array.length / chunkSize))
+    .fill(0)
+    .map((_, index) => index * chunkSize)
+    .map(begin => array.slice(begin, begin + chunkSize));
+
+  // Process each chunk
+  for (const chunk of chunks) {
+    await Promise.all(chunk.map(func));
+  }
+}
+
 /**
  * Replaces all instances of the given text with the given value in the file.
  * @param filePath The path of the file to modify.