Merge pull request #75 from richardrigutins/chunks

Limit number of files that are processed at the same time

.github/workflows/test-workflow.yml

@@ -38,6 +38,7 @@ jobs:
           files: '**/*.txt'
           replacement-text: '1.0.2'
           exclude: '**/*.check.txt'
+          max-parallelism: '1'
 
       - name: Replace in Files - replace using special characters
         uses: ./
@@ -91,6 +92,26 @@ jobs:
           encoding: 'invalid'
           exclude: '**/*.check.txt'
 
+      - name: Replace in Files - error on invalid max-parallelism (invalid number)
+        uses: ./
+        continue-on-error: true
+        with:
+          search-text: 'foo'
+          files: '**/*.txt'
+          replacement-text: 'bar'
+          exclude: '**/*.check.txt'
+          max-parallelism: '-1'
+
+      - name: Replace in Files - error on invalid max-parallelism (invalid string)
+        uses: ./
+        continue-on-error: true
+        with:
+          search-text: 'foo'
+          files: '**/*.txt'
+          replacement-text: 'bar'
+          exclude: '**/*.check.txt'
+          max-parallelism: 'invalid'
+
       - name: Verify changes
         run: |
           echo "  > Actual - test1.txt"

README.md

@@ -28,6 +28,9 @@ It can be useful for automating repetitive tasks such as updating version number
 - `exclude`:
 (Optional) The files to be excluded from the search. It can be the path to a file or a glob pattern matching one or more files (e.g. `**/*.md`). Defaults to an empty string.
 
+- `max-parallelism`:
+(Optional) The maximum number of files that will be processed in parallel. This can be used to control the performance impact of the operation on the system. It should be a positive integer. Defaults to `10`.
+
 ## Example usage
 
 ```yaml
@@ -41,6 +44,7 @@ It can be useful for automating repetitive tasks such as updating version number
     replacement-text: 'world'
     exclude: 'node_modules/**'
     encoding: 'utf8'
+    max-parallelism: 10
 
 # Replace all the occurrences of '{0}' with '42' in the README.md file
 - name: Replace single file

__tests__/utils.test.ts

@@ -1,6 +1,51 @@
 import fs from 'fs';
 import path from 'path';
-import { getFiles, isValidEncoding, replaceTextInFile } from '../src/utils';
+import {
+  getFiles,
+  isPositiveInteger,
+  isValidEncoding,
+  processInChunks,
+  replaceTextInFile,
+} from '../src/utils';
+
+describe('isPositiveInteger', () => {
+  it('should return true for positive integers', () => {
+    expect(isPositiveInteger('1')).toBe(true);
+    expect(isPositiveInteger('123')).toBe(true);
+  });
+
+  it('should return false for zero', () => {
+    expect(isPositiveInteger('0')).toBe(false);
+  });
+
+  it('should return false for negative integers', () => {
+    expect(isPositiveInteger('-1')).toBe(false);
+  });
+
+  it('should return false for non-integer numbers', () => {
+    expect(isPositiveInteger('1.5')).toBe(false);
+  });
+
+  it('should return false for non-numeric strings', () => {
+    expect(isPositiveInteger('abc')).toBe(false);
+  });
+
+  it('should return false for empty strings', () => {
+    expect(isPositiveInteger('')).toBe(false);
+  });
+
+  it('should return false for whitespace strings', () => {
+    expect(isPositiveInteger(' ')).toBe(false);
+  });
+
+  it('should return false for null', () => {
+    expect(isPositiveInteger(null as any)).toBe(false);
+  });
+
+  it('should return false for undefined', () => {
+    expect(isPositiveInteger(undefined as any)).toBe(false);
+  });
+});
 
 describe('isValidEncoding', () => {
   it('should return true for valid encodings', () => {
@@ -29,6 +74,33 @@ describe('getFiles', () => {
   });
 });
 
+describe('processInChunks', () => {
+  it('should process an array in chunks', async () => {
+    const totalItems = 999;
+    const array = Array.from({ length: totalItems }, (_, i) => i);
+    const func = jest.fn(async (item: number) => {
+      item++;
+    });
+    const chunkSize = 100;
+
+    await processInChunks(array, func, chunkSize);
+
+    expect(func).toHaveBeenCalledTimes(totalItems);
+  });
+
+  it('should process an array with less items than chunk size', async () => {
+    const array = Array.from({ length: 50 }, (_, i) => i);
+    const func = jest.fn(async (item: number) => {
+      item++;
+    });
+    const chunkSize = 100;
+
+    await processInChunks(array, func, chunkSize);
+
+    expect(func).toHaveBeenCalledTimes(50);
+  });
+});
+
 describe('replaceTextInFile', () => {
   const testFilePath = path.join(__dirname, 'test-file.txt');
   const testFileContent = '{0}, world!';

action.yml

@@ -30,6 +30,12 @@ inputs:
       It can be the path to a file or a glob pattern (e.g. `**/*.md`).
     default: ""
     required: false
+  max-parallelism:
+    description: |-
+      (Optional) The maximum number of files to process in parallel.
+      Defaults to `10`.
+    default: "10"
+    required: false
 
 branding:
   icon: "edit"

dist/index.js

@@ -18,30 +18,42 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", ({ value: true }));
 const core_1 = __nccwpck_require__(2186);
 const utils_1 = __nccwpck_require__(918);
+// Entry point for the action
 function run() {
     return __awaiter(this, void 0, void 0, function* () {
         try {
+            // Get the input parameters
             const filesPattern = (0, core_1.getInput)('files');
             const searchText = (0, core_1.getInput)('search-text');
             const replaceText = (0, core_1.getInput)('replacement-text');
             const excludePattern = (0, core_1.getInput)('exclude');
             const inputEncoding = (0, core_1.getInput)('encoding');
+            const maxParallelism = (0, core_1.getInput)('max-parallelism');
+            // Validate the encoding
             if (!(0, utils_1.isValidEncoding)(inputEncoding)) {
                 throw new Error(`Invalid encoding: ${inputEncoding}`);
             }
+            // Validate that maxParallelism is a positive integer
+            if (!(0, utils_1.isPositiveInteger)(maxParallelism)) {
+                throw new Error(`Invalid max-parallelism: ${maxParallelism}`);
+            }
+            // Get the file paths that match the files pattern and do not match the exclude pattern
             const filePaths = yield (0, utils_1.getFiles)(filesPattern, excludePattern);
+            // If no file paths were found, log a warning and exit
             if (filePaths.length === 0) {
                 (0, core_1.warning)(`No files found for the given pattern.`);
                 return;
             }
             (0, core_1.info)(`Found ${filePaths.length} files for the given pattern.`);
             (0, core_1.info)(`Replacing "${searchText}" with "${replaceText}".`);
+            // Process the file paths in chunks, replacing the search text with the replace text in each file
+            // This is done to avoid opening too many files at once
             const encoding = inputEncoding;
-            const promises = filePaths.map((filePath) => __awaiter(this, void 0, void 0, function* () {
+            const chunkSize = parseInt(maxParallelism);
+            yield (0, utils_1.processInChunks)(filePaths, (filePath) => __awaiter(this, void 0, void 0, function* () {
                 (0, core_1.info)(`Replacing text in file ${filePath}`);
                 yield (0, utils_1.replaceTextInFile)(filePath, searchText, replaceText, encoding);
-            }));
+            }), chunkSize);
-            yield Promise.all(promises);
             (0, core_1.info)(`Done!`);
         }
         catch (err) {
@@ -79,7 +91,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", ({ value: true }));
-exports.replaceTextInFile = exports.getFiles = exports.isValidEncoding = void 0;
+exports.replaceTextInFile = exports.processInChunks = exports.getFiles = exports.isValidEncoding = exports.isPositiveInteger = void 0;
 const fs_1 = __importDefault(__nccwpck_require__(7147));
 const glob_1 = __nccwpck_require__(8211);
 const encodings = [
@@ -90,6 +102,16 @@ const encodings = [
     'base64',
     'latin1',
 ];
+/**
+ * Checks if a given string represents a positive integer.
+ *
+ * @param value - The string to check.
+ * @returns True if the string represents a positive integer, false otherwise.
+ */
+function isPositiveInteger(value) {
+    return /^[1-9]\d*$/.test(value);
+}
+exports.isPositiveInteger = isPositiveInteger;
 /**
  * Checks if the given encoding is supported.
  * @param encoding The encoding to check.
@@ -117,6 +139,27 @@ function getFiles(filesPattern, exclude) {
     });
 }
 exports.getFiles = getFiles;
+/**
+ * Processes an array in chunks, applying a given function to each item.
+ * @param array The array to process.
+ * @param func The function to apply to each item.
+ * @param chunkSize The number of items to process at a time.
+ * @returns A Promise that resolves when all items have been processed.
+ */
+function processInChunks(array, func, chunkSize) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // Split the array into chunks
+        const chunks = Array(Math.ceil(array.length / chunkSize))
+            .fill(0)
+            .map((_, index) => index * chunkSize)
+            .map(begin => array.slice(begin, begin + chunkSize));
+        // Process each chunk
+        for (const chunk of chunks) {
+            yield Promise.all(chunk.map(func));
+        }
+    });
+}
+exports.processInChunks = processInChunks;
 /**
  * Replaces all instances of the given text with the given value in the file.
  * @param filePath The path of the file to modify.

src/main.ts

@@ -2,23 +2,37 @@ import { debug, getInput, info, setFailed, warning } from '@actions/core';
 import {
   Encoding,
   getFiles,
+  isPositiveInteger,
   isValidEncoding,
+  processInChunks,
   replaceTextInFile,
 } from './utils';
 
+// Entry point for the action
 async function run(): Promise<void> {
   try {
+    // Get the input parameters
     const filesPattern = getInput('files');
     const searchText = getInput('search-text');
     const replaceText = getInput('replacement-text');
     const excludePattern = getInput('exclude');
     const inputEncoding = getInput('encoding');
+    const maxParallelism = getInput('max-parallelism');
 
+    // Validate the encoding
     if (!isValidEncoding(inputEncoding)) {
       throw new Error(`Invalid encoding: ${inputEncoding}`);
     }
 
+    // Validate that maxParallelism is a positive integer
+    if (!isPositiveInteger(maxParallelism)) {
+      throw new Error(`Invalid max-parallelism: ${maxParallelism}`);
+    }
+
+    // Get the file paths that match the files pattern and do not match the exclude pattern
     const filePaths = await getFiles(filesPattern, excludePattern);
+
+    // If no file paths were found, log a warning and exit
     if (filePaths.length === 0) {
       warning(`No files found for the given pattern.`);
       return;
@@ -27,15 +41,19 @@ async function run(): Promise<void> {
     info(`Found ${filePaths.length} files for the given pattern.`);
     info(`Replacing "${searchText}" with "${replaceText}".`);
 
+    // Process the file paths in chunks, replacing the search text with the replace text in each file
+    // This is done to avoid opening too many files at once
     const encoding = inputEncoding as Encoding;
-    const promises: Promise<void>[] = filePaths.map(
+    const chunkSize = parseInt(maxParallelism);
+    await processInChunks(
+      filePaths,
       async (filePath: string) => {
         info(`Replacing text in file ${filePath}`);
         await replaceTextInFile(filePath, searchText, replaceText, encoding);
       },
+      chunkSize,
     );
 
-    await Promise.all(promises);
     info(`Done!`);
   } catch (err) {
     if (err instanceof Error) {

src/utils.ts

@@ -12,6 +12,16 @@ const encodings = [
 
 export type Encoding = (typeof encodings)[number];
 
+/**
+ * Checks if a given string represents a positive integer.
+ *
+ * @param value - The string to check.
+ * @returns True if the string represents a positive integer, false otherwise.
+ */
+export function isPositiveInteger(value: string): boolean {
+  return /^[1-9]\d*$/.test(value);
+}
+
 /**
  * Checks if the given encoding is supported.
  * @param encoding The encoding to check.
@@ -39,6 +49,30 @@ export async function getFiles(
   }
 }
 
+/**
+ * Processes an array in chunks, applying a given function to each item.
+ * @param array The array to process.
+ * @param func The function to apply to each item.
+ * @param chunkSize The number of items to process at a time.
+ * @returns A Promise that resolves when all items have been processed.
+ */
+export async function processInChunks<T>(
+  array: T[],
+  func: (item: T) => Promise<void>,
+  chunkSize: number,
+): Promise<void> {
+  // Split the array into chunks
+  const chunks = Array(Math.ceil(array.length / chunkSize))
+    .fill(0)
+    .map((_, index) => index * chunkSize)
+    .map(begin => array.slice(begin, begin + chunkSize));
+
+  // Process each chunk
+  for (const chunk of chunks) {
+    await Promise.all(chunk.map(func));
+  }
+}
+
 /**
  * Replaces all instances of the given text with the given value in the file.
  * @param filePath The path of the file to modify.