From 2246ca8b6685adcee470f460f7ca484ac52a10ed Mon Sep 17 00:00:00 2001 From: SmetDenis Date: Mon, 1 Apr 2024 21:59:02 +0400 Subject: [PATCH 1/6] Implement parallel processing functionality for CSV file validation In this commit, the parallel processing capability was added to improve performance when validating CSV files. The changes include updates to the workflow file and Dockerfile, adding support for a thread-safe version of PHP and the parallel extension. Also, a new dependency `hds-solutions/parallel-sdk` was included in the "composer.json" file, along with relevant changes in other files. --- .github/workflows/main.yml | 5 ++- .phan.php | 9 +++-- Dockerfile | 4 +-- composer.json | 13 +++++--- composer.lock | 67 +++++++++++++++++++++++++++++++++----- 5 files changed, 80 insertions(+), 18 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4dd7a50f..2ea9ce2e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,6 +45,9 @@ jobs: test-current-versions: name: Tests - Current runs-on: ubuntu-latest + strategy: + matrix: + ext-parallel: [ '', 'parallel' ] steps: - name: Checkout code uses: actions/checkout@v4 @@ -56,7 +59,7 @@ jobs: with: php-version: 8.3 coverage: xdebug - extensions: ast + extensions: ast, {{ matrix.ext-parallel }} - name: Build project run: make build --no-print-directory diff --git a/.phan.php b/.phan.php index bb249315..e767a30b 100644 --- a/.phan.php +++ b/.phan.php @@ -23,14 +23,17 @@ 'directory_list' => [ 'src', - 'vendor/jbzoo/ci-report-converter/src', - 'vendor/jbzoo/cli/src', 'vendor/jbzoo/data/src', + 'vendor/jbzoo/cli/src', 'vendor/jbzoo/utils/src', - 'vendor/league/csv/src', + 'vendor/jbzoo/ci-report-converter/src', + 'vendor/symfony/console', 'vendor/symfony/finder', + + 'vendor/league/csv/src', 'vendor/markrogoyski/math-php/src', 'vendor/respect/validation', + 'vendor/hds-solutions/parallel-sdk', ], ]); diff --git a/Dockerfile b/Dockerfile index e5cd65d4..2a32d95e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,11 +17,11 @@ COPY . /tmp RUN make build-version ######################################################################################## -FROM php:8.3-cli-alpine +FROM php:8.3-zts-alpine # Install PHP extensions ADD --chmod=0755 https://github.com/mlocati/docker-php-extension-installer/releases/latest/download/install-php-extensions /usr/local/bin/ -RUN install-php-extensions opcache @composer +RUN install-php-extensions opcache parallel @composer # Install application # run `make build-version` before! diff --git a/composer.json b/composer.json index 39305c66..f8a1b4d8 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { "name" : "jbzoo/csv-blueprint", - "type" : "project", + "type" : "project", "description" : "CLI Utility for Validating and Generating CSV Files Based on Custom Rules. It ensures your data meets specified criteria, streamlining data management and integrity checks.", "license" : "MIT", "keywords" : [ @@ -28,9 +28,7 @@ "require" : { "php" : "^8.1", - "ext-mbstring" : "*", - "league/csv" : "^9.15.0", "jbzoo/data" : "^7.1.1", "jbzoo/cli" : "^7.2.2", "jbzoo/utils" : "^7.2.1", @@ -39,11 +37,14 @@ "symfony/yaml" : ">=6.4.3", "symfony/filesystem" : ">=6.4.3", "symfony/finder" : ">=6.4.0", + "symfony/polyfill-mbstring" : "^1.29.0", + + "league/csv" : "^9.15.0", "markrogoyski/math-php" : "^2.9.0", "respect/validation" : "^2.3.6", "giggsey/libphonenumber-for-php-lite" : "^8.13.33", "giggsey/locale" : "^2.5", - "symfony/polyfill-mbstring" : "^1.29.0" + "hds-solutions/parallel-sdk" : "^2.1.4" }, "require-dev" : { @@ -53,6 +54,10 @@ "fakerphp/faker" : "^1.23.1" }, + "suggest" : { + "ext-parallel" : "Parallel processing of CSV files for better performance" + }, + "bin" : ["csv-blueprint"], "autoload" : { diff --git a/composer.lock b/composer.lock index 4009a556..5fa0db8a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "36f97cb9f275e6a0d29b9241ab3084ca", + "content-hash" : "77174ecf96d9315e44afe847234f52fa", "packages": [ { "name": "bluepsyduck/symfony-process-manager", @@ -199,6 +199,58 @@ }, "time": "2023-11-01T17:19:48+00:00" }, + { + "name" : "hds-solutions/parallel-sdk", + "version" : "v2.1.4", + "source" : { + "type" : "git", + "url" : "https://github.com/hschimpf/parallel-sdk.git", + "reference" : "91a38540d1b0c8723fb064781d25bfcf9ce5ce17" + }, + "dist" : { + "type" : "zip", + "url" : "https://api.github.com/repos/hschimpf/parallel-sdk/zipball/91a38540d1b0c8723fb064781d25bfcf9ce5ce17", + "reference" : "91a38540d1b0c8723fb064781d25bfcf9ce5ce17", + "shasum" : "" + }, + "require" : { + "php" : ">=8.0" + }, + "require-dev" : { + "phpunit/phpunit" : "^9.6", + "roave/security-advisories" : "dev-latest", + "symfony/console" : "^6.0" + }, + "suggest" : { + "ext-parallel" : "Allows to run multi-threaded processes", + "symfony/console" : "Allows usage of a shared ProgressBar between the Workers" + }, + "type" : "library", + "autoload" : { + "files" : [ + "src/helpers.php" + ], + "psr-4" : { + "HDSSolutions\\Console\\Parallel\\" : "src/" + } + }, + "notification-url" : "https://packagist.org/downloads/", + "license" : [ + "GPL-3.0" + ], + "authors" : [ + { + "name" : "Hermann D. Schimpf", + "email" : "hschimpf@hds-solutions.net" + } + ], + "description" : "SDK to implement parallel php extension", + "support" : { + "issues" : "https://github.com/hschimpf/parallel-sdk/issues", + "source" : "https://github.com/hschimpf/parallel-sdk/tree/v2.1.4" + }, + "time" : "2024-03-27T17:42:48+00:00" + }, { "name": "jbzoo/ci-report-converter", "version": "7.2.1", @@ -7362,16 +7414,16 @@ }, { "name": "squizlabs/php_codesniffer", - "version": "3.9.0", + "version" : "3.9.1", "source": { "type": "git", "url": "https://github.com/PHPCSStandards/PHP_CodeSniffer.git", - "reference": "d63cee4890a8afaf86a22e51ad4d97c91dd4579b" + "reference" : "267a4405fff1d9c847134db3a3c92f1ab7f77909" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/d63cee4890a8afaf86a22e51ad4d97c91dd4579b", - "reference": "d63cee4890a8afaf86a22e51ad4d97c91dd4579b", + "url" : "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/267a4405fff1d9c847134db3a3c92f1ab7f77909", + "reference" : "267a4405fff1d9c847134db3a3c92f1ab7f77909", "shasum": "" }, "require": { @@ -7438,7 +7490,7 @@ "type": "open_collective" } ], - "time": "2024-02-16T15:06:51+00:00" + "time" : "2024-03-31T21:03:09+00:00" }, { "name": "symfony/config", @@ -8537,8 +8589,7 @@ "prefer-stable": true, "prefer-lowest": false, "platform": { - "php": "^8.1", - "ext-mbstring": "*" + "php" : "^8.1" }, "platform-dev": [], "plugin-api-version": "2.6.0" From c7c0fb299e940a3e3ece71534aeaec651b177c87 Mon Sep 17 00:00:00 2001 From: SmetDenis Date: Mon, 1 Apr 2024 22:01:56 +0400 Subject: [PATCH 2/6] Correct PHP syntax in Github workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ea9ce2e..fccf657c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,7 +59,7 @@ jobs: with: php-version: 8.3 coverage: xdebug - extensions: ast, {{ matrix.ext-parallel }} + extensions: ast, ${{ matrix.ext-parallel }} - name: Build project run: make build --no-print-directory From b2ff9083b2bfbda79201f7af33267771731df5f3 Mon Sep 17 00:00:00 2001 From: SmetDenis Date: Mon, 1 Apr 2024 22:05:21 +0400 Subject: [PATCH 3/6] Correct PHP syntax in Github workflow --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fccf657c..97f20c8f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -60,6 +60,8 @@ jobs: php-version: 8.3 coverage: xdebug extensions: ast, ${{ matrix.ext-parallel }} + env: + phpts: zts - name: Build project run: make build --no-print-directory From 22f7e98fbf47d8ae576fa19083e693ec2c25275e Mon Sep 17 00:00:00 2001 From: SmetDenis Date: Mon, 1 Apr 2024 22:07:24 +0400 Subject: [PATCH 4/6] Correct PHP syntax in Github workflow --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 97f20c8f..e86ba44f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,7 +43,7 @@ env: jobs: test-current-versions: - name: Tests - Current + name: Tests runs-on: ubuntu-latest strategy: matrix: @@ -61,7 +61,7 @@ jobs: coverage: xdebug extensions: ast, ${{ matrix.ext-parallel }} env: - phpts: zts + phpts: ts - name: Build project run: make build --no-print-directory From 105dd63d2ad63c0ed151e7b78770f64cf0c19817 Mon Sep 17 00:00:00 2001 From: SmetDenis Date: Mon, 1 Apr 2024 22:10:32 +0400 Subject: [PATCH 5/6] Correct PHP syntax in Github workflow --- .github/workflows/main.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e86ba44f..23bd443b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,6 +56,7 @@ jobs: - name: Setup PHP uses: shivammathur/setup-php@v2 + id: setup-php with: php-version: 8.3 coverage: xdebug @@ -63,6 +64,18 @@ jobs: env: phpts: ts + - name: Print PHP version + run: echo ${{ steps.setup-php.outputs.php-version }} + + - name: Print PHP info + run: php -i + + - name: lscpu + run: lscpu + + - name: nproc + run: nproc + - name: Build project run: make build --no-print-directory From a47fb822fb960bb4d88cf4905fa8b4ee35df62e3 Mon Sep 17 00:00:00 2001 From: SmetDenis Date: Mon, 1 Apr 2024 22:10:54 +0400 Subject: [PATCH 6/6] Correct PHP syntax in Github workflow --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 23bd443b..e4a81900 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,6 +76,9 @@ jobs: - name: nproc run: nproc + - name: cpuinfo + run: cat /proc/cpuinfo + - name: Build project run: make build --no-print-directory