From f8f7e062c08d4d41aa768db362f8666cbc6d20db Mon Sep 17 00:00:00 2001 From: Tomasz Surowiec Date: Fri, 19 Oct 2018 16:58:18 +0200 Subject: [PATCH] remove rabbitmq, queue in redis lists, add keys to message instead of limit/offset quries --- .env.dist | 5 - Makefile | 2 +- README.md | 13 +- behat.yml.dist | 4 +- composer.json | 1 - composer.lock | 416 ++++++------------ config/bundles.php | 1 - config/packages/doctrine.yaml | 2 + config/packages/old_sound_rabbit_mq.yaml | 15 - config/packages/test/old_sound_rabbit_mq.yaml | 15 - config/services.yaml | 6 +- config/services_test.yaml | 8 - docker-compose.yml | 13 +- features/Run/Data/copyUTF8.feature | 37 ++ ...unterContext.php => ChunkCacheContext.php} | 22 +- features/bootstrap/CommandContext.php | 24 +- src/Command/ConsumerCommand.php | 66 +++ src/Command/FinishCommand.php | 29 +- src/Command/RunCommand.php | 29 +- src/Fogger/Data/ChunkCache.php | 75 ++++ src/Fogger/Data/ChunkConsumer.php | 24 +- src/Fogger/Data/ChunkCounter.php | 44 -- src/Fogger/Data/ChunkDivider.php | 55 --- src/Fogger/Data/ChunkMessage.php | 32 +- src/Fogger/Data/ChunkProducer.php | 54 +-- src/Fogger/Data/ChunkReader.php | 21 +- src/Fogger/Data/DataCopier.php | 5 +- src/Fogger/Data/SourceQuery.php | 61 +++ src/Fogger/Data/TableQuery.php | 32 -- src/Fogger/Mask/AbstractCachedMask.php | 9 +- src/Fogger/Mask/FakerMask.php | 8 +- src/Fogger/Recipe/RecipeFactory.php | 10 +- src/Fogger/Recipe/RecipeTableFactory.php | 9 +- src/Fogger/Refine/Refiner.php | 21 +- src/Fogger/Schema/SchemaManipulator.php | 13 + .../Subset/AbstratctHeadOrTailSubset.php | 30 +- src/Fogger/Subset/HeadSubset.php | 7 +- src/Fogger/Subset/NoSubset.php | 3 +- src/Fogger/Subset/RangeSubset.php | 5 +- src/Fogger/Subset/SubsetStrategyInterface.php | 2 +- src/Fogger/Subset/TailSubset.php | 7 +- symfony.lock | 12 - 42 files changed, 570 insertions(+), 677 deletions(-) delete mode 100644 config/packages/old_sound_rabbit_mq.yaml delete mode 100644 config/packages/test/old_sound_rabbit_mq.yaml delete mode 100644 config/services_test.yaml create mode 100644 features/Run/Data/copyUTF8.feature rename features/bootstrap/{ChunkCounterContext.php => ChunkCacheContext.php} (63%) create mode 100644 src/Command/ConsumerCommand.php create mode 100644 src/Fogger/Data/ChunkCache.php delete mode 100644 src/Fogger/Data/ChunkCounter.php delete mode 100644 src/Fogger/Data/ChunkDivider.php create mode 100644 src/Fogger/Data/SourceQuery.php delete mode 100644 src/Fogger/Data/TableQuery.php diff --git a/.env.dist b/.env.dist index a38ee37..7730f4d 100644 --- a/.env.dist +++ b/.env.dist @@ -8,11 +8,6 @@ APP_SECRET=1b90fda83888e1852b735fdf9d37cf40 SOURCE_DATABASE_URL=mysql://user:pass@source/source TARGET_DATABASE_URL=mysql://user:pass@target/target ###< doctrine/doctrine-bundle ### - -###> php-amqplib/rabbitmq-bundle ### -RABBITMQ_URL=amqp://user:pass@rabbit:5672 -###< php-amqplib/rabbitmq-bundle ### - ###> snc/redis-bundle ### # passwords that contain special characters (@, %, :, +) must be urlencoded REDIS_URL=redis://redis diff --git a/Makefile b/Makefile index 28d34b5..48939a4 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ install: start: ${DOCKER_COMPOSE} up -d echo "waiting for services to start..." - sleep 30 + sleep 16 .PHONY: stop stop: diff --git a/README.md b/README.md index dd08621..f5d0530 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ You can configure various masking and subsetting strategies, and when what *fogg ## How to use the docker image -*Fogger* requires docker environment, redis and rabbitMq services and two databases: source and target. You can set up this stack using for example this docker-compose file: +*Fogger* requires docker environment, redis for caching and two databases: source and target. You can set up this stack using for example this docker-compose file: ``` version: '2.0' services: @@ -21,24 +21,17 @@ services: environment: SOURCE_DATABASE_URL: mysql://user:pass@source:3306/source TARGET_DATABASE_URL: mysql://user:pass@target:3306/target - RABBITMQ_URL: amqp://user:pass@rabbit:5672 REDIS_URL: redis://redis worker: image: tshio/fogger:latest environment: SOURCE_DATABASE_URL: mysql://user:pass@source:3306/source TARGET_DATABASE_URL: mysql://user:pass@target:3306/target - RABBITMQ_URL: amqp://user:pass@rabbit:5672 REDIS_URL: redis://redis restart: always - command: rabbit:consumer --messages=200 fogger_data_chunks + command: fogger:consumer --messages=200 redis: image: redis:4 - rabbit: - image: rabbitmq:3 - environment: - RABBITMQ_DEFAULT_USER: user - RABBITMQ_DEFAULT_PASS: pass source: volumes: - ./dump.sql:/docker-entrypoint-initdb.d/dump.sql @@ -109,7 +102,7 @@ For the clarity and readability of the config files, all the tables that will no * faker - will use a marvelous [faker](https://github.com/fzaninotto/Faker) library. Pass the `method` of faker that you want to use here as an option. `email: { maskStrategy: "faker", options: { method: "safeEmail" }` - `date: { maskStrategy: "faker", options: { method: "date", parameters: ["Y::m::d", "2017-12-31 23:59:59"] }` + `date: { maskStrategy: "faker", options: { method: "date", arguments: ["Y::m::d", "2017-12-31 23:59:59"] }` #### Subsetting data diff --git a/behat.yml.dist b/behat.yml.dist index decbb69..3f36a83 100644 --- a/behat.yml.dist +++ b/behat.yml.dist @@ -10,8 +10,8 @@ default: - DatabaseContext: source: '@doctrine.dbal.source_connection' target: '@doctrine.dbal.target_connection' - - ChunkCounterContext: - chunkCounter: '@App\Fogger\Data\ChunkCounter' + - ChunkCacheContext: + chunkCache: '@App\Fogger\Data\ChunkCache' extensions: Behat\Symfony2Extension: diff --git a/composer.json b/composer.json index 22b0760..4ecd451 100644 --- a/composer.json +++ b/composer.json @@ -7,7 +7,6 @@ "ext-iconv": "*", "ext-json": "*", "fzaninotto/faker": "^1.8", - "php-amqplib/rabbitmq-bundle": "^1.14", "predis/predis": "^1.1", "snc/redis-bundle": "^2.1", "symfony/console": "*", diff --git a/composer.lock b/composer.lock index 63c7cd6..38de858 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b050f3bb1665cba364200b622d1331ae", + "content-hash": "d927a0e197823089d44403615a6bd5cf", "packages": [ { "name": "doctrine/annotations", @@ -1321,33 +1321,34 @@ }, { "name": "ocramius/proxy-manager", - "version": "2.1.1", + "version": "2.2.2", "source": { "type": "git", "url": "https://github.com/Ocramius/ProxyManager.git", - "reference": "e18ac876b2e4819c76349de8f78ccc8ef1554cd7" + "reference": "14b137b06b0f911944132df9d51e445a35920ab1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Ocramius/ProxyManager/zipball/e18ac876b2e4819c76349de8f78ccc8ef1554cd7", - "reference": "e18ac876b2e4819c76349de8f78ccc8ef1554cd7", + "url": "https://api.github.com/repos/Ocramius/ProxyManager/zipball/14b137b06b0f911944132df9d51e445a35920ab1", + "reference": "14b137b06b0f911944132df9d51e445a35920ab1", "shasum": "" }, "require": { - "ocramius/package-versions": "^1.1.1", - "php": "^7.1.0", - "zendframework/zend-code": "^3.1.0" + "ocramius/package-versions": "^1.1.3", + "php": "^7.2.0", + "zendframework/zend-code": "^3.3.0" }, "require-dev": { - "couscous/couscous": "^1.5.2", + "couscous/couscous": "^1.6.1", "ext-phar": "*", - "humbug/humbug": "dev-master@DEV", - "nikic/php-parser": "^3.0.4", + "humbug/humbug": "1.0.0-RC.0@RC", + "nikic/php-parser": "^3.1.1", + "padraic/phpunit-accelerator": "dev-master@DEV", "phpbench/phpbench": "^0.12.2", - "phpstan/phpstan": "^0.6.4", - "phpunit/phpunit": "^5.6.4", - "phpunit/phpunit-mock-objects": "^3.4.1", - "squizlabs/php_codesniffer": "^2.7.0" + "phpstan/phpstan": "dev-master#856eb10a81c1d27c701a83f167dc870fd8f4236a as 0.9.999", + "phpstan/phpstan-phpunit": "dev-master#5629c0a1f4a9c417cb1077cf6693ad9753895761", + "phpunit/phpunit": "^6.4.3", + "squizlabs/php_codesniffer": "^2.9.1" }, "suggest": { "ocramius/generated-hydrator": "To have very fast object to array to object conversion for ghost objects", @@ -1386,149 +1387,7 @@ "proxy pattern", "service proxies" ], - "time": "2017-05-04T11:12:50+00:00" - }, - { - "name": "php-amqplib/php-amqplib", - "version": "v2.7.2", - "source": { - "type": "git", - "url": "https://github.com/php-amqplib/php-amqplib.git", - "reference": "dfd3694a86f1a7394d3693485259d4074a6ec79b" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/php-amqplib/php-amqplib/zipball/dfd3694a86f1a7394d3693485259d4074a6ec79b", - "reference": "dfd3694a86f1a7394d3693485259d4074a6ec79b", - "shasum": "" - }, - "require": { - "ext-bcmath": "*", - "ext-mbstring": "*", - "php": ">=5.3.0" - }, - "replace": { - "videlalvaro/php-amqplib": "self.version" - }, - "require-dev": { - "phpdocumentor/phpdocumentor": "^2.9", - "phpunit/phpunit": "^4.8", - "scrutinizer/ocular": "^1.1", - "squizlabs/php_codesniffer": "^2.5" - }, - "suggest": { - "ext-sockets": "Use AMQPSocketConnection" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.7-dev" - } - }, - "autoload": { - "psr-4": { - "PhpAmqpLib\\": "PhpAmqpLib/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "LGPL-2.1-or-later" - ], - "authors": [ - { - "name": "Alvaro Videla", - "role": "Original Maintainer" - }, - { - "name": "John Kelly", - "email": "johnmkelly86@gmail.com", - "role": "Maintainer" - }, - { - "name": "Raúl Araya", - "email": "nubeiro@gmail.com", - "role": "Maintainer" - } - ], - "description": "Formerly videlalvaro/php-amqplib. This library is a pure PHP implementation of the AMQP protocol. It's been tested against RabbitMQ.", - "homepage": "https://github.com/php-amqplib/php-amqplib/", - "keywords": [ - "message", - "queue", - "rabbitmq" - ], - "time": "2018-02-11T19:28:00+00:00" - }, - { - "name": "php-amqplib/rabbitmq-bundle", - "version": "v1.14.4", - "source": { - "type": "git", - "url": "https://github.com/php-amqplib/RabbitMqBundle.git", - "reference": "cf67adaa4e306d8e9cb6855a72d79263b425ded8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/php-amqplib/RabbitMqBundle/zipball/cf67adaa4e306d8e9cb6855a72d79263b425ded8", - "reference": "cf67adaa4e306d8e9cb6855a72d79263b425ded8", - "shasum": "" - }, - "require": { - "php": "^5.3.9|^7.0", - "php-amqplib/php-amqplib": "^2.6", - "psr/log": "^1.0", - "symfony/config": "^2.7|^3.0|^4.0", - "symfony/console": "^2.7|^3.0|^4.0", - "symfony/dependency-injection": "^2.7|^3.0|^4.0", - "symfony/event-dispatcher": "^2.7|^3.0|^4.0", - "symfony/yaml": "^2.7|^3.0|^4.0" - }, - "replace": { - "oldsound/rabbitmq-bundle": "self.version" - }, - "require-dev": { - "phpunit/phpunit": "^4.8.35|^5.4.3", - "symfony/debug": "^2.7|^3.0|^4.0", - "symfony/serializer": "^2.7|^3.0|^4.0" - }, - "suggest": { - "symfony/framework-bundle": "To use this lib as a full Symfony Bundle and to use the profiler data collector" - }, - "type": "symfony-bundle", - "extra": { - "branch-alias": { - "dev-master": "1.10.x-dev" - } - }, - "autoload": { - "psr-4": { - "OldSound\\RabbitMqBundle\\": "" - }, - "exclude-from-classmap": [ - "/Tests/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Alvaro Videla" - } - ], - "description": "Integrates php-amqplib with Symfony & RabbitMq. Formerly oldsound/rabbitmq-bundle.", - "keywords": [ - "AMQP", - "Symfony2", - "message", - "queue", - "rabbitmq", - "symfony", - "symfony3", - "symfony4" - ], - "time": "2018-05-02T13:12:32+00:00" + "time": "2018-09-27T13:45:01+00:00" }, { "name": "phpdocumentor/reflection-common", @@ -1924,16 +1783,16 @@ }, { "name": "snc/redis-bundle", - "version": "2.1.6", + "version": "2.1.7", "source": { "type": "git", "url": "https://github.com/snc/SncRedisBundle.git", - "reference": "fd24b115b35d54ec2f0f356e36e793933ee57bb8" + "reference": "13dee3562945bbdbfc02286a476837f9f3ce9cd1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/snc/SncRedisBundle/zipball/fd24b115b35d54ec2f0f356e36e793933ee57bb8", - "reference": "fd24b115b35d54ec2f0f356e36e793933ee57bb8", + "url": "https://api.github.com/repos/snc/SncRedisBundle/zipball/13dee3562945bbdbfc02286a476837f9f3ce9cd1", + "reference": "13dee3562945bbdbfc02286a476837f9f3ce9cd1", "shasum": "" }, "require": { @@ -1986,20 +1845,20 @@ "redis", "symfony" ], - "time": "2018-07-31T08:05:28+00:00" + "time": "2018-10-15T12:07:05+00:00" }, { "name": "symfony/cache", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/cache.git", - "reference": "b8440ff4635c6631aca21a09ab72e0b7e3a6cb7a" + "reference": "05ce0ddc8bc1ffe592105398fc2c725cb3080a38" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/cache/zipball/b8440ff4635c6631aca21a09ab72e0b7e3a6cb7a", - "reference": "b8440ff4635c6631aca21a09ab72e0b7e3a6cb7a", + "url": "https://api.github.com/repos/symfony/cache/zipball/05ce0ddc8bc1ffe592105398fc2c725cb3080a38", + "reference": "05ce0ddc8bc1ffe592105398fc2c725cb3080a38", "shasum": "" }, "require": { @@ -2055,20 +1914,20 @@ "caching", "psr6" ], - "time": "2018-08-27T09:36:56+00:00" + "time": "2018-09-30T03:38:13+00:00" }, { "name": "symfony/config", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/config.git", - "reference": "76015a3cc372b14d00040ff58e18e29f69eba717" + "reference": "991fec8bbe77367fc8b48ecbaa8a4bd6e905a238" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/config/zipball/76015a3cc372b14d00040ff58e18e29f69eba717", - "reference": "76015a3cc372b14d00040ff58e18e29f69eba717", + "url": "https://api.github.com/repos/symfony/config/zipball/991fec8bbe77367fc8b48ecbaa8a4bd6e905a238", + "reference": "991fec8bbe77367fc8b48ecbaa8a4bd6e905a238", "shasum": "" }, "require": { @@ -2118,20 +1977,20 @@ ], "description": "Symfony Config Component", "homepage": "https://symfony.com", - "time": "2018-08-08T06:37:38+00:00" + "time": "2018-10-31T09:09:42+00:00" }, { "name": "symfony/console", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "ca80b8ced97cf07390078b29773dc384c39eee1f" + "reference": "432122af37d8cd52fba1b294b11976e0d20df595" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/ca80b8ced97cf07390078b29773dc384c39eee1f", - "reference": "ca80b8ced97cf07390078b29773dc384c39eee1f", + "url": "https://api.github.com/repos/symfony/console/zipball/432122af37d8cd52fba1b294b11976e0d20df595", + "reference": "432122af37d8cd52fba1b294b11976e0d20df595", "shasum": "" }, "require": { @@ -2186,20 +2045,20 @@ ], "description": "Symfony Console Component", "homepage": "https://symfony.com", - "time": "2018-07-26T11:24:31+00:00" + "time": "2018-10-31T09:30:44+00:00" }, { "name": "symfony/debug", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/debug.git", - "reference": "47ead688f1f2877f3f14219670f52e4722ee7052" + "reference": "19090917b848a799cbae4800abf740fe4eb71c1d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/debug/zipball/47ead688f1f2877f3f14219670f52e4722ee7052", - "reference": "47ead688f1f2877f3f14219670f52e4722ee7052", + "url": "https://api.github.com/repos/symfony/debug/zipball/19090917b848a799cbae4800abf740fe4eb71c1d", + "reference": "19090917b848a799cbae4800abf740fe4eb71c1d", "shasum": "" }, "require": { @@ -2242,20 +2101,20 @@ ], "description": "Symfony Debug Component", "homepage": "https://symfony.com", - "time": "2018-08-03T11:13:38+00:00" + "time": "2018-10-31T09:09:42+00:00" }, { "name": "symfony/dependency-injection", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/dependency-injection.git", - "reference": "bae4983003c9d451e278504d7d9b9d7fc1846873" + "reference": "e72ee2c23d952e4c368ee98610fa22b79b89b483" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/dependency-injection/zipball/bae4983003c9d451e278504d7d9b9d7fc1846873", - "reference": "bae4983003c9d451e278504d7d9b9d7fc1846873", + "url": "https://api.github.com/repos/symfony/dependency-injection/zipball/e72ee2c23d952e4c368ee98610fa22b79b89b483", + "reference": "e72ee2c23d952e4c368ee98610fa22b79b89b483", "shasum": "" }, "require": { @@ -2313,20 +2172,20 @@ ], "description": "Symfony DependencyInjection Component", "homepage": "https://symfony.com", - "time": "2018-08-08T11:48:58+00:00" + "time": "2018-10-31T10:54:16+00:00" }, { "name": "symfony/doctrine-bridge", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/doctrine-bridge.git", - "reference": "58e331b3f6bbbd0beeb41cc924455bf85f660f50" + "reference": "ca2726213685a5d0f21d9a4d8b694c7a1226450b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/doctrine-bridge/zipball/58e331b3f6bbbd0beeb41cc924455bf85f660f50", - "reference": "58e331b3f6bbbd0beeb41cc924455bf85f660f50", + "url": "https://api.github.com/repos/symfony/doctrine-bridge/zipball/ca2726213685a5d0f21d9a4d8b694c7a1226450b", + "reference": "ca2726213685a5d0f21d9a4d8b694c7a1226450b", "shasum": "" }, "require": { @@ -2393,20 +2252,20 @@ ], "description": "Symfony Doctrine Bridge", "homepage": "https://symfony.com", - "time": "2018-08-24T10:22:26+00:00" + "time": "2018-10-30T17:00:46+00:00" }, { "name": "symfony/event-dispatcher", - "version": "v4.1.6", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/event-dispatcher.git", - "reference": "bfb30c2ad377615a463ebbc875eba64a99f6aa3e" + "reference": "552541dad078c85d9414b09c041ede488b456cd5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/bfb30c2ad377615a463ebbc875eba64a99f6aa3e", - "reference": "bfb30c2ad377615a463ebbc875eba64a99f6aa3e", + "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/552541dad078c85d9414b09c041ede488b456cd5", + "reference": "552541dad078c85d9414b09c041ede488b456cd5", "shasum": "" }, "require": { @@ -2456,20 +2315,20 @@ ], "description": "Symfony EventDispatcher Component", "homepage": "https://symfony.com", - "time": "2018-07-26T09:10:45+00:00" + "time": "2018-10-10T13:52:42+00:00" }, { "name": "symfony/filesystem", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "c0f5f62db218fa72195b8b8700e4b9b9cf52eb5e" + "reference": "fd7bd6535beb1f0a0a9e3ee960666d0598546981" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/c0f5f62db218fa72195b8b8700e4b9b9cf52eb5e", - "reference": "c0f5f62db218fa72195b8b8700e4b9b9cf52eb5e", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/fd7bd6535beb1f0a0a9e3ee960666d0598546981", + "reference": "fd7bd6535beb1f0a0a9e3ee960666d0598546981", "shasum": "" }, "require": { @@ -2506,11 +2365,11 @@ ], "description": "Symfony Filesystem Component", "homepage": "https://symfony.com", - "time": "2018-08-18T16:52:46+00:00" + "time": "2018-10-30T13:18:25+00:00" }, { "name": "symfony/finder", - "version": "v4.1.6", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", @@ -2606,16 +2465,16 @@ }, { "name": "symfony/framework-bundle", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/framework-bundle.git", - "reference": "f62dc69959253acf717c3d89cd509975daf10e02" + "reference": "5f05a52128de2fdd1285bd58d19a912a97bd290f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/framework-bundle/zipball/f62dc69959253acf717c3d89cd509975daf10e02", - "reference": "f62dc69959253acf717c3d89cd509975daf10e02", + "url": "https://api.github.com/repos/symfony/framework-bundle/zipball/5f05a52128de2fdd1285bd58d19a912a97bd290f", + "reference": "5f05a52128de2fdd1285bd58d19a912a97bd290f", "shasum": "" }, "require": { @@ -2639,6 +2498,7 @@ "symfony/asset": "<3.4", "symfony/console": "<3.4", "symfony/form": "<4.1", + "symfony/messenger": ">=4.2", "symfony/property-info": "<3.4", "symfony/serializer": "<4.1", "symfony/stopwatch": "<3.4", @@ -2718,20 +2578,20 @@ ], "description": "Symfony FrameworkBundle", "homepage": "https://symfony.com", - "time": "2018-08-17T12:07:19+00:00" + "time": "2018-10-31T09:09:42+00:00" }, { "name": "symfony/http-foundation", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/http-foundation.git", - "reference": "3a5c91e133b220bb882b3cd773ba91bf39989345" + "reference": "82d494c1492b0dd24bbc5c2d963fb02eb44491af" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-foundation/zipball/3a5c91e133b220bb882b3cd773ba91bf39989345", - "reference": "3a5c91e133b220bb882b3cd773ba91bf39989345", + "url": "https://api.github.com/repos/symfony/http-foundation/zipball/82d494c1492b0dd24bbc5c2d963fb02eb44491af", + "reference": "82d494c1492b0dd24bbc5c2d963fb02eb44491af", "shasum": "" }, "require": { @@ -2772,20 +2632,20 @@ ], "description": "Symfony HttpFoundation Component", "homepage": "https://symfony.com", - "time": "2018-08-27T17:47:02+00:00" + "time": "2018-10-31T09:09:42+00:00" }, { "name": "symfony/http-kernel", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/http-kernel.git", - "reference": "33de0a1ff2e1720096189e3ced682d7a4e8f5e35" + "reference": "958be64ab13b65172ad646ef5ae20364c2305fae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-kernel/zipball/33de0a1ff2e1720096189e3ced682d7a4e8f5e35", - "reference": "33de0a1ff2e1720096189e3ced682d7a4e8f5e35", + "url": "https://api.github.com/repos/symfony/http-kernel/zipball/958be64ab13b65172ad646ef5ae20364c2305fae", + "reference": "958be64ab13b65172ad646ef5ae20364c2305fae", "shasum": "" }, "require": { @@ -2859,11 +2719,11 @@ ], "description": "Symfony HttpKernel Component", "homepage": "https://symfony.com", - "time": "2018-08-28T06:17:42+00:00" + "time": "2018-11-03T11:11:23+00:00" }, { "name": "symfony/inflector", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/inflector.git", @@ -2949,16 +2809,16 @@ }, { "name": "symfony/polyfill-mbstring", - "version": "v1.9.0", + "version": "v1.10.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "d0cd638f4634c16d8df4508e847f14e9e43168b8" + "reference": "c79c051f5b3a46be09205c73b80b346e4153e494" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d0cd638f4634c16d8df4508e847f14e9e43168b8", - "reference": "d0cd638f4634c16d8df4508e847f14e9e43168b8", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/c79c051f5b3a46be09205c73b80b346e4153e494", + "reference": "c79c051f5b3a46be09205c73b80b346e4153e494", "shasum": "" }, "require": { @@ -3004,20 +2864,20 @@ "portable", "shim" ], - "time": "2018-08-06T14:22:27+00:00" + "time": "2018-09-21T13:07:52+00:00" }, { "name": "symfony/property-access", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/property-access.git", - "reference": "ae8561ba08af38e12dc5e21582ddb4baf66719ca" + "reference": "ae5620fb79729bc8b5dd83b75507cbcae24f83ee" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/property-access/zipball/ae8561ba08af38e12dc5e21582ddb4baf66719ca", - "reference": "ae8561ba08af38e12dc5e21582ddb4baf66719ca", + "url": "https://api.github.com/repos/symfony/property-access/zipball/ae5620fb79729bc8b5dd83b75507cbcae24f83ee", + "reference": "ae5620fb79729bc8b5dd83b75507cbcae24f83ee", "shasum": "" }, "require": { @@ -3071,11 +2931,11 @@ "property path", "reflection" ], - "time": "2018-08-24T10:22:26+00:00" + "time": "2018-10-02T12:40:59+00:00" }, { "name": "symfony/property-info", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/property-info.git", @@ -3151,16 +3011,16 @@ }, { "name": "symfony/routing", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/routing.git", - "reference": "a5784c2ec4168018c87b38f0e4f39d2278499f51" + "reference": "d4a3c14cfbe6b9c05a1d6e948654022d4d1ad3fd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/routing/zipball/a5784c2ec4168018c87b38f0e4f39d2278499f51", - "reference": "a5784c2ec4168018c87b38f0e4f39d2278499f51", + "url": "https://api.github.com/repos/symfony/routing/zipball/d4a3c14cfbe6b9c05a1d6e948654022d4d1ad3fd", + "reference": "d4a3c14cfbe6b9c05a1d6e948654022d4d1ad3fd", "shasum": "" }, "require": { @@ -3224,20 +3084,20 @@ "uri", "url" ], - "time": "2018-08-03T07:58:40+00:00" + "time": "2018-10-28T18:38:52+00:00" }, { "name": "symfony/serializer", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/serializer.git", - "reference": "2f134b6cad1cefa0613a4339b08e480124914c85" + "reference": "93a1967333cdd4f66c4a00f30895d124326f145b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/serializer/zipball/2f134b6cad1cefa0613a4339b08e480124914c85", - "reference": "2f134b6cad1cefa0613a4339b08e480124914c85", + "url": "https://api.github.com/repos/symfony/serializer/zipball/93a1967333cdd4f66c4a00f30895d124326f145b", + "reference": "93a1967333cdd4f66c4a00f30895d124326f145b", "shasum": "" }, "require": { @@ -3304,7 +3164,7 @@ ], "description": "Symfony Serializer Component", "homepage": "https://symfony.com", - "time": "2018-07-26T09:10:45+00:00" + "time": "2018-10-28T18:38:52+00:00" }, { "name": "symfony/serializer-pack", @@ -3339,16 +3199,16 @@ }, { "name": "symfony/yaml", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/yaml.git", - "reference": "b832cc289608b6d305f62149df91529a2ab3c314" + "reference": "367e689b2fdc19965be435337b50bc8adf2746c9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/yaml/zipball/b832cc289608b6d305f62149df91529a2ab3c314", - "reference": "b832cc289608b6d305f62149df91529a2ab3c314", + "url": "https://api.github.com/repos/symfony/yaml/zipball/367e689b2fdc19965be435337b50bc8adf2746c9", + "reference": "367e689b2fdc19965be435337b50bc8adf2746c9", "shasum": "" }, "require": { @@ -3394,7 +3254,7 @@ ], "description": "Symfony Yaml Component", "homepage": "https://symfony.com", - "time": "2018-08-18T16:52:46+00:00" + "time": "2018-10-02T16:36:10+00:00" }, { "name": "webmozart/assert", @@ -3868,22 +3728,22 @@ }, { "name": "phpspec/phpspec", - "version": "5.0.3", + "version": "5.1.0", "source": { "type": "git", "url": "https://github.com/phpspec/phpspec.git", - "reference": "6c0351a46582be027f3e504c93789427cdd4bcda" + "reference": "4badea737c34a6c8e2921fca0f6a1cbe4f724f2f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/phpspec/zipball/6c0351a46582be027f3e504c93789427cdd4bcda", - "reference": "6c0351a46582be027f3e504c93789427cdd4bcda", + "url": "https://api.github.com/repos/phpspec/phpspec/zipball/4badea737c34a6c8e2921fca0f6a1cbe4f724f2f", + "reference": "4badea737c34a6c8e2921fca0f6a1cbe4f724f2f", "shasum": "" }, "require": { "doctrine/instantiator": "^1.0.5", "ext-tokenizer": "*", - "php": "^7.1, <7.3", + "php": "^7.1, <7.4", "phpspec/php-diff": "^1.0.0", "phpspec/prophecy": "^1.7", "sebastian/exporter": "^1.0 || ^2.0 || ^3.0", @@ -3907,7 +3767,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "5.0.x-dev" + "dev-master": "5.1.x-dev" } }, "autoload": { @@ -3945,7 +3805,7 @@ "testing", "tests" ], - "time": "2018-10-01T07:32:39+00:00" + "time": "2018-10-29T08:12:52+00:00" }, { "name": "phpspec/prophecy", @@ -4308,16 +4168,16 @@ }, { "name": "symfony/dotenv", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/dotenv.git", - "reference": "22ca63c46e252b8a8f37b8f9e6da66bff5b3d3e7" + "reference": "9f3074b55bc56627f61fb2c17d573ee7df8e1319" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/dotenv/zipball/22ca63c46e252b8a8f37b8f9e6da66bff5b3d3e7", - "reference": "22ca63c46e252b8a8f37b8f9e6da66bff5b3d3e7", + "url": "https://api.github.com/repos/symfony/dotenv/zipball/9f3074b55bc56627f61fb2c17d573ee7df8e1319", + "reference": "9f3074b55bc56627f61fb2c17d573ee7df8e1319", "shasum": "" }, "require": { @@ -4361,20 +4221,20 @@ "env", "environment" ], - "time": "2018-07-26T11:24:31+00:00" + "time": "2018-10-12T12:56:03+00:00" }, { "name": "symfony/polyfill-php72", - "version": "v1.9.0", + "version": "v1.10.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php72.git", - "reference": "95c50420b0baed23852452a7f0c7b527303ed5ae" + "reference": "9050816e2ca34a8e916c3a0ae8b9c2fccf68b631" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php72/zipball/95c50420b0baed23852452a7f0c7b527303ed5ae", - "reference": "95c50420b0baed23852452a7f0c7b527303ed5ae", + "url": "https://api.github.com/repos/symfony/polyfill-php72/zipball/9050816e2ca34a8e916c3a0ae8b9c2fccf68b631", + "reference": "9050816e2ca34a8e916c3a0ae8b9c2fccf68b631", "shasum": "" }, "require": { @@ -4416,20 +4276,20 @@ "portable", "shim" ], - "time": "2018-08-06T14:22:27+00:00" + "time": "2018-09-21T13:07:52+00:00" }, { "name": "symfony/process", - "version": "v4.1.6", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "ee33c0322a8fee0855afcc11fff81e6b1011b529" + "reference": "3e83acef94d979b1de946599ef86b3a352abcdc9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/ee33c0322a8fee0855afcc11fff81e6b1011b529", - "reference": "ee33c0322a8fee0855afcc11fff81e6b1011b529", + "url": "https://api.github.com/repos/symfony/process/zipball/3e83acef94d979b1de946599ef86b3a352abcdc9", + "reference": "3e83acef94d979b1de946599ef86b3a352abcdc9", "shasum": "" }, "require": { @@ -4465,20 +4325,20 @@ ], "description": "Symfony Process Component", "homepage": "https://symfony.com", - "time": "2018-10-02T12:40:59+00:00" + "time": "2018-10-14T20:48:13+00:00" }, { "name": "symfony/translation", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/translation.git", - "reference": "fa2182669f7983b7aa5f1a770d053f79f0ef144f" + "reference": "aa04dc1c75b7d3da7bd7003104cd0cfc5dff635c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/translation/zipball/fa2182669f7983b7aa5f1a770d053f79f0ef144f", - "reference": "fa2182669f7983b7aa5f1a770d053f79f0ef144f", + "url": "https://api.github.com/repos/symfony/translation/zipball/aa04dc1c75b7d3da7bd7003104cd0cfc5dff635c", + "reference": "aa04dc1c75b7d3da7bd7003104cd0cfc5dff635c", "shasum": "" }, "require": { @@ -4534,20 +4394,20 @@ ], "description": "Symfony Translation Component", "homepage": "https://symfony.com", - "time": "2018-08-07T12:45:11+00:00" + "time": "2018-10-28T18:38:52+00:00" }, { "name": "symfony/var-dumper", - "version": "v4.1.4", + "version": "v4.1.7", "source": { "type": "git", "url": "https://github.com/symfony/var-dumper.git", - "reference": "a05426e27294bba7b0226ffc17dd01a3c6ef9777" + "reference": "60319b45653580b0cdacca499344577d87732f16" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/var-dumper/zipball/a05426e27294bba7b0226ffc17dd01a3c6ef9777", - "reference": "a05426e27294bba7b0226ffc17dd01a3c6ef9777", + "url": "https://api.github.com/repos/symfony/var-dumper/zipball/60319b45653580b0cdacca499344577d87732f16", + "reference": "60319b45653580b0cdacca499344577d87732f16", "shasum": "" }, "require": { @@ -4609,7 +4469,7 @@ "debug", "dump" ], - "time": "2018-08-02T09:24:26+00:00" + "time": "2018-10-02T16:36:10+00:00" } ], "aliases": [], diff --git a/config/bundles.php b/config/bundles.php index a4a8ac1..0057e19 100644 --- a/config/bundles.php +++ b/config/bundles.php @@ -5,6 +5,5 @@ Doctrine\Bundle\DoctrineCacheBundle\DoctrineCacheBundle::class => ['all' => true], Doctrine\Bundle\DoctrineBundle\DoctrineBundle::class => ['all' => true], Doctrine\Bundle\MigrationsBundle\DoctrineMigrationsBundle::class => ['all' => true], - OldSound\RabbitMqBundle\OldSoundRabbitMqBundle::class => ['all' => true], Snc\RedisBundle\SncRedisBundle::class => ['all' => true], ]; diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index c4241b0..c04052f 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -11,6 +11,8 @@ doctrine: connections: source: url: '%env(resolve:SOURCE_DATABASE_URL)%' + options: + !php/const PDO::MYSQL_ATTR_USE_BUFFERED_QUERY: 0 target: options: !php/const PDO::MYSQL_ATTR_LOCAL_INFILE: 1 diff --git a/config/packages/old_sound_rabbit_mq.yaml b/config/packages/old_sound_rabbit_mq.yaml deleted file mode 100644 index c94f3c5..0000000 --- a/config/packages/old_sound_rabbit_mq.yaml +++ /dev/null @@ -1,15 +0,0 @@ -old_sound_rabbit_mq: - connections: - default: - url: '%env(RABBITMQ_URL)%' - producers: - fogger_data_chunks: - connection: default - exchange_options: { name: 'fogger::data_chunks', type: direct } - queue_options: { name: 'fogger::data_chunks'} - consumers: - fogger_data_chunks: - connection: default - exchange_options: { name: 'fogger::data_chunks', type: direct } - queue_options: { name: 'fogger::data_chunks'} - callback: App\Fogger\Data\ChunkConsumer diff --git a/config/packages/test/old_sound_rabbit_mq.yaml b/config/packages/test/old_sound_rabbit_mq.yaml deleted file mode 100644 index 48956fb..0000000 --- a/config/packages/test/old_sound_rabbit_mq.yaml +++ /dev/null @@ -1,15 +0,0 @@ -old_sound_rabbit_mq: - connections: - default: - url: '%env(RABBITMQ_URL)%' - producers: - fogger_data_chunks_test: - connection: default - exchange_options: { name: 'fogger::data_chunks_test', type: direct } - queue_options: { name: 'fogger::data_chunks_test'} - consumers: - fogger_data_chunks_test: - connection: default - exchange_options: { name: 'fogger::data_chunks_test', type: direct } - queue_options: { name: 'fogger::data_chunks_test'} - callback: App\Fogger\Data\ChunkConsumer diff --git a/config/services.yaml b/config/services.yaml index 324c920..b8aa0ad 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -52,7 +52,7 @@ services: tags: - { name: 'serializer.normalizer'} - App\Fogger\Data\ChunkCounter: + App\Fogger\Data\ChunkCache: arguments: - '@snc_redis.default' @@ -60,10 +60,6 @@ services: arguments: - '@snc_redis.default' - App\Fogger\Data\ChunkProducer: - arguments: - - '@old_sound_rabbit_mq.fogger_data_chunks_producer' - App\Fogger\Data\Writer\GenericInsertWriter: arguments: - '@doctrine.dbal.target_connection' diff --git a/config/services_test.yaml b/config/services_test.yaml deleted file mode 100644 index b7eb5c2..0000000 --- a/config/services_test.yaml +++ /dev/null @@ -1,8 +0,0 @@ -services: - App\Fogger\Data\ChunkProducer: - arguments: - - '@old_sound_rabbit_mq.fogger_data_chunks_test_producer' - - '@App\Fogger\Data\ChunkDivider' - - '@serializer' - - '@App\Fogger\Data\ChunkCounter' - - '@App\Fogger\Data\ChunkError' diff --git a/docker-compose.yml b/docker-compose.yml index 4bfaafa..8214c3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,8 +3,7 @@ services: app: depends_on: - redis - - rabbit - image: tshio/fogger:latest + build: . volumes: - .:/app - ./var/:/fogger @@ -16,8 +15,7 @@ services: worker: depends_on: - redis - - rabbit - image: tshio/fogger:latest + build: . volumes: - .:/app - ./var/:/fogger @@ -27,14 +25,9 @@ services: RABBITMQ_URL: amqp://user:pass@rabbit:5672 REDIS_URL: redis://redis restart: always - command: rabbit:consumer --messages=200 fogger_data_chunks + command: fogger:consumer redis: image: redis:4 - rabbit: - image: rabbitmq:3 - environment: - RABBITMQ_DEFAULT_USER: user - RABBITMQ_DEFAULT_PASS: pass source: environment: MYSQL_DATABASE: source diff --git a/features/Run/Data/copyUTF8.feature b/features/Run/Data/copyUTF8.feature new file mode 100644 index 0000000..d9a60c9 --- /dev/null +++ b/features/Run/Data/copyUTF8.feature @@ -0,0 +1,37 @@ +Feature: + In order to copy masked data properly + As a user + I want to get correctly encoded data in target database + + Scenario: We want to subset the table that is referenced by other table (Not Null column) + Given there is a source database + And there is a table test with following columns: + | name | type | length | index | + | id | integer | | primary | + | text | string | 64 | | + And the table test contains following data: + | id | text | + | 1 | zażółć gęślą jaźń | + | 2 | المملكة العربية السعودية | + | 3 | 中华人民共和国 | + And there is an empty target database + And the task queue is empty + And the config test.yaml contains: + """ + tables: + test: ~ + """ + When I run "run" command with input: + | --chunk-size | 1000 | + | --file | test.yaml | + | --dont-wait | true | + And worker processes 1 task + And I run "finish" command with input: + | --file | test.yaml | + Then the command should exit with code 0 + And the table test in target database should have 3 rows + And the table test in target database should contain rows: + | id | text | + | 1 | zażółć gęślą jaźń | + | 2 | المملكة العربية السعودية | + | 3 | 中华人民共和国 | diff --git a/features/bootstrap/ChunkCounterContext.php b/features/bootstrap/ChunkCacheContext.php similarity index 63% rename from features/bootstrap/ChunkCounterContext.php rename to features/bootstrap/ChunkCacheContext.php index 1505c0a..2ed796e 100644 --- a/features/bootstrap/ChunkCounterContext.php +++ b/features/bootstrap/ChunkCacheContext.php @@ -1,15 +1,15 @@ chunkCounter = $chunkCounter; + $this->chunkCache = $chunkCache; } /** @@ -39,7 +39,7 @@ private function assertCountEquals(int $value, int $expected) */ public function publishedTasksCounterShouldEqual(int $expected) { - $this->assertCountEquals($this->chunkCounter->getPublishedCount(), $expected); + $this->assertCountEquals($this->chunkCache->getPublishedCount(), $expected); } /** @@ -49,6 +49,14 @@ public function publishedTasksCounterShouldEqual(int $expected) */ public function processedTasksCounterShouldEqual(int $expected) { - $this->assertCountEquals($this->chunkCounter->getProcessedCount(), $expected); + $this->assertCountEquals($this->chunkCache->getProcessedCount(), $expected); + } + + /** + * @Given the task queue is empty + */ + public function theTaskQueueIsEmpty() + { + $this->chunkCache->reset(); } } diff --git a/features/bootstrap/CommandContext.php b/features/bootstrap/CommandContext.php index 1c7820d..895b25e 100644 --- a/features/bootstrap/CommandContext.php +++ b/features/bootstrap/CommandContext.php @@ -57,7 +57,12 @@ public function iRunCommandWithInput($name, TableNode $table) { $this->runCommand( $this->foggerName($name), - array_map(function ($item) { return $item === 'true' ? true : $item; }, $table->getRowsHash()) + array_map( + function ($item) { + return $item === 'true' ? true : $item; + }, + $table->getRowsHash() + ) ); } @@ -99,20 +104,6 @@ public function printCommandsOutput() dump($this->tester->getDisplay()); } - /** - * @Given the task queue is empty - */ - public function theTaskQueueIsEmpty() - { - $this->runCommand( - 'rabbitmq:purge', - [ - 'name' => 'fogger_data_chunks_test', - '--no-confirmation' => true, - ] - ); - } - /** * @When worker processes :count task(s) * @param int $count @@ -120,10 +111,9 @@ public function theTaskQueueIsEmpty() public function workerProcessTask(int $count) { $this->runCommand( - 'rabbitmq:consumer', + 'fogger:consumer', [ '--messages' => $count, - 'name' => 'fogger_data_chunks_test', ] ); } diff --git a/src/Command/ConsumerCommand.php b/src/Command/ConsumerCommand.php new file mode 100644 index 0000000..34670ae --- /dev/null +++ b/src/Command/ConsumerCommand.php @@ -0,0 +1,66 @@ +chunkCache = $chunkCache; + $this->chunkConsumer = $chunkConsumer; + + parent::__construct(); + } + + protected function configure() + { + $this + ->setName('fogger:consumer') + ->addOption( + 'file', + 'f', + InputOption::VALUE_REQUIRED, + 'Where should the command look for a config file. Defaults to fogger.yaml in root folder.', + ConfigLoader::DEFAULT_FILENAME + ) + ->addOption( + 'messages', + 'm', + InputOption::VALUE_REQUIRED, + 'How many messages to process.', + 200 + ) + ->setDescription('Consumes a message'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + for ($i = 0; $i < $input->getOption('messages'); $i++) { + + /** @var ChunkMessage $message */ + $message = $this->chunkCache->popMessage(); + + if ($message instanceof ChunkMessage) { + $this->chunkConsumer->execute($message); + } else { + echo('.'); + usleep(500000); + } + } + } +} diff --git a/src/Command/FinishCommand.php b/src/Command/FinishCommand.php index 407c485..9a5a779 100644 --- a/src/Command/FinishCommand.php +++ b/src/Command/FinishCommand.php @@ -3,8 +3,9 @@ namespace App\Command; use App\Config\ConfigLoader; -use App\Fogger\Data\ChunkCounter; +use App\Fogger\Data\ChunkCache; use App\Fogger\Data\ChunkError; +use App\Fogger\Recipe\RecipeFactory; use App\Fogger\Refine\Refiner; use App\Fogger\Schema\SchemaManipulator; use Symfony\Component\Console\Command\Command; @@ -17,22 +18,28 @@ class FinishCommand extends Command { protected $schemaManipulator; - protected $chunkCounter; + protected $chunkCache; protected $chunkError; + protected $recipeFactory; + + protected $recipe = null; + private $refiner; public function __construct( SchemaManipulator $schemaManipulator, Refiner $refiner, - ChunkCounter $chunkCounter, - ChunkError $chunkError + ChunkCache $chunkCache, + ChunkError $chunkError, + RecipeFactory $recipeFactory ) { $this->schemaManipulator = $schemaManipulator; $this->refiner = $refiner; - $this->chunkCounter = $chunkCounter; + $this->chunkCache = $chunkCache; $this->chunkError = $chunkError; + $this->recipeFactory = $recipeFactory; parent::__construct(); } @@ -56,12 +63,12 @@ protected function execute(InputInterface $input, OutputInterface $output) $output->writeln('Fogger finish procedure'); $io = new SymfonyStyle($input, $output); - if ($this->chunkCounter->getProcessedCount() < $this->chunkCounter->getPublishedCount()) { + if ($this->chunkCache->getProcessedCount() < $this->chunkCache->getPublishedCount()) { $this->outputMessage( sprintf( "We are still working on it, please try again later (%d/%d)", - $this->chunkCounter->getProcessedCount(), - $this->chunkCounter->getPublishedCount() + $this->chunkCache->getProcessedCount(), + $this->chunkCache->getPublishedCount() ), $io, 'fg=black;bg=yellow' @@ -77,10 +84,12 @@ protected function execute(InputInterface $input, OutputInterface $output) } try { + $output->writeln(' - refining database...'); + $this->refiner->refine( + $this->recipe ?? $this->recipeFactory->createRecipe($input->getOption('file')) + ); $output->writeln(' - recreating indexes...'); $this->schemaManipulator->recreateIndexes(); - $output->writeln(' - refining database...'); - $this->refiner->refineBasedOnConfig($input->getOption('file')); $output->writeln(' - recreating foreign keys...'); $this->schemaManipulator->recreateForeignKeys(); } catch (\Exception $exception) { diff --git a/src/Command/RunCommand.php b/src/Command/RunCommand.php index ca6edf2..3ebb372 100644 --- a/src/Command/RunCommand.php +++ b/src/Command/RunCommand.php @@ -3,8 +3,9 @@ namespace App\Command; use App\Config\ConfigLoader; -use App\Fogger\Data\ChunkCounter; +use App\Fogger\Data\ChunkCache; use App\Fogger\Data\ChunkError; +use App\Fogger\Data\ChunkMessage; use App\Fogger\Data\ChunkProducer; use App\Fogger\Recipe\RecipeFactory; use App\Fogger\Refine\Refiner; @@ -19,21 +20,18 @@ class RunCommand extends FinishCommand { private $chunkProducer; - private $recipeFactory; - public function __construct( SchemaManipulator $schemaManipulator, ChunkProducer $chunkProducer, RecipeFactory $recipeFactory, Refiner $refiner, - ChunkCounter $chunkCounter, + ChunkCache $chunkCache, ChunkError $chunkError ) { $this->chunkProducer = $chunkProducer; - $this->recipeFactory = $recipeFactory; - parent::__construct($schemaManipulator, $refiner, $chunkCounter, $chunkError); + parent::__construct($schemaManipulator, $refiner, $chunkCache, $chunkError, $recipeFactory); } protected function configure() @@ -51,8 +49,11 @@ protected function configure() 'chunk-size', 'c', InputOption::VALUE_REQUIRED, - 'Data is moved in chunks. What should be the size of a chunk. Defaults to 1000', - 1000 + sprintf( + 'Data is moved in chunks. What should be the size of a chunk. Defaults to %d', + ChunkMessage::DEFAULT_CHUNK_SIZE + ), + ChunkMessage::DEFAULT_CHUNK_SIZE ) ->addOption( 'dont-wait', @@ -65,7 +66,7 @@ protected function configure() private function showProgressBar(OutputInterface $output) { - $published = $this->chunkCounter->getPublishedCount(); + $published = $this->chunkCache->getPublishedCount(); $output->writeln(''); $output->writeln('If you are masking big database, you can stop this process with Cmd/Ctrl + C'); @@ -79,9 +80,9 @@ private function showProgressBar(OutputInterface $output) $progressBar->start(); do { - $processed = $this->chunkCounter->getProcessedCount(); + $processed = $this->chunkCache->getProcessedCount(); $progressBar->setProgress($processed); - sleep(0.5); + usleep(100000); } while ($processed < $published); $progressBar->finish(); @@ -101,9 +102,9 @@ protected function execute(InputInterface $input, OutputInterface $output) try { $this->schemaManipulator->copySchemaDroppingIndexesAndForeignKeys(); - $recipe = $this->recipeFactory + $this->recipe = $this->recipeFactory ->createRecipe($input->getOption('file'), $chunkSize); - $this->chunkProducer->run($recipe); + $this->chunkProducer->run($this->recipe); } catch (\Exception $exception) { $this->outputMessage("There has been an error:\n\n".$exception->getMessage(), $io); @@ -123,7 +124,7 @@ protected function execute(InputInterface $input, OutputInterface $output) ); $output->writeln(''); $output->writeln( - sprintf('%d chunks have been added to queue', $this->chunkCounter->getPublishedCount()) + sprintf('%d chunks have been added to queue', $this->chunkCache->getPublishedCount()) ); return 0; diff --git a/src/Fogger/Data/ChunkCache.php b/src/Fogger/Data/ChunkCache.php new file mode 100644 index 0000000..238543a --- /dev/null +++ b/src/Fogger/Data/ChunkCache.php @@ -0,0 +1,75 @@ +redis = $redis; + $this->serializer = $serializer; + } + + public function reset() + { + $this->redis->del( + [ + self::CHUNKS_PUBLISHED, + self::CHUNKS_PROCESSED, + self::LIST_NAME, + ] + ); + } + + public function pushMessage(Table $table, array $keys = []) + { + $message = $this->serializer->serialize(new ChunkMessage($table, $keys), 'json'); + $this->redis->rpush(self::LIST_NAME, [$message]); + $this->increasePublishedCount(); + } + + public function popMessage() + { + if (null === $content = $this->redis->lpop(self::LIST_NAME)) { + return null; + } + + return $this->serializer->deserialize( + $content, + ChunkMessage::class, + 'json' + ); + } + + public function increasePublishedCount() + { + $this->redis->incr(self::CHUNKS_PUBLISHED); + } + + public function increaseProcessedCount() + { + $this->redis->incr(self::CHUNKS_PROCESSED); + } + + public function getPublishedCount(): int + { + return $this->redis->get(self::CHUNKS_PUBLISHED) ?? 0; + } + + public function getProcessedCount(): int + { + return $this->redis->get(self::CHUNKS_PROCESSED) ?? 0; + } +} diff --git a/src/Fogger/Data/ChunkConsumer.php b/src/Fogger/Data/ChunkConsumer.php index 04b0248..d68e01b 100644 --- a/src/Fogger/Data/ChunkConsumer.php +++ b/src/Fogger/Data/ChunkConsumer.php @@ -2,43 +2,33 @@ namespace App\Fogger\Data; -use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface; -use PhpAmqpLib\Message\AMQPMessage; -use Symfony\Component\Serializer\SerializerInterface; - -class ChunkConsumer implements ConsumerInterface +class ChunkConsumer { private $dataCopier; - private $serializer; - - private $counter; + private $cache; private $error; public function __construct( DataCopier $dataCopier, - SerializerInterface $serializer, - ChunkCounter $counter, + ChunkCache $cache, ChunkError $error ) { $this->dataCopier = $dataCopier; - $this->serializer = $serializer; - $this->counter = $counter; + $this->cache = $cache; $this->error = $error; } - public function execute(AMQPMessage $msg) + public function execute(ChunkMessage $message) { try { - /** @var ChunkMessage $chunkMessage */ - $chunkMessage = $this->serializer->deserialize($msg->getBody(), ChunkMessage::class, 'json'); - $this->dataCopier->copyDataChunk($chunkMessage); + $this->dataCopier->copyDataChunk($message); } catch (\Exception $exception) { $this->error->addError($exception->getMessage()); } - $this->counter->increaseProcessedCount(); + $this->cache->increaseProcessedCount(); } } diff --git a/src/Fogger/Data/ChunkCounter.php b/src/Fogger/Data/ChunkCounter.php deleted file mode 100644 index 8e51acc..0000000 --- a/src/Fogger/Data/ChunkCounter.php +++ /dev/null @@ -1,44 +0,0 @@ -redis = $redis; - } - - public function reset() - { - $this->redis->set(self::CHUNKS_PUBLISHED, 0); - $this->redis->set(self::CHUNKS_PROCESSED, 0); - } - - public function increasePublishedCount() - { - $this->redis->incr(self::CHUNKS_PUBLISHED); - } - - public function increaseProcessedCount() - { - $this->redis->incr(self::CHUNKS_PROCESSED); - } - - public function getPublishedCount(): int - { - return $this->redis->get(self::CHUNKS_PUBLISHED); - } - - public function getProcessedCount(): int - { - return $this->redis->get(self::CHUNKS_PROCESSED); - } -} diff --git a/src/Fogger/Data/ChunkDivider.php b/src/Fogger/Data/ChunkDivider.php deleted file mode 100644 index 1a84321..0000000 --- a/src/Fogger/Data/ChunkDivider.php +++ /dev/null @@ -1,55 +0,0 @@ -tableQuery = $query; - $this->subsetStrategyProvider = $subsetStrategyProvider; - } - - /** - * @param Table $table - * @return int - * @throws \App\Fogger\Subset\Exception\UnknownSubsetStrategyException - */ - protected function getSubsetRowsCount(Table $table): int - { - $query = $this->tableQuery->getAllRowsQuery($table, true); - $subsetStrategy = $this->subsetStrategyProvider->getSubsetStrategy($table->getSubsetName()); - $subsetStrategy->subsetQuery($query, $table); - - return (int)$query->execute()->fetchColumn(); - } - - /** - * @param Table $table - * @return int - * @throws \App\Fogger\Subset\Exception\UnknownSubsetStrategyException - */ - public function getNumberOfChunks(Table $table): int - { - $rowsCount = $this->getSubsetRowsCount($table); - if ($table->getSortBy() === null) { - $table->setChunkSize($rowsCount); - - return 1; - } - - return ceil($rowsCount / $table->getChunkSize()); - } -} diff --git a/src/Fogger/Data/ChunkMessage.php b/src/Fogger/Data/ChunkMessage.php index 18147fa..030fbb5 100644 --- a/src/Fogger/Data/ChunkMessage.php +++ b/src/Fogger/Data/ChunkMessage.php @@ -6,14 +6,16 @@ class ChunkMessage { + const DEFAULT_CHUNK_SIZE = 2000; + private $table; - private $chunkNumber; + private $keys; - public function __construct(Table $table, int $chunkNumber) + public function __construct(Table $table, array $keys) { $this->table = $table; - $this->chunkNumber = $chunkNumber; + $this->keys = $keys; } public function getTable(): Table @@ -21,28 +23,8 @@ public function getTable(): Table return $this->table; } - public function getLimit(): int - { - return $this->table->getChunkSize(); - } - - public function getOffset(): int - { - return $this->getTable()->getChunkSize() * $this->chunkNumber; - } - - public function getStrategyName(): string - { - return $this->table->getSubset()->getName(); - } - - public function getMasks(): array - { - return $this->table->getMasks(); - } - - public function getChunkNumber(): int + public function getKeys(): array { - return $this->chunkNumber; + return $this->keys; } } diff --git a/src/Fogger/Data/ChunkProducer.php b/src/Fogger/Data/ChunkProducer.php index 794556c..257bf9e 100644 --- a/src/Fogger/Data/ChunkProducer.php +++ b/src/Fogger/Data/ChunkProducer.php @@ -4,33 +4,22 @@ use App\Fogger\Recipe\Recipe; use App\Fogger\Recipe\Table; -use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; -use Symfony\Component\Serializer\SerializerInterface; class ChunkProducer { - private $rabbitProducer; + private $sourceQuery; - private $chunkDivider; - - private $serializer; - - private $chunkCounter; + private $chunkCache; private $chunkError; public function __construct( - ProducerInterface $rabbitProducer, - ChunkDivider $chunkDivider, - SerializerInterface $serializer, - ChunkCounter $chunkCounter, + SourceQuery $sourceQuery, + ChunkCache $chunkCache, ChunkError $chunkError - ) - { - $this->chunkDivider = $chunkDivider; - $this->rabbitProducer = $rabbitProducer; - $this->serializer = $serializer; - $this->chunkCounter = $chunkCounter; + ) { + $this->sourceQuery = $sourceQuery; + $this->chunkCache = $chunkCache; $this->chunkError = $chunkError; } @@ -40,14 +29,29 @@ public function __construct( */ private function queueTableChunks(Table $table) { - $count = $this->chunkDivider->getNumberOfChunks($table); - for ($i = 0; $i < $count; $i++) { - $this->rabbitProducer->publish( - $this->serializer->serialize(new ChunkMessage($table, $i), 'json') - ); + if (null === $table->getSortBy()) { + $this->sourceQuery->getAllKeysQuery($table); + $this->chunkCache->pushMessage($table); + + return; + } + + $result = $this->sourceQuery->getAllKeysQuery($table)->execute(); - $this->chunkCounter->increasePublishedCount(); + $counter = 0; + $keys = []; + + while ($key = $result->fetchColumn()) { + $keys[] = $key; + $counter++; + if (0 === $counter % $table->getChunkSize()) { + $this->chunkCache->pushMessage($table, $keys); + $keys = []; + } + } + if (0 !== $counter % $table->getChunkSize()) { + $this->chunkCache->pushMessage($table, $keys); } } @@ -57,7 +61,7 @@ private function queueTableChunks(Table $table) */ public function run(Recipe $recipe) { - $this->chunkCounter->reset(); + $this->chunkCache->reset(); $this->chunkError->reset(); foreach ($recipe->getTables() as $table) { $this->queueTableChunks($table); diff --git a/src/Fogger/Data/ChunkReader.php b/src/Fogger/Data/ChunkReader.php index 9df235a..aee6cea 100644 --- a/src/Fogger/Data/ChunkReader.php +++ b/src/Fogger/Data/ChunkReader.php @@ -2,18 +2,13 @@ namespace App\Fogger\Data; -use App\Fogger\Subset\SubsetStrategyProvider; - class ChunkReader { - private $query; - - private $subsetStrategyProvider; + private $sourceQuery; - public function __construct(TableQuery $query, SubsetStrategyProvider $subsetStrategyProvider) + public function __construct(SourceQuery $sourceQuery) { - $this->query = $query; - $this->subsetStrategyProvider = $subsetStrategyProvider; + $this->sourceQuery = $sourceQuery; } /** @@ -23,12 +18,10 @@ public function __construct(TableQuery $query, SubsetStrategyProvider $subsetStr */ public function getDataChunk(ChunkMessage $chunkMessage): array { - $query = $this->query->getAllRowsQuery($chunkMessage->getTable()); - $subsetStrategy = $this->subsetStrategyProvider->getSubsetStrategy($chunkMessage->getStrategyName()); - $subsetStrategy->subsetQuery($query, $chunkMessage->getTable()); - $query - ->setMaxResults($chunkMessage->getLimit()) - ->setFirstResult($chunkMessage->getOffset()); + $query = $this->sourceQuery->getAllRowsQuery( + $chunkMessage->getTable(), + $chunkMessage->getKeys() + ); return $query->execute()->fetchAll(); } diff --git a/src/Fogger/Data/DataCopier.php b/src/Fogger/Data/DataCopier.php index eef5c05..3d21791 100644 --- a/src/Fogger/Data/DataCopier.php +++ b/src/Fogger/Data/DataCopier.php @@ -31,9 +31,10 @@ public function __construct( public function copyDataChunk(ChunkMessage $chunkMessage) { $data = $this->chunkReader->getDataChunk($chunkMessage); + $table = $chunkMessage->getTable(); $this->chunkWriterProvider->getWriter()->insert( - $chunkMessage->getTable()->getName(), - $this->masker->applyMasks($data, $chunkMessage->getMasks()) + $table->getName(), + $this->masker->applyMasks($data, $table->getMasks()) ); } } diff --git a/src/Fogger/Data/SourceQuery.php b/src/Fogger/Data/SourceQuery.php new file mode 100644 index 0000000..aeea115 --- /dev/null +++ b/src/Fogger/Data/SourceQuery.php @@ -0,0 +1,61 @@ +source = $source; + $this->provider = $provider; + } + + /** + * @param Table $table + * @return QueryBuilder + * @throws \App\Fogger\Subset\Exception\UnknownSubsetStrategyException + */ + public function getAllKeysQuery(Table $table) + { + $query = $this->getAllRowsQuery($table); + $query + ->resetQueryPart('select') + ->select($this->source->quoteIdentifier($table->getSortBy())); + + return $query; + } + + /** + * @param Table $table + * @param array $keys + * @return QueryBuilder + * @throws \App\Fogger\Subset\Exception\UnknownSubsetStrategyException + */ + public function getAllRowsQuery(Table $table, array $keys = []): QueryBuilder + { + $query = $this->source->createQueryBuilder(); + $query + ->select('*') + ->from($this->source->quoteIdentifier($table->getName())); + + if (count($keys)) { + $query + ->where($query->expr()->in($table->getSortBy(), ':keys')) + ->setParameter('keys', $keys, Connection::PARAM_STR_ARRAY); + + return $query; + } + $subset = $this->provider->getSubsetStrategy($table->getSubsetName()); + + return $subset->subsetQuery($query, $table); + } +} diff --git a/src/Fogger/Data/TableQuery.php b/src/Fogger/Data/TableQuery.php deleted file mode 100644 index 7a13aeb..0000000 --- a/src/Fogger/Data/TableQuery.php +++ /dev/null @@ -1,32 +0,0 @@ -source = $source; - } - - public function getAllRowsQuery(Table $table, $countOnly = false): QueryBuilder - { - $queryBuilder = $this->source->createQueryBuilder(); - - $queryBuilder - ->select($countOnly ? 'count(*)' : '*') - ->from($this->source->quoteIdentifier($table->getName())); - - if (!$countOnly && $table->getSortBy() !== null) { - $queryBuilder->OrderBy($table->getSortBy()); - } - - return $queryBuilder; - } -} diff --git a/src/Fogger/Mask/AbstractCachedMask.php b/src/Fogger/Mask/AbstractCachedMask.php index 702aeb1..5f8c8b3 100644 --- a/src/Fogger/Mask/AbstractCachedMask.php +++ b/src/Fogger/Mask/AbstractCachedMask.php @@ -20,6 +20,11 @@ public function __construct(CacheItemPoolInterface $cache) abstract protected function getSubstitution(array $options = []): ?string; + private function forgeCacheKey(string $value, array $options) + { + return md5(sprintf("%s.%s.%s", $value, $this->getMaskName(), json_encode($options))); + } + /** * @param null|string $value * @param array $options @@ -33,7 +38,7 @@ public function apply(?string $value, array $options = []): ?string } do { - $originalValueCacheItem = $this->cache->getItem(md5($value)); + $originalValueCacheItem = $this->cache->getItem($this->forgeCacheKey($value, $options)); } while ($originalValueCacheItem->get() === self::LOCK_VALUE); if ($originalValueCacheItem->isHit()) { @@ -45,7 +50,7 @@ public function apply(?string $value, array $options = []): ?string do { $substitution = $this->getSubstitution($options); - $substitutionCacheItem = $this->cache->getItem(md5($substitution)); + $substitutionCacheItem = $this->cache->getItem($this->forgeCacheKey($substitution, $options)); } while ($substitutionCacheItem->isHit()); $this->cache->save($substitutionCacheItem); diff --git a/src/Fogger/Mask/FakerMask.php b/src/Fogger/Mask/FakerMask.php index 6632e39..42fce58 100644 --- a/src/Fogger/Mask/FakerMask.php +++ b/src/Fogger/Mask/FakerMask.php @@ -21,13 +21,13 @@ public function __construct(Generator $generator, CacheItemPoolInterface $cache) public function getSubstitution(array $options = []): ?string { $method = $options['method'] ?? self::DEFAULT_METHOD; - $parameters = $options['parameters'] ?? []; + $parameters = $options['arguments'] ?? []; $result = $this->generator->$method(...$parameters); if (is_array($result)) { - return implode(' ', $result); - } else if ($result instanceof \DateTime) { - return $result->format("Y-m-d H:i:s"); + $result = implode(' ', $result); + } elseif ($result instanceof \DateTime) { + $result = $result->format("Y-m-d H:i:s"); } return $result; diff --git a/src/Fogger/Recipe/RecipeFactory.php b/src/Fogger/Recipe/RecipeFactory.php index bb1a1d8..b87fec2 100644 --- a/src/Fogger/Recipe/RecipeFactory.php +++ b/src/Fogger/Recipe/RecipeFactory.php @@ -3,6 +3,7 @@ namespace App\Fogger\Recipe; use App\Config\ConfigLoader; +use App\Fogger\Data\ChunkMessage; use Doctrine\DBAL\Connection; class RecipeFactory @@ -20,8 +21,7 @@ public function __construct( Connection $connection, RecipeTableFactory $recipeTableFactory, MaskReplicator $maskReplicator - ) - { + ) { $this->configLoader = $configLoader; $this->sourceSchema = $connection->getSchemaManager(); $this->recipeTableFactory = $recipeTableFactory; @@ -30,11 +30,11 @@ public function __construct( /** * @param string $configFilename - * @param int $configChangSize + * @param int $chunkSize * @return Recipe * @throws \Doctrine\DBAL\DBALException */ - public function createRecipe(string $configFilename, int $configChangSize = 1000) + public function createRecipe(string $configFilename, int $chunkSize = ChunkMessage::DEFAULT_CHUNK_SIZE) { $config = $this->configLoader->load($configFilename); $recipe = new Recipe($config->getExcludes()); @@ -44,7 +44,7 @@ public function createRecipe(string $configFilename, int $configChangSize = 1000 if (!in_array($tableName, $config->getExcludes())) { $recipe->addTable( $tableName, - $this->recipeTableFactory->createRecipeTable($dbalTable, $configChangSize, $config->getTable($tableName)) + $this->recipeTableFactory->createRecipeTable($dbalTable, $chunkSize, $config->getTable($tableName)) ); } } diff --git a/src/Fogger/Recipe/RecipeTableFactory.php b/src/Fogger/Recipe/RecipeTableFactory.php index 5ec4092..ff3bdce 100644 --- a/src/Fogger/Recipe/RecipeTableFactory.php +++ b/src/Fogger/Recipe/RecipeTableFactory.php @@ -24,7 +24,7 @@ private function addMask(Table $table, Config\ColumnConfig $column, $columnName) */ private function findSortBy(DBAL\Table $table): ?string { - if ($table->getPrimaryKey() && count($table->getPrimaryKeyColumns())) { + if ($table->getPrimaryKey() && 1 === count($table->getPrimaryKeyColumns())) { return $table->getPrimaryKeyColumns()[0]; } foreach ($table->getIndexes() as $index) { @@ -43,8 +43,11 @@ private function findSortBy(DBAL\Table $table): ?string * @return Table * @throws \Doctrine\DBAL\DBALException */ - public function createRecipeTable(DBAL\Table $dbalTable, int $chunkSize, Config\TableConfig $tableConfig = null): Table - { + public function createRecipeTable( + DBAL\Table $dbalTable, + int $chunkSize, + Config\TableConfig $tableConfig = null + ): Table { if ($tableConfig && $subsetStrategy = $tableConfig->getSubsetStrategy()) { $subset = new StrategyDefinition($subsetStrategy, $tableConfig->getSubsetOptions()); } diff --git a/src/Fogger/Refine/Refiner.php b/src/Fogger/Refine/Refiner.php index b0c1a6e..d5efc77 100644 --- a/src/Fogger/Refine/Refiner.php +++ b/src/Fogger/Refine/Refiner.php @@ -2,7 +2,7 @@ namespace App\Fogger\Refine; -use App\Fogger\Recipe\RecipeFactory; +use App\Fogger\Recipe\Recipe; use App\Fogger\Recipe\Table; use App\Fogger\Schema\ForeignKeysExtractor; use App\Fogger\Subset\NoSubset; @@ -10,18 +10,14 @@ class Refiner { - private $recipeFactory; - private $extractor; private $refineExecutor; public function __construct( - RecipeFactory $recipeFactory, ForeignKeysExtractor $extractor, RefineExecutor $refineExecutor ) { - $this->recipeFactory = $recipeFactory; $this->extractor = $extractor; $this->refineExecutor = $refineExecutor; } @@ -44,6 +40,13 @@ private function refineIfSubsetted(Table $table): void */ private function runQueryFor(Schema\ForeignKeyConstraint $foreignKey) { + echo(sprintf( + " - %s.%s => %s.%s\n", + $foreignKey->getLocalTableName(), + implode('_', $foreignKey->getLocalColumns()), + $foreignKey->getForeignTableName(), + implode('_', $foreignKey->getForeignColumns()) + )); if ($this->extractor->isLocalColumnNullable($foreignKey)) { $this->refineExecutor->setNulls($foreignKey); @@ -60,20 +63,20 @@ private function runQueryFor(Schema\ForeignKeyConstraint $foreignKey) */ private function refineTable(string $tabletableName) { + echo(' - refining '.$tabletableName."\n"); /** @var Schema\ForeignKeyConstraint $foreignKey */ foreach ($this->extractor->findForeignKeysReferencingTable($tabletableName) as $foreignKey) { $this->runQueryFor($foreignKey); } + } /** - * @param string $filename + * @param Recipe $recipe * @throws Schema\SchemaException - * @throws \Doctrine\DBAL\DBALException */ - public function refineBasedOnConfig(string $filename) + public function refine(Recipe $recipe) { - $recipe = $this->recipeFactory->createRecipe($filename); /** @var Table $table */ foreach ($recipe->getTables() as $table) { $this->refineIfSubsetted($table); diff --git a/src/Fogger/Schema/SchemaManipulator.php b/src/Fogger/Schema/SchemaManipulator.php index d75f219..5d50962 100644 --- a/src/Fogger/Schema/SchemaManipulator.php +++ b/src/Fogger/Schema/SchemaManipulator.php @@ -41,6 +41,12 @@ public function copySchemaDroppingIndexesAndForeignKeys() private function recreateIndexesOnTable(DBAL\Table $table) { foreach ($table->getIndexes() as $index) { + echo(sprintf( + " - %s's index %s on %s\n", + $table->getName(), + $index->getName(), + implode(', ', $index->getColumns()) + )); $this->targetSchema->createIndex($index, $table->getName()); } /** @var DBAL\Column $column */ @@ -56,6 +62,13 @@ private function recreateIndexesOnTable(DBAL\Table $table) private function recreateForeignKeysOnTable(DBAL\Table $table) { foreach ($table->getForeignKeys() as $fk) { + echo(sprintf( + " - %s.%s => %s.%s\n", + $fk->getLocalTableName(), + implode('_', $fk->getLocalColumns()), + $fk->getForeignTableName(), + implode('_', $fk->getForeignColumns()) + )); $this->targetSchema->createForeignKey($fk, $table->getName()); } } diff --git a/src/Fogger/Subset/AbstratctHeadOrTailSubset.php b/src/Fogger/Subset/AbstratctHeadOrTailSubset.php index e9f21e1..3cc2be6 100644 --- a/src/Fogger/Subset/AbstratctHeadOrTailSubset.php +++ b/src/Fogger/Subset/AbstratctHeadOrTailSubset.php @@ -2,25 +2,25 @@ namespace App\Fogger\Subset; -use App\Fogger\Data\TableQuery; use App\Fogger\Recipe\Table; use App\Fogger\Subset\Exception\SortByColumnRequired; -use Doctrine\DBAL\Query\QueryBuilder; +use Doctrine\Common\Collections\Criteria; +use Doctrine\DBAL\Connection; abstract class AbstratctHeadOrTailSubset extends AbstractSubset { - private $tableQuery; + private $source; - public function __construct(TableQuery $query) + public function __construct(Connection $source) { - $this->tableQuery = $query; + $this->source = $source; } /** * @param Table $table * @throws SortByColumnRequired */ - protected function ensureValidPrimaryKey(Table $table) + protected function ensureSortByColumn(Table $table) { if (null === $table->getSortBy()) { throw new SortByColumnRequired( @@ -32,26 +32,18 @@ protected function ensureValidPrimaryKey(Table $table) } } - private function reverseOrderBy(QueryBuilder $queryBuilder, Table $table) - { - $queryBuilder->resetQueryPart('orderBy'); - $queryBuilder->addOrderBy($table->getSortBy(), 'DESC'); - } - protected function findOffsetId(Table $table, bool $reverse) { $options = $table->getSubset()->getOptions(); - $findOffsetId = $this->tableQuery->getAllRowsQuery($table); - if ($reverse) { - $this->reverseOrderBy($findOffsetId, $table); - } + $findOffsetId = $this->source->createQueryBuilder(); $findOffsetId + ->select($this->source->quoteIdentifier($table->getSortBy())) + ->from($this->source->quoteIdentifier($table->getName())) + ->addOrderBy($table->getSortBy(), $reverse ? Criteria::DESC : Criteria::ASC) ->setFirstResult($options['length'] - 1) ->setMaxResults(1); - $idRow = $findOffsetId->execute()->fetch(); - - return $idRow[$table->getSortBy()]; + return $findOffsetId->execute()->fetchColumn(); } } diff --git a/src/Fogger/Subset/HeadSubset.php b/src/Fogger/Subset/HeadSubset.php index d282d12..1f4cdf1 100644 --- a/src/Fogger/Subset/HeadSubset.php +++ b/src/Fogger/Subset/HeadSubset.php @@ -11,15 +11,16 @@ class HeadSubset extends AbstratctHeadOrTailSubset /** * @param QueryBuilder $queryBuilder * @param Table $table + * @return QueryBuilder * @throws Exception\RequiredOptionMissingException * @throws SortByColumnRequired */ - public function subsetQuery(QueryBuilder $queryBuilder, Table $table) + public function subsetQuery(QueryBuilder $queryBuilder, Table $table): QueryBuilder { $this->ensureOptionIsSet($table->getSubset()->getOptions(), 'length'); - $this->ensureValidPrimaryKey($table); + $this->ensureSortByColumn($table); - $queryBuilder + return $queryBuilder ->andWhere(sprintf('%s <= ?', $table->getSortBy())) ->setParameter(0, $this->findOffsetId($table, false)); } diff --git a/src/Fogger/Subset/NoSubset.php b/src/Fogger/Subset/NoSubset.php index 9919db5..f9445af 100644 --- a/src/Fogger/Subset/NoSubset.php +++ b/src/Fogger/Subset/NoSubset.php @@ -9,8 +9,9 @@ class NoSubset extends AbstractSubset { const STRATEGY_NAME = 'noSubset'; - public function subsetQuery(QueryBuilder $queryBuilder, Table $table) + public function subsetQuery(QueryBuilder $queryBuilder, Table $table): QueryBuilder { + return $queryBuilder; } public function getSubsetStrategyName(): string diff --git a/src/Fogger/Subset/RangeSubset.php b/src/Fogger/Subset/RangeSubset.php index f1d05ca..f77c8c0 100644 --- a/src/Fogger/Subset/RangeSubset.php +++ b/src/Fogger/Subset/RangeSubset.php @@ -21,13 +21,14 @@ private function ensureValidOptions(array $options) /** * @param QueryBuilder $queryBuilder * @param Table $table + * @return QueryBuilder * @throws Exception\RequiredOptionMissingException */ - public function subsetQuery(QueryBuilder $queryBuilder, Table $table) + public function subsetQuery(QueryBuilder $queryBuilder, Table $table): QueryBuilder { $this->ensureValidOptions($options = $table->getSubset()->getOptions()); - $queryBuilder + return $queryBuilder ->where(sprintf('%s >= ?', $options['column'])) ->andWhere(sprintf('%s <= ?', $options['column'])) ->setParameter(0, $options['from']) diff --git a/src/Fogger/Subset/SubsetStrategyInterface.php b/src/Fogger/Subset/SubsetStrategyInterface.php index 3a8a358..66f4f12 100644 --- a/src/Fogger/Subset/SubsetStrategyInterface.php +++ b/src/Fogger/Subset/SubsetStrategyInterface.php @@ -7,7 +7,7 @@ interface SubsetStrategyInterface { - public function subsetQuery(QueryBuilder $queryBuilder, Table $table); + public function subsetQuery(QueryBuilder $queryBuilder, Table $table): QueryBuilder; public function supports(string $name): bool; } diff --git a/src/Fogger/Subset/TailSubset.php b/src/Fogger/Subset/TailSubset.php index dac4100..763bc5e 100644 --- a/src/Fogger/Subset/TailSubset.php +++ b/src/Fogger/Subset/TailSubset.php @@ -10,15 +10,16 @@ class TailSubset extends AbstratctHeadOrTailSubset /** * @param QueryBuilder $queryBuilder * @param Table $table + * @return QueryBuilder * @throws Exception\RequiredOptionMissingException * @throws Exception\SortByColumnRequired */ - public function subsetQuery(QueryBuilder $queryBuilder, Table $table) + public function subsetQuery(QueryBuilder $queryBuilder, Table $table): QueryBuilder { $this->ensureOptionIsSet($table->getSubset()->getOptions(), 'length'); - $this->ensureValidPrimaryKey($table); + $this->ensureSortByColumn($table); - $queryBuilder + return $queryBuilder ->andWhere(sprintf('%s >= ?', $table->getSortBy())) ->setParameter(0, $this->findOffsetId($table, true)); } diff --git a/symfony.lock b/symfony.lock index 55ad198..699b61d 100644 --- a/symfony.lock +++ b/symfony.lock @@ -98,18 +98,6 @@ "ocramius/proxy-manager": { "version": "2.1.1" }, - "php-amqplib/php-amqplib": { - "version": "v2.7.2" - }, - "php-amqplib/rabbitmq-bundle": { - "version": "1.12", - "recipe": { - "repo": "github.com/symfony/recipes-contrib", - "branch": "master", - "version": "1.12", - "ref": "ce3ca2e4577270c4f518539aa6b9e878998fcfef" - } - }, "phpdocumentor/reflection-common": { "version": "1.0.1" },