Add two new features: 1) exclude($tableName) 2) outputInChunksOf($chunkSize)

README.md

@@ -17,6 +17,48 @@ composer require beyondcode/laravel-masked-db-dump
 
 The documentation can be found on [our website](https://beyondco.de/docs/laravel-masked-db-dump).
 
+### Exclude tables from the export
+
+Sometimes you might not want to include all tables in the export. You can achieve this with:
+
+```
+return [
+    'default' => DumpSchema::define()
+                    ->allTables()
+                    ->exclude('password_resets')
+                    ->exclude('migrations');
+];
+```
+
+
+### Create INSERTs with multiple rows
+
+When you have a table with many rows (1000+) creating INSERT statements for each row results in a very slow import process.
+For these cases it is better to create INSERT statements with multiple rows.
+
+```
+INSERT INTO table_name (column1, column2, column3, ...)
+VALUES
+    (list of values 1),
+    (list of values 2),
+    (list of values 3),
+    ...
+    (list of values n);
+```
+
+You can achieved this with `->outputInChunksOf($n)`.
+
+```
+return [
+    'default' => DumpSchema::define()
+        ->allTables(),
+        ->table('users', function($table) { 
+                return $table->outputInChunksOf(25); 
+            });
+];
+```
+
+
 ### Changelog
 
 Please see [CHANGELOG](CHANGELOG.md) for more information on what has changed recently.

composer.json

@@ -23,7 +23,7 @@
         "illuminate/support": "^7.0|^8.0"
     },
     "require-dev": {
-        "orchestra/testbench": "^6.12",
+        "orchestra/testbench": "^6.13",
         "phpunit/phpunit": "^8.0",
         "spatie/phpunit-snapshot-assertions": "^4.2"
     },

src/DumpSchema.php

@@ -15,6 +15,7 @@ class DumpSchema
 
     protected $loadAllTables = false;
     protected $customizedTables = [];
+    protected $excludedTables = [];
 
     public function __construct($connectionName = null)
     {
@@ -26,6 +27,12 @@ public static function define($connectionName = null)
         return new static($connectionName);
     }
 
+    public function exclude(string $tableName)
+    {
+        $this->excludedTables[] = $tableName;
+        return $this;
+    }
+
     public function schemaOnly(string $tableName)
     {
         return $this->table($tableName, function (TableDefinition $table) {
@@ -47,6 +54,15 @@ public function allTables()
         return $this;
     }
 
+    // Let's you exclude a table from the list of allTables()
+
+    public function exclude(string $tableName)
+    {
+        $this->excludedTables[] = $tableName; 
+
+        return $this;
+    }
+
     /**
      * @return \Illuminate\Database\ConnectionInterface
      */
@@ -92,7 +108,14 @@ public function load()
         if ($this->loadAllTables) {
             $this->dumpTables = collect($this->availableTables)->mapWithKeys(function (Table $table) {
                 return [$table->getName() => new TableDefinition($table)];
-            })->toArray();
+            });
+
+            $excluded = $this->excludedTables;
+            $this->dumpTables = $this->dumpTables
+                        // filter excluded tables from list of all tables
+                        ->filter(function($table, $tableName) use($excluded) {
+                                    return !in_array($tableName, $excluded);
+                        })->toArray();
         }
 
         foreach ($this->customizedTables as $tableName => $tableDefinition) {

src/LaravelMaskedDump.php

@@ -93,31 +93,83 @@ protected function dumpTableData(TableDefinition $table)
     {
         $query = '';
 
-        $queryBuilder = $this->definition->getConnection()
-            ->table($table->getDoctrineTable()->getName());
+        $queryBuilder = $this->definition->getConnection()->table($table->getDoctrineTable()->getName());
 
         $table->modifyQuery($queryBuilder);
 
-        $queryBuilder->get()
-            ->each(function ($row, $index) use ($table, &$query) {
-                $row = $this->transformResultForInsert((array)$row, $table);
-                $tableName = $table->getDoctrineTable()->getName();
 
-                $query .= "INSERT INTO `${tableName}` (`" . implode('`, `', array_keys($row)) . '`) VALUES ';
-                $query .= "(";
+        if($table->getChunkSize() > 0) {
 
-                $firstColumn = true;
-                foreach ($row as $value) {
-                    if (!$firstColumn) {
-                        $query .= ", ";
-                    }
-                    $query .= $value;
-                    $firstColumn = false;
-                }
+            $data = $queryBuilder->get();
+
+            if($data->isEmpty()) {
+                return "";
+            }
 
-                $query .= ");" . PHP_EOL;
+            $tableName = $table->getDoctrineTable()->getName();
+            $columns = array_keys((array)$data->first());
+            $column_names = "(`" . join('`, `', $columns) . "`)";
+
+            // When tables have 1000+ rows we must split them in reasonably sized chunks of e.g. 100
+            // otherwise the INSERT statement will fail
+            // this returns a collection of value tuples
+
+            $valuesChunks = $data
+                        ->chunk($table->getChunkSize())
+                        ->map(function($chunk) use($table) {
+                                // for each chunk we generate a list of VALUES for the INSERT statement
+                                // (1, 'some 1', 'data A'),
+                                // (2, 'some 2', 'data B'),
+                                // (3, 'some 3', 'data C'),
+                                // ... etc
+
+                                $values = $chunk->map(function($row) use($table) {
+                                            $row = $this->transformResultForInsert((array)$row, $table);
+                                            $query = '(' . join(', ', $row) . ')';
+                                            return $query;
+                                })->join(', ');
+
+                            return $values;
             });
 
-        return $query;
+            // Now we generate the INSERT statements for each chunk of values
+            // INSERT INTO table <list of columns> VALUES (1, 'some 1', 'data A'), (2, 'some 2', 'data B'), (3, 'some 3', 'data C')...
+            $insert_statement = $valuesChunks->map(
+
+                    function($values) use($table, $tableName, $column_names) {
+
+                        return "INSERT INTO `${tableName}` $column_names VALUES " . $values .';';
+
+                    })
+                    ->join(PHP_EOL);
+
+            return $insert_statement . PHP_EOL;
+
+        } else {
+
+            // orig
+            $queryBuilder->get()
+                ->each(function ($row, $index) use ($table, &$query) {
+                    $row = $this->transformResultForInsert((array)$row, $table);
+                    $tableName = $table->getDoctrineTable()->getName();
+
+                    $query .= "INSERT INTO `${tableName}` (`" . implode('`, `', array_keys($row)) . '`) VALUES ';
+                    $query .= "(";
+
+                    $firstColumn = true;
+                    foreach ($row as $value) {
+                        if (!$firstColumn) {
+                            $query .= ", ";
+                        }
+                        $query .= $value;
+                        $firstColumn = false;
+                    }
+
+                    $query .= ");" . PHP_EOL;
+                });
+
+            return $query;
+        }
     }
+
 }

src/TableDefinitions/TableDefinition.php

@@ -15,6 +15,7 @@ class TableDefinition
     protected $dumpType;
     protected $query;
     protected $columns = [];
+    protected $chunkSize = 0;
 
     public function __construct(Table $table)
     {
@@ -36,6 +37,13 @@ public function fullDump()
         return $this;
     }
 
+    public function outputInChunksOf(int $chunkSize)
+    {
+        $this->chunkSize = $chunkSize;
+
+        return $this;
+    }
+
     public function query(callable $callable)
     {
         $this->query = $callable;
@@ -68,6 +76,11 @@ public function findColumn(string $column)
         return false;
     }
 
+    public function getChunkSize()
+    {
+        return $this->chunkSize;
+    }
+
     public function getDoctrineTable()
     {
         return $this->table;

tests/DumperTest.php

@@ -5,7 +5,8 @@
 use BeyondCode\LaravelMaskedDumper\DumpSchema;
 use BeyondCode\LaravelMaskedDumper\LaravelMaskedDumpServiceProvider;
 use BeyondCode\LaravelMaskedDumper\TableDefinitions\TableDefinition;
-use Faker\Generator;
+use Faker\Generator as Faker;
+
 use Illuminate\Auth\Authenticatable;
 use Illuminate\Support\Facades\DB;
 use Orchestra\Testbench\TestCase;
@@ -49,7 +50,8 @@ public function it_can_dump_all_tables_without_modifications()
 
         $this->app['config']['masked-dump.default'] = DumpSchema::define()->allTables();
 
-        $this->artisan('db:dump', [
+        // $this->artisan('db:masked-dump', [
+        $this->artisan('db:masked-dump', [
             'output' => $outputFile
         ]);
 
@@ -78,7 +80,7 @@ public function it_can_mask_user_names()
                 $table->mask('name');
             });
 
-        $this->artisan('db:dump', [
+        $this->artisan('db:masked-dump', [
             'output' => $outputFile
         ]);
 
@@ -107,7 +109,7 @@ public function it_can_replace_columns_with_static_values()
                 $table->replace('password', 'test');
             });
 
-        $this->artisan('db:dump', [
+        $this->artisan('db:masked-dump', [
             'output' => $outputFile
         ]);
 
@@ -132,12 +134,14 @@ public function it_can_replace_columns_with_faker_values()
 
         $this->app['config']['masked-dump.default'] = DumpSchema::define()
             ->allTables()
-            ->table('users', function (TableDefinition $table, Generator $faker) {
-                $faker->seed(1);
-                $table->replace('email', $faker->safeEmail());
+            ->table('users', function (TableDefinition $table) {
+                $table->replace('email', function(Faker $faker) {
+                    $faker->seed(1);
+                    $faker->safeEmail();
+                });
             });
 
-        $this->artisan('db:dump', [
+        $this->artisan('db:masked-dump', [
             'output' => $outputFile
         ]);
 
@@ -165,10 +169,83 @@ public function it_can_dump_certain_tables_as_schema_only()
             ->schemaOnly('migrations')
             ->schemaOnly('users');
 
-        $this->artisan('db:dump', [
+        $this->artisan('db:masked-dump', [
+            'output' => $outputFile
+        ]);
+
+        $this->assertMatchesTextSnapshot(file_get_contents($outputFile));
+    }
+
+    /** @test */
+    public function it_does_remove_excluded_tables_from_allTables()
+    {
+        $this->loadLaravelMigrations();
+
+        DB::table('users')
+            ->insert([
+                'name' => 'Marcel',
+                'email' => 'marcel@beyondco.de',
+                'password' => 'test',
+                'created_at' => '2021-01-01 00:00:00',
+                'updated_at' => '2021-01-01 00:00:00',
+            ]);
+
+        $outputFile = base_path('test.sql');
+
+        $this->app['config']['masked-dump.default'] = DumpSchema::define()
+                            ->allTables()
+                            ->exclude(['users']);
+
+        $this->artisan('db:masked-dump', [
             'output' => $outputFile
         ]);
 
         $this->assertMatchesTextSnapshot(file_get_contents($outputFile));
     }
+
+    /** @test */
+    public function it_creates_chunked_insert_statements_for_a_table()
+    {
+        $this->loadLaravelMigrations();
+
+        DB::table('users')
+            ->insert(['name' => 'Marcel1', 'email' => 'marcel1@beyondco.de', 'password' => 'test',
+                'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
+            ]);
+        DB::table('users')
+            ->insert(['name' => 'Marcel2', 'email' => 'marcel2@beyondco.de', 'password' => 'test',
+                'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
+            ]);
+        DB::table('users')
+            ->insert(['name' => 'Marcel3', 'email' => 'marcel3@beyondco.de', 'password' => 'test',
+                'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
+            ]);
+        DB::table('users')
+            ->insert(['name' => 'Marcel4', 'email' => 'marcel4@beyondco.de', 'password' => 'test',
+                'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
+            ]);
+        DB::table('users')
+            ->insert(['name' => 'Marcel5', 'email' => 'marcel5@beyondco.de', 'password' => 'test',
+                'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
+            ]);
+
+        $outputFile = base_path('test.sql');
+
+        $this->app['config']['masked-dump.default'] = DumpSchema::define()
+                            ->allTables()
+                            ->table('users', function($table) { 
+                                    return $table->outputInChunksOf(3); 
+                                });
+
+        $this->artisan('db:masked-dump', [
+            'output' => $outputFile
+        ]);
+
+        $this->assertMatchesTextSnapshot(file_get_contents($outputFile));
+    }
+
+    public function it_can_exclude_certain_tables_from_allTables()
+    {
+        $this->assertTrue(1===1);
+    }
 }

tests/__snapshots__/DumperTest__it_can_replace_columns_with_faker_values__1.txt

@@ -23,6 +23,6 @@ DROP TABLE IF EXISTS `users`;
 CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL COLLATE BINARY, email VARCHAR(255) NOT NULL COLLATE BINARY, email_verified_at DATETIME DEFAULT NULL, password VARCHAR(255) NOT NULL COLLATE BINARY, remember_token VARCHAR(255) DEFAULT NULL COLLATE BINARY, created_at DATETIME DEFAULT NULL, updated_at DATETIME DEFAULT NULL);CREATE UNIQUE INDEX users_email_unique ON users (email);
 LOCK TABLES `users` WRITE;
 ALTER TABLE `users` DISABLE KEYS;
-INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel', 'monty93@example.net', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00');
+INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel', NULL, NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00');
 ALTER TABLE `users` ENABLE KEYS;
 UNLOCK TABLES;