Skip to content

Commit

Permalink
Built in support for auto-queueing chunked imports
Browse files Browse the repository at this point in the history
  • Loading branch information
patrickbrouwers committed Dec 10, 2015
1 parent 7b638fb commit 9bbcf03
Show file tree
Hide file tree
Showing 11 changed files with 447 additions and 199 deletions.
2 changes: 1 addition & 1 deletion .gitignore
@@ -1,4 +1,4 @@
/vendor
composer.phar
composer.lock
.DS_Store
.DS_Store
5 changes: 3 additions & 2 deletions composer.json
Expand Up @@ -31,12 +31,13 @@
"phpseclib/phpseclib": "~1.0",
"phpunit/phpunit": "~4.0",
"mockery/mockery": "~0.9",
"orchestra/testbench": "3.0.*"
"orchestra/testbench": "3.1.*"
},
"suggest": {
"illuminate/http": "5.0.*|5.1.*|5.2.*",
"illuminate/routing": "5.0.*|5.1.*|5.2.*",
"illuminate/view": "5.0.*|5.1.*|5.2.*"
"illuminate/view": "5.0.*|5.1.*|5.2.*",
"illuminate/queue": "5.0.*|5.1.*|5.2.*"
},
"autoload": {
"classmap": [
Expand Down
20 changes: 20 additions & 0 deletions docs/import/chunk.md
Expand Up @@ -41,3 +41,23 @@ Injected ExcelFile example:
// or return true if you want to stop importing.
});
}

## Queued chunks

We automatically queue every chunk for you, if you have enabled the queue driver in your config.

```
Excel::filter('chunk')->load(storage_path('test2.xls'))->chunk(10, function ($results) {
// This will be handled inside the queue
app('log')->info('Import: ' . count($results));
});
```

If you want to by-pass the behaviour, you can pass `false` as third param of `chunk($size, $callback, $shouldQueue)`.

```
Excel::filter('chunk')->load(storage_path('test2.xls'))->chunk(10, function ($results) {
// This will be handled inside the queue
app('log')->info('Import: ' . count($results));
}, false);
```
1 change: 1 addition & 0 deletions dump.rdb
@@ -0,0 +1 @@
REDIS0006锟杰矯锟絑锟斤拷V
3 changes: 3 additions & 0 deletions phpunit.xml
Expand Up @@ -15,4 +15,7 @@
<directory suffix=".php">./tests/</directory>
</testsuite>
</testsuites>
<php>
<env name="APP_KEY" value="AckfSECXIvnK5r28GVIWUAxmbBSjTsmF"/>
</php>
</phpunit>
114 changes: 114 additions & 0 deletions src/Maatwebsite/Excel/Readers/ChunkedReadJob.php
@@ -0,0 +1,114 @@
<?php

namespace Maatwebsite\Excel\Readers;

use Illuminate\Contracts\Bus\SelfHandling;
use Illuminate\Contracts\Queue\ShouldQueue;
use Maatwebsite\Excel\Filters\ChunkReadFilter;
use SuperClosure\Serializer;

class ChunkedReadJob implements SelfHandling, ShouldQueue
{
/**
* @var int
*/
private $startRow;

/**
* @var callable
*/
private $callback;

/**
* @var
*/
private $chunkSize;

/**
* @var
*/
private $startIndex;

/**
* @var
*/
private $file;

/**
* @var null
*/
private $sheets;

/**
* @var bool
*/
private $shouldQueue;

/**
* ChunkedReadJob constructor.
*
* @param $file
* @param null $sheets
* @param int $startRow
* @param $startIndex
* @param $chunkSize
* @param callable $callback
* @param bool $shouldQueue
*/
public function __construct(
$file,
$sheets = null,
$startRow,
$startIndex,
$chunkSize,
callable $callback,
$shouldQueue = true
) {
$this->startRow = $startRow;
$this->chunkSize = $chunkSize;
$this->startIndex = $startIndex;
$this->file = $file;

$this->callback = $shouldQueue ? (new Serializer)->serialize($callback) : $callback;
$this->sheets = $sheets;
$this->shouldQueue = $shouldQueue;
}

/***
* Handle the read job
*/
public function handle()
{
$reader = app('excel.reader');
$reader->injectExcel(app('phpexcel'));
$reader->_init($this->file);

$filter = new ChunkReadFilter();
$reader->reader->setLoadSheetsOnly($this->sheets);
$reader->reader->setReadFilter($filter);
$reader->reader->setReadDataOnly(true);

// Set the rows for the chunking
$filter->setRows($this->startRow, $this->chunkSize);

// Load file with chunk filter enabled
$reader->excel = $reader->reader->load($this->file);

// Slice the results
$results = $reader->get()->slice($this->startIndex, $this->chunkSize);

$callback = $this->shouldQueue ? (new Serializer)->unserialize($this->callback) : $this->callback;

// Do a callback
if (is_callable($callback)) {
$break = call_user_func($callback, $results);
}

$reader->_reset();
unset($reader, $results);

if ($break) {
return true;
}
}
}

0 comments on commit 9bbcf03

Please sign in to comment.