Skip to content

Commit

Permalink
fix (tableexporter): add inline documentation
Browse files Browse the repository at this point in the history
  • Loading branch information
ondrejhlavacek committed Jun 6, 2014
1 parent 6bdc41e commit 4ae5b23
Showing 1 changed file with 15 additions and 3 deletions.
18 changes: 15 additions & 3 deletions src/Keboola/StorageApi/TableExporter.php
Expand Up @@ -2,7 +2,8 @@
/**
* Storage API Client - Table Exporter
*
*
* Downloads a table from Storage API and saves it to destination path. Merges all parts of a sliced file and adds
* header if missing.
*
* @author Ondrej Hlavacek <ondrej.hlavacek@keboola.com>
* @date: 5.6.14
Expand Down Expand Up @@ -43,16 +44,19 @@ public function exportTable($tableId, $destination, $exportOptions)
$exportOptions['gzip'] = false;
}

// Export table from Storage API to S3
$table = $this->client->getTable($tableId);
$fileId = $this->client->exportTableAsync($tableId, $exportOptions);

$fileInfo = $this->client->getFile($fileId["file"]["id"], (new \Keboola\StorageApi\Options\GetFileOptions())->setFederationToken(true));

// Initialize S3Client with credentials from Storage API
$s3Client = S3Client::factory(array(
"key" => $fileInfo["credentials"]["AccessKeyId"],
"secret" => $fileInfo["credentials"]["SecretAccessKey"],
"token" => $fileInfo["credentials"]["SessionToken"]
));

// Temporary folder to save downloaded files from S3
$tmpFilePath = sys_get_temp_dir() . '/sapi-php-client' . '/' . uniqid('sapi-export-');

$fs = new Filesystem();
Expand All @@ -61,8 +65,12 @@ public function exportTable($tableId, $destination, $exportOptions)
/**
* sliced file - combine files together
*/

// Download manifest with all sliced files
$manifest = json_decode(file_get_contents($fileInfo["url"]), true);
$files = array();

// Download all sliced files
foreach($manifest["entries"] as $part) {
$fileKey = substr($part["url"], strpos($part["url"], '/', 5) + 1);
$filePath = $tmpFilePath . '_' . md5(str_replace('/', '_', $fileKey));
Expand All @@ -74,13 +82,15 @@ public function exportTable($tableId, $destination, $exportOptions)
));
}

// Create file with header
$header = '"' . join($table["columns"], '","') . '"' . "\n";
if ($exportOptions["gzip"] === true) {
$fs->dumpFile($destination . '.tmp', $header);
} else {
$fs->dumpFile($destination, $header);
}

// Concat all files into one, compressed files need to be decompressed first
foreach($files as $file) {
if ($exportOptions["gzip"]) {
$catCmd = "gunzip " . escapeshellarg($file) . " --to-stdout >> " . escapeshellarg($destination) . ".tmp";
Expand All @@ -90,6 +100,8 @@ public function exportTable($tableId, $destination, $exportOptions)
(new Process($catCmd))->mustRun();
$fs->remove($file);
}

// Compress the file afterwards if required
if ($exportOptions["gzip"]) {
$gZipCmd = "gzip " . escapeshellarg($destination) . ".tmp --fast";
(new Process($gZipCmd))->mustRun();
Expand All @@ -98,7 +110,7 @@ public function exportTable($tableId, $destination, $exportOptions)

} else {
/**
* NonSliced file, just move
* NonSliced file, just move from temp to destination file
*/
$s3Client->getObject(array(
'Bucket' => $fileInfo["s3Path"]["bucket"],
Expand Down

0 comments on commit 4ae5b23

Please sign in to comment.