added ability to use local files as temporary tables

the-tinderbox · Jun 18, 2017 · 97d2698 · 97d2698
1 parent f61e173
commit 97d2698
Show file tree

Hide file tree

Showing 11 changed files with 215 additions and 65 deletions.
diff --git a/README.md b/README.md
@@ -179,6 +179,31 @@ $buulder->from('table')->leftJoin('table', 'any', ['column']);
 $buulder->from('table')->innerJoin('table', 'all', ['column']);
 ```
 
+### Temporary tables usage
+
+There are some cases when you need to filter f.e. users by their ids, but amount of ids is huge. You can
+store users ids in local file, upload it to server and use it as temporary table.
+
+```php
+/*
+ * Add file with users ids to builder as _users table
+ * Also, we must define data structure in file. In example below
+ * structure will be like ['UInt64']
+ */
+$builder->addFile('users.csv', '_users', ['UInt64']);
+$builder->select(raw('count()'))->from('clicks')->whereIn('userId', new Identifier('_users'));
+```
+
+Will produce:
+
+```sql
+SELECT count() FROM `clicks` WHERE `userId` IN `_users`
+```
+
+**If you want tables to be detected automatically, call `addFile` method before calling `whereIn`.**
+
+You can use local files in `whereIn`, `prewhereIn`, `havingIn` and `join` statements of query builder.
+
 ### Prewhere, where, having
 All example will be about where, but same behavior also is for prewhere and having. 
 
@@ -375,7 +400,7 @@ SELECT * FROM `table` UNION ALL SELECT `column1` FROM `table` UNION ALL SELECT `
 ### Performing request and getting result.
 
 After building request you must call `get()` method for sending request to the server.
-Also there has opportunity to make asynchronous requests. Its works almost like `unitAll`.
+Also there has opportunity to make asynchronous requests. Its works almost like `unionAll`.
 
 ```php
 $builder->from('table')->asyncWithQuery(function($query) {

diff --git a/README_RU.md b/README_RU.md
@@ -203,6 +203,32 @@ $buulder->from('table')->leftJoin('table', 'any', ['column']);
 $buulder->from('table')->innerJoin('table', 'all', ['column']);
 ```
 
+### Temporary tables usage
+
+Бывают случаи когда вам требуется отфильтровать данные, например пользователей по их идентификаторам, но кол-во
+идентификаторов исчисляется миллионами. Вы можете разместить их в локальном файле и использовать его в качестве
+временной таблицы на сервере.
+
+```php
+/*
+ * Добавим файл с идентификаторами пользователей как таблицу _users
+ * Так же мы должны указать структуру таблицы. В примере ниже
+ * структура таблицы будет ['UInt64']
+ */
+$builder->addFile('users.csv', '_users', ['UInt64']);
+$builder->select(raw('count()'))->from('clicks')->whereIn('userId', new Identifier('_users'));
+```
+
+Конечный запрос будет таким:
+
+```sql
+SELECT count() FROM `clicks` WHERE `userId` IN `_users`
+```
+
+**Если вы хотите, что бы таблицы подхватывались автоматически, то вам следует вызывать `addFile` до того как вы вызываете `whereIn`.**
+
+Локальные файлы можно так же использовать в `whereIn`, `prewhereIn`, `havingIn` и `join` методах билдера.
+
 ### Prewhere, where, having
 
 Условия фильтрации данных prewhere, where и having. Все примеры описаны для where, но prewhere и having имеют

diff --git a/src/Exceptions/BuilderException.php b/src/Exceptions/BuilderException.php
@@ -8,4 +8,9 @@ public static function cannotDetermineAliasForColumn()
     {
         return new static('Cannot determine alias for the column');
     }
+
+    public static function temporaryTableAlreadyExists($tableName)
+    {
+        return new static("Temporary table {$tableName} already exists in query");
+    }
 }
diff --git a/src/Integrations/Laravel/Builder.php b/src/Integrations/Laravel/Builder.php
@@ -37,7 +37,7 @@ public function get()
         if (!empty($this->async)) {
             return $this->connection->selectAsync($this->toAsyncSqls());
         } else {
-            return $this->connection->select($this->toSql());
+            return $this->connection->select($this->toSql(), [], $this->getFiles());
         }
     }
 

diff --git a/src/Integrations/Laravel/Connection.php b/src/Integrations/Laravel/Connection.php
@@ -269,11 +269,11 @@ public function setClient(Client $client) : self
      *
      * @param string $query
      * @param array  $bindings
-     * @param bool   $useReadPdo
+     * @param array $tables
      *
      * @return array
      */
-    public function select($query, $bindings = [], $useReadPdo = true)
+    public function select($query, $bindings = [], $tables = [])
     {
         $result = $this->getClient()->select($query, $bindings);
 

diff --git a/src/Query/BaseBuilder.php b/src/Query/BaseBuilder.php
@@ -3,6 +3,8 @@
 namespace Tinderbox\ClickhouseBuilder\Query;
 
 use Closure;
+use Tinderbox\Clickhouse\Common\TempTable;
+use Tinderbox\ClickhouseBuilder\Exceptions\BuilderException;
 use Tinderbox\ClickhouseBuilder\Query\Enums\Format;
 use Tinderbox\ClickhouseBuilder\Query\Enums\JoinStrict;
 use Tinderbox\ClickhouseBuilder\Query\Enums\JoinType;
@@ -115,6 +117,13 @@ abstract class BaseBuilder
      * @var array
      */
     protected $async = [];
+
+    /**
+     * Files which should be sent on server to store into temporary table
+     *
+     * @var array
+     */
+    protected $files = [];
 
     /**
      * Set columns for select statement.
@@ -739,6 +748,8 @@ public function preWhereIn($column, $values, $boolean = Operator::AND, $not = fa
 
         if (is_array($values)) {
             $values = new Tuple($values);
+        } elseif (is_string($values) && isset($this->files[$values])) {
+            $values = new Identifier($values);
         }
 
         return $this->preWhere($column, $type, $values, $boolean);
@@ -980,6 +991,8 @@ public function whereIn($column, $values, $boolean = Operator::AND, $not = false
 
         if (is_array($values)) {
             $values = new Tuple($values);
+        } elseif (is_string($values) && isset($this->files[$values])) {
+            $values = new Identifier($values);
         }
 
         return $this->where($column, $type, $values, $boolean);
@@ -1001,6 +1014,8 @@ public function whereGlobalIn($column, $values, $boolean = Operator::AND, $not =
 
         if (is_array($values)) {
             $values = new Tuple($values);
+        } elseif (is_string($values) && isset($this->files[$values])) {
+            $values = new Identifier($values);
         }
 
         return $this->where($column, $type, $values, $boolean);
@@ -1250,6 +1265,8 @@ public function havingIn($column, $values, $boolean = Operator::AND, $not = fals
 
         if (is_array($values)) {
             $values = new Tuple($values);
+        } elseif (is_string($values) && isset($this->files[$values])) {
+            $values = new Identifier($values);
         }
 
         return $this->having($column, $type, $values, $boolean);
@@ -1651,7 +1668,9 @@ public function toSql() : string
      */
     public function toAsyncSqls() : array
     {
-        return $this->grammar->compileAsyncQueries($this);
+        return array_map(function ($query) {
+            return [$query->toSql(), [], $query->getFiles()];
+        }, $this->flatAsyncQueries($this));
     }
 
     /**
@@ -1793,4 +1812,59 @@ public function getAsync() : array
     {
         return $this->async;
     }
+
+    /**
+     * Add file which should be sent on server
+     *
+     * @param string      $filePath
+     * @param string      $tableName
+     * @param array       $structure
+     * @param string|null $format
+     *
+     * @return self
+     *
+     * @throws BuilderException
+     */
+    public function addFile(string $filePath, string $tableName, array $structure, string $format = Format::CSV) : self
+    {
+        if (isset($this->files[$tableName])) {
+            throw BuilderException::temporaryTableAlreadyExists($tableName);
+        }
+
+        $this->files[$tableName] = new TempTable($tableName, $filePath, $structure, $format);
+
+        return $this;
+    }
+
+    /**
+     * Returns files which should be sent on server
+     *
+     * @return array
+     */
+    public function getFiles(): array
+    {
+        return $this->files;
+    }
+
+    /**
+     * Gather all builders from builder. Including nested in async builders.
+     *
+     * @param BaseBuilder $builder
+     *
+     * @return array
+     */
+    public function flatAsyncQueries(BaseBuilder $builder) : array
+    {
+        $result = [];
+
+        foreach ($builder->getAsync() as $query) {
+            if (!empty($query->getAsync())) {
+                $result = array_merge($result, $this->flatAsyncQueries($query));
+            } else {
+                $result[] = $query;
+            }
+        }
+
+        return array_merge([$builder], $result);
+    }
 }
diff --git a/src/Query/Builder.php b/src/Query/Builder.php
@@ -36,7 +36,7 @@ public function get()
         if (!empty($this->async)) {
             return $this->client->selectAsync($this->toAsyncSqls());
         } else {
-            return $this->client->select($this->toSql());
+            return $this->client->select($this->toSql(), [], $this->getFiles());
         }
     }
 

diff --git a/src/Query/Grammar.php b/src/Query/Grammar.php
@@ -176,40 +176,4 @@ public function wrap($value)
             return;
         }
     }
-
-    /**
-     * Gather all builders from builder. Including nested in async builders.
-     *
-     * @param BaseBuilder $builder
-     *
-     * @return array
-     */
-    private function flatAsyncQueries(BaseBuilder $builder) : array
-    {
-        $result = [];
-
-        foreach ($builder->getAsync() as $query) {
-            if (!empty($query->getAsync())) {
-                $result = array_merge($result, $this->flatAsyncQueries($query));
-            } else {
-                $result[] = $query;
-            }
-        }
-
-        return array_merge([$builder], $result);
-    }
-
-    /**
-     * Gather all builders from builder on any nested level, and return array of sqls from all that builders.
-     *
-     * @param BaseBuilder $builder
-     *
-     * @return array
-     */
-    public function compileAsyncQueries(BaseBuilder $builder) : array
-    {
-        return array_map(function ($query) {
-            return $query->toSql();
-        }, $this->flatAsyncQueries($builder));
-    }
 }