This repository has been archived by the owner on Jun 2, 2023. It is now read-only.
forked from mmerian/phpcrawl
-
Notifications
You must be signed in to change notification settings - Fork 4
/
PHPCrawlerDocumentInfoQueue.php
153 lines (126 loc) · 4.48 KB
/
PHPCrawlerDocumentInfoQueue.php
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
<?php
namespace PHPCrawl\ProcessCommunication;
use Exception;
use PDO;
use PDOStatement;
use PHPCrawl\PHPCrawlerDocumentInfo;
use RuntimeException;
/**
* Queue for PHPCrawlerDocumentInfo-objects
*
* @package phpcrawl
* @internal
*/
class PHPCrawlerDocumentInfoQueue
{
/**
* @var PDO
*/
protected $PDO;
protected $sqlite_db_file;
protected $prepared_statements_created = false;
/**
* Prepared statement for inserting PHPCrawlerDocumentInfo-objects
* @var PDOStatement
*/
protected PDOStatement $preparedInsertStatement;
/**
* Prepared statement for selecting/fetching PHPCrawlerDocumentInfo-objects
* @var PDOStatement
*/
protected PDOStatement $preparedSelectStatement;
protected string $working_directory;
protected int $queue_max_size = 50;
/**
* Initiates a PHPCrawlerDocumentInfoQueue
*
* @param string $file The SQLite-fiel to use.
* @param bool $create_tables Defines whether all necessary tables should be created
* @throws Exception
*/
public function __construct($file, $create_tables = false)
{
$this->sqlite_db_file = $file;
$this->working_directory = dirname($file) . '/';
$this->openConnection($create_tables);
}
/**
* Returns the current number of PHPCrawlerDocumentInfo-objects in the queue
*/
public function getDocumentInfoCount()
{
$Result = $this->PDO->query('SELECT count(id) as sum FROM document_infos;');
$row = $Result->fetch(PDO::FETCH_ASSOC);
$Result->closeCursor();
return $row['sum'];
}
/**
* Adds a PHPCrawlerDocumentInfo-object to the queue
* @param PHPCrawlerDocumentInfo $DocInfo
*/
public function addDocumentInfo(PHPCrawlerDocumentInfo $DocInfo): void
{
// If queue is full -> wait a little
while ($this->getDocumentInfoCount() >= $this->queue_max_size) {
usleep(500000);
}
$this->createPreparedStatements();
$ser = serialize($DocInfo);
$this->PDO->exec('BEGIN EXCLUSIVE TRANSACTION');
$this->preparedInsertStatement->bindParam(1, $ser, PDO::PARAM_STR);
$this->preparedInsertStatement->execute();
$this->preparedSelectStatement->closeCursor();
$this->PDO->exec('COMMIT');
}
/**
* Returns a PHPCrawlerDocumentInfo-object from the queue
*/
public function getNextDocumentInfo()
{
$this->createPreparedStatements();
$this->preparedSelectStatement->execute();
$this->preparedSelectStatement->bindColumn('document_info', $doc_info, PDO::PARAM_STR);
$this->preparedSelectStatement->bindColumn('id', $id);
$row = $this->preparedSelectStatement->fetch(PDO::FETCH_BOUND);
$this->preparedSelectStatement->closeCursor();
if ($id == null) {
return null;
}
$this->PDO->exec('DELETE FROM document_infos WHERE id = ' . $id . ';');
return unserialize($doc_info);
}
/**
* Creates all prepared statemenst
*/
protected function createPreparedStatements(): void
{
if (!$this->prepared_statements_created) {
$this->preparedInsertStatement = $this->PDO->prepare('INSERT INTO document_infos (document_info) VALUES (?);');
$this->preparedSelectStatement = $this->PDO->prepare('SELECT * FROM document_infos limit 1;');
$this->prepared_statements_created = true;
}
}
/**
* Creates the sqlite-db-file and opens connection to it.
*
* @param bool $create_tables Defines whether all necessary tables should be created
* @throws Exception
*/
protected function openConnection($create_tables = false): void
{
// Open sqlite-file
try {
$this->PDO = new PDO('sqlite:' . $this->sqlite_db_file);
} catch (Exception $e) {
throw new RuntimeException('Error creating SQLite-cache-file, ' . $e->getMessage() . ', try installing sqlite3-extension for PHP.');
}
$this->PDO->exec('PRAGMA journal_mode = OFF');
$this->PDO->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_WARNING);
$this->PDO->setAttribute(PDO::ATTR_TIMEOUT, 100);
if ($create_tables) {
$this->PDO->exec('CREATE TABLE IF NOT EXISTS document_infos (id integer PRIMARY KEY AUTOINCREMENT,
document_info blob);');
$this->PDO->exec('ANALYZE;');
}
}
}