Skip to content

Commit

Permalink
Merge pull request #5 from richard-muvirimi/development
Browse files Browse the repository at this point in the history
Update dependencies, update crawl status
  • Loading branch information
richard-muvirimi committed Jun 3, 2023
2 parents 47718ef + 53015e1 commit 7f8e85e
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 26 deletions.
42 changes: 21 additions & 21 deletions app/Entities/Rate.php
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ class Rate extends Entity
* @var string[]
*/
protected $casts = [
'rate' => 'float',
'enabled' => 'boolean',
'status' => 'boolean',
'rate' => 'float',
'enabled' => 'boolean',
'status' => 'boolean',
'javascript' => 'boolean',
];

Expand All @@ -54,9 +54,9 @@ class Rate extends Entity
* @var string[]
*/
protected $datamap = [
'lastChecked' => 'last_checked',
'lastChecked' => 'last_checked',
'lastUpdatedSelector' => 'last_updated_selector',
'lastUpdated' => 'last_updated',
'lastUpdated' => 'last_updated',
];

/**
Expand Down Expand Up @@ -86,12 +86,12 @@ public function crawlSite(): void
/**
* Get site html file and scan for required fields
*/
if (! $this->site) {
if (!$this->site) {
$this->getHtmlContent();
}

if ($this->site) {
$this->status = $this->parseHtml($this->site) === true ? 1 : 0;
$this->status = $this->parseHtml($this->site) === true;

//last checked
$this->lastChecked = Time::now();
Expand Down Expand Up @@ -138,7 +138,7 @@ private function parseHtml(string $html): bool
$converter = new CssSelectorConverter();

$selector = $this->selector;
if (! $this->isXpath($selector)) {
if (!$this->isXpath($selector)) {
$selector = $converter->toXPath($selector);
}

Expand All @@ -151,7 +151,7 @@ private function parseHtml(string $html): bool
$this->rate = $rate;

$lastUpdatedSelector = $this->lastUpdatedSelector;
if (! $this->isXpath($lastUpdatedSelector)) {
if (!$this->isXpath($lastUpdatedSelector)) {
$lastUpdatedSelector = $converter->toXPath($lastUpdatedSelector);
}

Expand Down Expand Up @@ -188,7 +188,7 @@ private function fixDateOffset(Time $date): Time
/**
* Convert number to an int
*
* @param string $value Rate.
* @param string $value Rate.
* @param string $locale Locale to parse number
*
* @return float
Expand All @@ -200,7 +200,7 @@ private function cleanRate(string $value, string $locale): float
{
$amount = $this->clean($value);

if (! is_numeric($amount)) {
if (!is_numeric($amount)) {
$words = explode(' ', $amount);

//remove non-numeric words
Expand All @@ -213,7 +213,7 @@ private function cleanRate(string $value, string $locale): float
//join to allow removing non-numeric characters
$numbers = implode(' ', $numbered);

$fmt = new NumberFormatter($locale, NumberFormatter::DECIMAL);
$fmt = new NumberFormatter($locale, NumberFormatter::DECIMAL);
$numbers = $fmt->parse($numbers);

//split by non-numeric
Expand Down Expand Up @@ -300,7 +300,7 @@ private function cleanDate(string $value): Time
/**
* Return parsed date substituting with defaults on none existent parts
*/
// phpcs:ignore Generic.Files.LineLength.TooLong
// phpcs:ignore Generic.Files.LineLength.TooLong
return Time::create($date['year'] ?: date('Y'), $date['month'] ?: date('n'), $date['day'] ?: date('j'), $date['hour'] ?: 0, $date['minute'] ?: 0, $date['second'] ?: 0);
}

Expand Down Expand Up @@ -340,19 +340,19 @@ public function getHtmlContent(): void
$client = Services::curlrequest();

$data = [
'url' => $this->url,
'format' => 'html',
'timeout' => getenv('scrappy.timeout'),
'url' => $this->url,
'format' => 'html',
'timeout' => getenv('scrappy.timeout'),
'user_agent' => $this->getUserAgent(),
'css' => 'body',
'css' => 'body',
];

try {
$response = $client->post(getenv('scrappy.server') . '/scrape', [
'user_agent' => $this->getUserAgent(),
'multipart' => $data,
'verify' => false,
'headers' => [
'multipart' => $data,
'verify' => false,
'headers' => [
'Authorization' => 'Bearer ' . getenv('scrappy.authKey'),
],
]);
Expand Down Expand Up @@ -383,7 +383,7 @@ private function getUserAgent(): string
{
$agent = cache('user-agent');

if (! $agent) {
if (!$agent) {
$agent = getenv("USER_AGENT");

$agent = preg_replace('/headless/i', '', $agent);
Expand Down
10 changes: 5 additions & 5 deletions composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,22 @@
"php": ">=8.1",
"codeigniter4/framework": "^4.3",
"symfony/polyfill-php82": "^1.27",
"webonyx/graphql-php": "^15.4",
"webonyx/graphql-php": "^15.5",
"ext-json": "*",
"ext-intl": "*",
"symfony/dom-crawler": "^6.2",
"symfony/css-selector": "^6.2",
"symfony/dom-crawler": "^6.3",
"symfony/css-selector": "^6.3",
"br33f/php-ga4-mp": "^0.1.3"
},
"require-dev": {
"fakerphp/faker": "^1.21",
"fakerphp/faker": "^1.22",
"mikey179/vfsstream": "^1.6",
"phpunit/phpunit": "^9.6",
"squizlabs/php_codesniffer": "^3.7",
"phpcompatibility/php-compatibility": "^9.3",
"dealerdirect/phpcodesniffer-composer-installer": "^1.0",
"inpsyde/composer-assets-compiler": "^3.0",
"friendsofphp/php-cs-fixer": "^3.16"
"friendsofphp/php-cs-fixer": "^3.17"
},
"suggest": {
"ext-fileinfo": "Improves mime type detection for files"
Expand Down

0 comments on commit 7f8e85e

Please sign in to comment.