diff --git a/src/Entity/Article.php b/src/Entity/Article.php
index 2c62e0e..6a2ad4d 100644
--- a/src/Entity/Article.php
+++ b/src/Entity/Article.php
@@ -66,7 +66,7 @@ public function getDate()
*/
public function getAuthor()
{
- return (isset($this->data['author'])) ? $this->data['author'] : null;
+ return $this->getOrDefault('author');
}
/**
@@ -75,7 +75,7 @@ public function getAuthor()
*/
public function getAuthorUrl()
{
- return (isset($this->data['authorUrl'])) ? $this->data['authorUrl'] : null;
+ return $this->getOrDefault('authorUrl');
}
/**
@@ -118,7 +118,7 @@ public function getTags()
*/
public function getNumPages()
{
- return (isset($this->data['numPages'])) ? $this->data['numPages'] : 1;
+ return $this->getOrDefault('numPages', 1);
}
/**
@@ -129,7 +129,7 @@ public function getNumPages()
*/
public function getNextPages()
{
- return (isset($this->data['nextPages'])) ? $this->data['nextPages'] : [];
+ return $this->getOrDefault('nextPages', []);
}
/**
@@ -139,7 +139,7 @@ public function getNextPages()
*/
public function getSentiment()
{
- return (isset($this->data['sentiment'])) ? $this->data['sentiment'] : null;
+ return $this->getOrDefault('sentiment');
}
/**
@@ -172,7 +172,7 @@ public function getSentiment()
*/
public function getImages()
{
- return (isset($this->data['images'])) ? $this->data['images'] : [];
+ return $this->getOrDefault('images', []);
}
/**
@@ -199,7 +199,7 @@ public function getImages()
*/
public function getVideos()
{
- return (isset($this->data['videos'])) ? $this->data['videos'] : [];
+ return $this->getOrDefault('videos', []);
}
/**
@@ -210,4 +210,48 @@ public function getDiscussion()
{
return $this->discussion;
}
+
+ /**
+ * The plain-text name of the site (e.g. The New York Times or Diffbot).
+ *
+ * If no site name is automatically determined, the root domain (diffbot.com) will be returned.
+ *
+ * @return string | null
+ */
+ public function getSiteName()
+ {
+ return $this->getOrDefault('siteName');
+ }
+
+ /**
+ * If known, the country of the article publication.
+ *
+ * @return string | null
+ */
+ public function getPublisherCountry()
+ {
+ return $this->getOrDefault('publisherCountry', null);
+ }
+
+ /**
+ * If known, the region of the article publication.
+ *
+ * @return string | null
+ */
+ public function getPublisherRegion()
+ {
+ return $this->getOrDefault('publisherRegion', null);
+ }
+
+ /**
+ * If an article's date is ambiguous, Diffbot will attempt to estimate a
+ * more specific timestamp using various factors. This will not be
+ * generated for articles older than two days, or articles without an identified date.
+ *
+ * @return string | null
+ */
+ public function getEstimatedDate()
+ {
+ return $this->getOrDefault('estimatedDate', $this->getDate());
+ }
}
\ No newline at end of file
diff --git a/src/Traits/StandardEntity.php b/src/Traits/StandardEntity.php
index d3f24c4..10ca851 100644
--- a/src/Traits/StandardEntity.php
+++ b/src/Traits/StandardEntity.php
@@ -113,4 +113,10 @@ public function getDiffbotUri()
return $this->data['diffbotUri'];
}
+ protected function getOrDefault($key, $default = null, $data = null)
+ {
+ $data = ($data !== null) ?: $this->data;
+ return (isset($data[$key]) ? $data[$key] : $default);
+ }
+
}
\ No newline at end of file
diff --git a/tests/Entity/AbstractTest.php b/tests/Entity/AbstractTest.php
index c68db35..f125be9 100644
--- a/tests/Entity/AbstractTest.php
+++ b/tests/Entity/AbstractTest.php
@@ -9,28 +9,10 @@
class AbstractTest extends ResponseProvider
{
- /** @var array */
- protected $responses = [];
-
- protected $files = [
+ protected static $staticFiles = [
'Products/dogbrush.json'
];
- protected function ei($file)
- {
- $ef = new Entity();
- return $ef->createAppropriateIterator($this->prepareResponses()[$file]);
- }
-
- public function returnFiles()
- {
- $files = [];
- foreach ($this->files as $file) {
- $files[] = [$file];
- }
- return $files;
- }
-
public function queryStringProvider()
{
return [
diff --git a/tests/Entity/ArticleTest.php b/tests/Entity/ArticleTest.php
index 6f5a54b..cb30eba 100644
--- a/tests/Entity/ArticleTest.php
+++ b/tests/Entity/ArticleTest.php
@@ -3,42 +3,23 @@
namespace Swader\Diffbot\Test\Entity;
use Swader\Diffbot\Entity\Article;
-use Swader\Diffbot\Factory\Entity;
use Swader\Diffbot\Test\ResponseProvider;
class ArticleTest extends ResponseProvider
{
- /** @var array */
- protected $responses = [];
-
- protected $files = [
+ protected static $staticFiles = [
'Articles/diffbot-sitepoint-basic.json',
// http%3A%2F%2Fwww.sitepoint.com%2Fdiffbot-crawling-visual-machine-learning
'Articles/diffbot-sitepoint-extended.json',
'Articles/apple-watch-verge-basic.json',
// http%3A%2F%2Fwww.theverge.com%2Fa%2Fapple-watch-review
- 'Articles/apple-watch-verge-extended.json'
+ 'Articles/apple-watch-verge-extended.json',
+ 'Articles/15-11-07/diffbot-sitepoint-basic.json',
];
- protected function ei($file)
- {
- $ef = new Entity();
-
- return $ef->createAppropriateIterator($this->prepareResponses()[$file]);
- }
-
- public function returnFiles()
- {
- $files = [];
- foreach ($this->files as $file) {
- $files[] = [$file];
- }
-
- return $files;
- }
-
/**
* @dataProvider returnFiles
+ * @param $file
*/
public function testType($file)
{
@@ -303,4 +284,89 @@ public function testDiscussion($file, $articles)
}
}
}
+
+ public function siteNameProvider()
+ {
+ return [
+ ['Articles/15-11-07/diffbot-sitepoint-basic.json', 'SitePoint'],
+ ];
+ }
+
+ /**
+ * @dataProvider siteNameProvider
+ * @param $file
+ * @param $value1
+ */
+ public function testSiteName($file, $value1)
+ {
+ $value1 = (is_array($value1)) ? $value1 : [$value1];
+ /** @var Article $entity */
+ foreach ($this->ei($file) as $i => $entity) {
+ $this->assertEquals($value1[$i], $entity->getSiteName());
+ }
+ }
+
+ public function publisherCountryProvider()
+ {
+ return [
+ ['Articles/15-11-07/diffbot-sitepoint-basic.json', 'Australia'],
+ ];
+ }
+
+ /**
+ * @dataProvider publisherCountryProvider
+ * @param $file
+ * @param $value1
+ */
+ public function testPublisherCountry($file, $value1)
+ {
+ $value1 = (is_array($value1)) ? $value1 : [$value1];
+ /** @var Article $entity */
+ foreach ($this->ei($file) as $i => $entity) {
+ $this->assertEquals($value1[$i], $entity->getPublisherCountry());
+ }
+ }
+
+ public function publisherRegionProvider()
+ {
+ return [
+ ['Articles/15-11-07/diffbot-sitepoint-basic.json', 'Australia and New Zealand'],
+ ];
+ }
+
+ /**
+ * @dataProvider publisherRegionProvider
+ * @param $file
+ * @param $value1
+ */
+ public function testPublisherRegion($file, $value1)
+ {
+ $value1 = (is_array($value1)) ? $value1 : [$value1];
+ /** @var Article $entity */
+ foreach ($this->ei($file) as $i => $entity) {
+ $this->assertEquals($value1[$i], $entity->getPublisherRegion());
+ }
+ }
+
+ public function estimatedDateProvider()
+ {
+ return [
+ ['Articles/15-11-07/diffbot-sitepoint-basic.json', 'Sun, 27 Jul 2014 00:00:00 GMT'],
+ ];
+ }
+
+ /**
+ * @dataProvider estimatedDateProvider
+ * @param $file
+ * @param $value1
+ */
+ public function testEstimatedDate($file, $value1)
+ {
+ $value1 = (is_array($value1)) ? $value1 : [$value1];
+ /** @var Article $entity */
+ foreach ($this->ei($file) as $i => $entity) {
+ $this->assertEquals($value1[$i], $entity->getEstimatedDate());
+ }
+ }
+
}
diff --git a/tests/Entity/CrawlJobTest.php b/tests/Entity/CrawlJobTest.php
index 37262e3..37f2550 100644
--- a/tests/Entity/CrawlJobTest.php
+++ b/tests/Entity/CrawlJobTest.php
@@ -10,19 +10,16 @@
class CrawlJobTest extends ResponseProvider
{
- /** @var array */
- protected $responses = [];
-
- protected $files = [
+ protected static $staticFiles = [
'Crawlbot/15-05-18/sitepoint_01_maxCrawled.json',
'Crawlbot/15-05-20/multiplejobs01.json'
];
protected function ei($file)
{
- $this->prepareResponses();
+ $responses = parent::prepareResponsesStatic();
/** @var ResponseInterface $response */
- $response = $this->responses[$file];
+ $response = $responses[$file];
$jobs = [];
foreach (json_decode($response->getBody(), true)['jobs'] as $data) {
$jobs[] = new Job($data);
@@ -31,16 +28,6 @@ protected function ei($file)
return new EntityIterator($jobs, $response);
}
- public function returnFiles()
- {
- $files = [];
- foreach ($this->files as $file) {
- $files[] = [$file];
- }
-
- return $files;
- }
-
/**
* @dataProvider returnFiles
*/
diff --git a/tests/Entity/DiscussionTest.php b/tests/Entity/DiscussionTest.php
index 67034f0..52c1c5e 100644
--- a/tests/Entity/DiscussionTest.php
+++ b/tests/Entity/DiscussionTest.php
@@ -8,31 +8,11 @@
class DiscussionTest extends ResponseProvider
{
- /** @var array */
- protected $responses = [];
-
- protected $files = [
+ protected static $staticFiles = [
'Discussions/15-05-01/sp_discourse_php7_recap.json',
//http%3A%2F%2Fcommunity.sitepoint.com%2Ft%2Fphp7-resource-recap%2F174325%2F14
];
- protected function ei($file)
- {
- $ef = new Entity();
-
- return $ef->createAppropriateIterator($this->prepareResponses()[$file]);
- }
-
- public function returnFiles()
- {
- $files = [];
- foreach ($this->files as $file) {
- $files[] = [$file];
- }
-
- return $files;
- }
-
/**
* @dataProvider returnFiles
* @param $file
diff --git a/tests/Entity/ImageTest.php b/tests/Entity/ImageTest.php
index 2fbccdc..8a024ec 100644
--- a/tests/Entity/ImageTest.php
+++ b/tests/Entity/ImageTest.php
@@ -3,36 +3,15 @@
namespace Swader\Diffbot\Test\Entity;
use Swader\Diffbot\Entity\Image;
-use Swader\Diffbot\Factory\Entity;
use Swader\Diffbot\Test\ResponseProvider;
class ImageTest extends ResponseProvider
{
- /** @var array */
- protected $responses = [];
-
- protected $files = [
+ protected static $staticFiles = [
'Images/multi_images_smittenkitchen.json',
'Images/one_image_zola.json',
];
- protected function ei($file)
- {
- $ef = new Entity();
-
- return $ef->createAppropriateIterator($this->prepareResponses()[$file]);
- }
-
- public function returnFiles()
- {
- $files = [];
- foreach ($this->files as $file) {
- $files[] = [$file];
- }
-
- return $files;
- }
-
/**
* @dataProvider returnFiles
*/
diff --git a/tests/Entity/PostTest.php b/tests/Entity/PostTest.php
index 776d52e..8bcadb0 100644
--- a/tests/Entity/PostTest.php
+++ b/tests/Entity/PostTest.php
@@ -3,37 +3,16 @@
namespace Swader\Diffbot\Test\Entity;
use Swader\Diffbot\Entity\Discussion;
-use Swader\Diffbot\Factory\Entity;
use Swader\Diffbot\Test\ResponseProvider;
use Swader\Diffbot\Entity\Post;
class PostTest extends ResponseProvider
{
- /** @var array */
- protected $responses = [];
-
- protected $files = [
+ protected static $staticFiles = [
'Discussions/15-05-01/sp_discourse_php7_recap.json',
//http%3A%2F%2Fcommunity.sitepoint.com%2Ft%2Fphp7-resource-recap%2F174325%2F14
];
- protected function ei($file)
- {
- $ef = new Entity();
-
- return $ef->createAppropriateIterator($this->prepareResponses()[$file]);
- }
-
- public function returnFiles()
- {
- $files = [];
- foreach ($this->files as $file) {
- $files[] = [$file];
- }
-
- return $files;
- }
-
/**
* @dataProvider returnFiles
*/
diff --git a/tests/Entity/ProductTest.php b/tests/Entity/ProductTest.php
index 8ba1846..0f36127 100644
--- a/tests/Entity/ProductTest.php
+++ b/tests/Entity/ProductTest.php
@@ -3,36 +3,15 @@
namespace Swader\Diffbot\Test\Entity;
use Swader\Diffbot\Entity\Product;
-use Swader\Diffbot\Factory\Entity;
use Swader\Diffbot\Test\ResponseProvider;
class ProductTest extends ResponseProvider
{
- /** @var array */
- protected $responses = [];
-
- protected $files = [
+ protected static $staticFiles = [
'Products/dogbrush.json',
'Products/15-05-03/shoes-sportsdirect.json'
];
- protected function ei($file)
- {
- $ef = new Entity();
-
- return $ef->createAppropriateIterator($this->prepareResponses()[$file]);
- }
-
- public function returnFiles()
- {
- $files = [];
- foreach ($this->files as $file) {
- $files[] = [$file];
- }
-
- return $files;
- }
-
/**
* @dataProvider returnFiles
*/
diff --git a/tests/Entity/WildCardTest.php b/tests/Entity/WildCardTest.php
index e1be13b..1cd0c55 100644
--- a/tests/Entity/WildCardTest.php
+++ b/tests/Entity/WildCardTest.php
@@ -3,35 +3,14 @@
namespace Swader\Diffbot\Test\Entity;
use Swader\Diffbot\Entity\Wildcard;
-use Swader\Diffbot\Factory\Entity;
use Swader\Diffbot\Test\ResponseProvider;
class WildCardTest extends ResponseProvider
{
- /** @var array */
- protected $responses = [];
-
- protected $files = [
+ protected static $staticFiles = [
'Custom/AuthorFolioNew/15-05-03/bskvorc.json',
];
- protected function ei($file)
- {
- $ef = new Entity();
-
- return $ef->createAppropriateIterator($this->prepareResponses()[$file]);
- }
-
- public function returnFiles()
- {
- $files = [];
- foreach ($this->files as $file) {
- $files[] = [$file];
- }
-
- return $files;
- }
-
public function customFieldProvider()
{
return [
@@ -57,7 +36,7 @@ public function testCustomFields($file, $posts)
foreach ($this->ei($file) as $i => $entity) {
$property = $posts[$i][0];
- $method = 'get'.ucfirst($property);
+ $method = 'get' . ucfirst($property);
$value = $posts[$i][1];
if (!isset($posts[$i][2])) {
diff --git a/tests/Mocks/Articles/15-11-07/diffbot-sitepoint-basic.json b/tests/Mocks/Articles/15-11-07/diffbot-sitepoint-basic.json
new file mode 100644
index 0000000..9ec617b
--- /dev/null
+++ b/tests/Mocks/Articles/15-11-07/diffbot-sitepoint-basic.json
@@ -0,0 +1,2 @@
+
+{"request":{"options":["mentos"],"pageUrl":"http://www.sitepoint.com/diffbot-crawling-visual-machine-learning/","api":"article","version":3},"objects":[{"date":"Sun, 27 Jul 2014 00:00:00 GMT","images":[{"naturalHeight":727,"width":749,"diffbotUri":"image|3|-851701004","url":"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624455201.png","naturalWidth":1063,"primary":true,"height":512},{"naturalHeight":216,"width":523,"diffbotUri":"image|3|762494522","url":"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624487602.png","naturalWidth":523,"height":216},{"naturalHeight":184,"width":664,"diffbotUri":"image|3|302236938","url":"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624509003.png","naturalWidth":664,"height":184},{"naturalHeight":972,"width":749,"diffbotUri":"image|3|-1836356546","url":"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624552704.png","naturalWidth":966,"height":753},{"naturalHeight":184,"width":749,"diffbotUri":"image|3|1297360030","url":"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624650505.png","naturalWidth":918,"height":150},{"naturalHeight":188,"width":749,"diffbotUri":"image|3|502449852","url":"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624665606.png","naturalWidth":929,"height":151},{"naturalHeight":237,"width":749,"diffbotUri":"image|3|-2007985802","url":"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624803507.png","naturalWidth":1053,"height":168},{"naturalHeight":604,"width":749,"diffbotUri":"image|3|-140134863","url":"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624809508.png","naturalWidth":906,"height":499},{"naturalHeight":727,"width":749,"diffbotUri":"image|3|1129235416","url":"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624834309.png","naturalWidth":1063,"height":512}],"author":"Bruno Skvorc","estimatedDate":"Sun, 27 Jul 2014 00:00:00 GMT","publisherRegion":"Australia and New Zealand","icon":"http://www.sitepoint.com/wp-content/themes/sitepoint/assets/images/apple-touch-icon-144x144-precomposed.png","diffbotUri":"article|3|-1226129584","siteName":"SitePoint","type":"article","title":"Diffbot: Crawling with Visual Machine Learning","publisherCountry":"Australia","humanLanguage":"en","authorUrl":"http://www.sitepoint.com/author/bskvorc/","metaTags":[{"name":"ai"},{"name":"Artificial Intelligence"},{"name":"crawling"},{"name":"Diffbot"},{"name":"framework"},{"name":"laravel"},{"name":"machine learning"},{"name":"OOPHP"},{"name":"PHP"},{"name":"scraping"},{"name":"visual learning"}],"pageUrl":"http://www.sitepoint.com/diffbot-crawling-visual-machine-learning/","html":"
Have you ever wondered how social networks do URL previews so well when you share links? How do they know which images to grab, whom to cite as an author, or which tags to attach to the preview? Is it all crawling with complex regexes over source code? Actually, more often than not, it isn’t. Meta information defined in the source can be unreliable, and sites with less than stellar reputation often use them as keyword carriers, attempting to get search engines to rank them higher. Isn’t what we, the humans, see in front of us what matters anyway?<\/p>\n
![\"\"](\"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624455201.png\")
<\/img><\/figure>\n![\"\"](\"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624487602.png\")
<\/img><\/figure>\n![\"\"](\"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624509003.png\")
<\/img><\/figure>\n![\"\"](\"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624552704.png\")
<\/img><\/figure>\n![\"\"](\"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624650505.png\")
<\/img><\/figure>\n![\"\"](\"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624665606.png\")
<\/img><\/figure>\n![\"\"](\"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624803507.png\")
<\/img><\/figure>\n![\"\"](\"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624809508.png\")
<\/img><\/figure>\n![\"\"](\"http://dab1nmslvvntp.cloudfront.net/wp-content/uploads/2014/07/140624834309.png\")
<\/img><\/figure>\n