Skip to content

Commit

Permalink
Update scraper.php
Browse files Browse the repository at this point in the history
  • Loading branch information
OmairXcluesiv committed Feb 3, 2017
1 parent eca0abc commit 8d6bea1
Showing 1 changed file with 6 additions and 39 deletions.
45 changes: 6 additions & 39 deletions scraper.php
Expand Up @@ -2,16 +2,16 @@
// This is a template for a PHP scraper on morph.io (https://morph.io)
// including some code snippets below that you should find helpful

// require 'scraperwiki.php';
// require 'scraperwiki/simple_html_dom.php';
require 'scraperwiki.php';
require 'scraperwiki/simple_html_dom.php';
//
// // Read in a page
// $html = scraperwiki::scrape("http://foo.com");
$html = scraperwiki::scrape("https://www.olx.com.pk/cars/");
//
// // Find something on the page using css selectors
// $dom = new simple_html_dom();
// $dom->load($html);
// print_r($dom->find("table.list"));
$dom = new simple_html_dom();
$dom->load($html);
print_r($dom->find("large lheight20 margintop10"));
//
// // Write out to the sqlite database using scraperwiki library
// scraperwiki::save_sqlite(array('name'), array('name' => 'susan', 'occupation' => 'software developer'));
Expand All @@ -25,39 +25,6 @@
// called "data.sqlite" in the current working directory which has at least a table
// called "data".

$db = new PDO('sqlite:data.sqlite');
$db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);

try {
$db->query('CREATE TABLE data(
guid VARCHAR(100),
description TEXT,
title VARCHAR(100),
article_timestamp VARCHAR(10),
PRIMARY KEY (guid))');
} catch (Exception $e) {
}


$articles = array(array('guid' => "3", 'description' => 'this is a test', 'title' => 'this is a title', 'article_timestamp' => 'date'));


foreach ($articles as $article) {
$exists = $db->query("SELECT * FROM data WHERE guid = " . $db->quote($article->guid))->fetchObject();
if (!$exists) {
$sql = "INSERT INTO data(guid, description, title, article_timestamp) VALUES(:guid, :description, :title, :article_timestamp)";
} else {
$sql = "UPDATE data SET description = :description, article_timestamp = :article_timestamp WHERE guid = :guid";
}
$statement = $db->prepare($sql);
$statement->execute(array(
':guid' => $article['guid'],
':description' => $article['description'],
':title' => $article['title'],
':article_timestamp' => $article['article_timestamp']
));
}



?>

0 comments on commit 8d6bea1

Please sign in to comment.