Skip to content

Commit

Permalink
Refactor bookmark import using a generic Netscape parser
Browse files Browse the repository at this point in the history 
Relates to shaarli#607
Relates to shaarli#608
Relates to shaarli#493 (abandoned)

Additions:
- use Composer's autoload to load 3rd-party dependencies under vendor/

Modifications:
- [router] use the same endpoint for bookmark upload/import dialog

WIP:
- [import] replace the current parser with a generic, stable parser

TODO:
- [index] display an error if 3rd-party dependencies are missing
- [template] add options for bookmark imports
  - flat vs. nested structure
  - default tags
  - default visibility
- [tests] ensure bookmarks are properly parsed and imported in the LinkDB
  - reuse reference input from the parser's test data

See:
- https://github.com/shaarli/netscape-bookmark-parser
- https://getcomposer.org/doc/01-basic-usage.md#autoloading

Signed-off-by: VirtualTam <virtualtam@flibidi.net>
  • Loading branch information
@virtualtam
virtualtam committed Jul 28, 2016
1 parent efc0c86 commit 35dae13
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 120 deletions.
59 changes: 59 additions & 0 deletions application/NetscapeBookmarkUtils.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,63 @@ public static function filterAndFormat($linkDb, $selection, $prependNoteUrl, $in

return $bookmarkLinks;
}

/**
* Generates a Javascript alert call to display the import status
*
* @param string $filename name of the file to import
* @param int $filesize size of the file to import
* @param int $importCount how many links were imported
*/
private static function generateImportNotification($filename, $filesize, $importCount=0)
{
$alert = '<script>alert("File '.$filename;
$alert .= ' ('.$filesize.' bytes) ';
if ($importCount == 0) {
$alert .= 'has an unknown file format. Nothing was imported.';
} else {
$alert .= 'was successfully processed: '.$importCount.' links imported.';
}
$alert .= '");document.location=\'?\';</script>';
return $alert;
}

/**
* Process the import file form.
*
* @param array $_POST Server POST parameters
* @param array $_FILES Server FILES parameters
* @param LinkDB $linkDb Loaded LinkDB instance.
* @param ConfigManager $conf Configuration Manager instance.
*/
function import($post, $files, $linkDb, $conf)
{
$filename = $files['filetoupload']['name'];
$filesize = $files['filetoupload']['size'];
$data = file_get_contents($files['filetoupload']['tmp_name']);

// Should the links be imported as private?
$private = (empty($post['private']) ? 0 : 1);

// Should the imported links overwrite existing ones?
$overwrite = !empty($post['overwrite']);

$import_count = 0;

// Sniff file type
if (! startsWith($data, '<!DOCTYPE NETSCAPE-Bookmark-file-1>')) {
echo self::generateImportNotification($filename, $filesize);
return;
}

// TODO: parametrize the parser
$parser = new NetscapeBookmarkParser();
$bookmarks = $parser->parseString($data);

foreach ($bookmarks as $bkm) {
// TODO: import bookmarks to $linkDb
}

echo self::generateImportNotification($filename, $filesize, $importCount);
}
}
133 changes: 26 additions & 107 deletions index.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@
//error_reporting(-1);


// 3rd-party libraries
require_once 'inc/rain.tpl.class.php';
require_once __DIR__ . '/vendor/autoload.php';

// Shaarli library
require_once 'application/ApplicationUtils.php';
require_once 'application/Cache.php';
Expand All @@ -64,7 +68,6 @@
require_once 'application/PluginManager.php';
require_once 'application/Router.php';
require_once 'application/Updater.php';
require_once 'inc/rain.tpl.class.php';

// Ensure the PHP version is supported
try {
Expand Down Expand Up @@ -1474,27 +1477,32 @@ function renderPage($conf, $pluginManager)
exit;
}

// -------- User is uploading a file for import
if (isset($_SERVER['QUERY_STRING']) && startsWith($_SERVER['QUERY_STRING'], 'do=upload'))
// Netscape bookmark upload/import page
if ($targetPage == Router::$PAGE_IMPORT)
{
// If file is too big, some form field may be missing.
if (!isset($_POST['token']) || (!isset($_FILES)) || (isset($_FILES['filetoupload']['size']) && $_FILES['filetoupload']['size']==0))
{
$returnurl = ( empty($_SERVER['HTTP_REFERER']) ? '?' : $_SERVER['HTTP_REFERER'] );
echo '<script>alert("The file you are trying to upload is probably bigger than what this webserver can accept ('.getMaxFileSize().' bytes). Please upload in smaller chunks.");document.location=\''.escape($returnurl).'\';</script>';
if (! isset($_POST['token']) || ! isset($_FILES['filetoupload'])) {
// Show import dialog
$PAGE->assign('token', getToken($conf));
$PAGE->assign('maxfilesize', getMaxFileSize());
$PAGE->renderPage('import');
exit;
}
if (!tokenOk($_POST['token'])) die('Wrong token.');
importFile($LINKSDB);
exit;
}

// -------- Show upload/import dialog:
if ($targetPage == Router::$PAGE_IMPORT)
{
$PAGE->assign('token',getToken($conf));
$PAGE->assign('maxfilesize',getMaxFileSize());
$PAGE->renderPage('import');
// Import bookmarks from an uploaded file
if (isset($_FILES['filetoupload']['size']) && $_FILES['filetoupload']['size'] == 0) {
// The file is too big or some form field may be missing.
$returnurl = empty($_SERVER['HTTP_REFERER']) ? '?' : $_SERVER['HTTP_REFERER'];
echo '<script>alert("The file you are trying to upload is probably'
.' bigger than what this webserver can accept ('
.getMaxFileSize().' bytes).'
.' Please upload in smaller chunks.");document.location=\''
.escape($returnurl).'\';</script>';
exit;
}
if (! tokenOk($_POST['token'])) {
die('Wrong token.');
}
NetscapeBookmarkUtils::import($_POST, $_FILES, $LINKSDB);
exit;
}

Expand Down Expand Up @@ -1551,95 +1559,6 @@ function($a, $b) { return $a['order'] - $b['order']; }
exit;
}

/**
* Process the import file form.
*
* @param LinkDB $LINKSDB Loaded LinkDB instance.
* @param ConfigManager $conf Configuration Manager instance.
*/
function importFile($LINKSDB, $conf)
{
if (!isLoggedIn()) { die('Not allowed.'); }

$filename=$_FILES['filetoupload']['name'];
$filesize=$_FILES['filetoupload']['size'];
$data=file_get_contents($_FILES['filetoupload']['tmp_name']);
$private = (empty($_POST['private']) ? 0 : 1); // Should the links be imported as private?
$overwrite = !empty($_POST['overwrite']) ; // Should the imported links overwrite existing ones?
$import_count=0;

// Sniff file type:
$type='unknown';
if (startsWith($data,'<!DOCTYPE NETSCAPE-Bookmark-file-1>')) $type='netscape'; // Netscape bookmark file (aka Firefox).

// Then import the bookmarks.
if ($type=='netscape')
{
// This is a standard Netscape-style bookmark file.
// This format is supported by all browsers (except IE, of course), also Delicious, Diigo and others.
foreach(explode('<DT>',$data) as $html) // explode is very fast
{
$link = array('linkdate'=>'','title'=>'','url'=>'','description'=>'','tags'=>'','private'=>0);
$d = explode('<DD>',$html);
if (startsWith($d[0], '<A '))
{
$link['description'] = (isset($d[1]) ? html_entity_decode(trim($d[1]),ENT_QUOTES,'UTF-8') : ''); // Get description (optional)
preg_match('!<A .*?>(.*?)</A>!i',$d[0],$matches); $link['title'] = (isset($matches[1]) ? trim($matches[1]) : ''); // Get title
$link['title'] = html_entity_decode($link['title'],ENT_QUOTES,'UTF-8');
preg_match_all('! ([A-Z_]+)=\"(.*?)"!i',$html,$matches,PREG_SET_ORDER); // Get all other attributes
$raw_add_date=0;
foreach($matches as $m)
{
$attr=$m[1]; $value=$m[2];
if ($attr=='HREF') $link['url']=html_entity_decode($value,ENT_QUOTES,'UTF-8');
elseif ($attr=='ADD_DATE')
{
$raw_add_date=intval($value);
if ($raw_add_date>30000000000) $raw_add_date/=1000; //If larger than year 2920, then was likely stored in milliseconds instead of seconds
}
elseif ($attr=='PRIVATE') $link['private']=($value=='0'?0:1);
elseif ($attr=='TAGS') $link['tags']=html_entity_decode(str_replace(',',' ',$value),ENT_QUOTES,'UTF-8');
}
if ($link['url']!='')
{
if ($private==1) $link['private']=1;
$dblink = $LINKSDB->getLinkFromUrl($link['url']); // See if the link is already in database.
if ($dblink==false)
{ // Link not in database, let's import it...
if (empty($raw_add_date)) $raw_add_date=time(); // In case of shitty bookmark file with no ADD_DATE

// Make sure date/time is not already used by another link.
// (Some bookmark files have several different links with the same ADD_DATE)
// We increment date by 1 second until we find a date which is not used in DB.
// (so that links that have the same date/time are more or less kept grouped by date, but do not conflict.)
while (!empty($LINKSDB[date('Ymd_His',$raw_add_date)])) { $raw_add_date++; }// Yes, I know it's ugly.
$link['linkdate']=date('Ymd_His',$raw_add_date);
$LINKSDB[$link['linkdate']] = $link;
$import_count++;
}
else // Link already present in database.
{
if ($overwrite)
{ // If overwrite is required, we import link data, except date/time.
$link['linkdate']=$dblink['linkdate'];
$LINKSDB[$link['linkdate']] = $link;
$import_count++;
}
}

}
}
}
$LINKSDB->savedb($conf->get('resource.page_cache'));

echo '<script>alert("File '.json_encode($filename).' ('.$filesize.' bytes) was successfully processed: '.$import_count.' links imported.");document.location=\'?\';</script>';
}
else
{
echo '<script>alert("File '.json_encode($filename).' ('.$filesize.' bytes) has an unknown file format. Nothing was imported.");document.location=\'?\';</script>';
}
}

/**
* Template for the list of links (<div id="linklist">)
* This function fills all the necessary fields in the $PAGE for the template 'linklist.html'
Expand Down
29 changes: 16 additions & 13 deletions tpl/import.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,22 @@
<head>{include="includes"}</head>
<body onload="document.uploadform.filetoupload.focus();">
<div id="pageheader">
{include="page.header"}
<div id="uploaddiv">
Import Netscape HTML bookmarks (as exported from Firefox/Chrome/Opera/Delicious/Diigo...) (Max: {$maxfilesize} bytes).
<form method="POST" action="?do=upload" enctype="multipart/form-data" name="uploadform" id="uploadform">
<input type="hidden" name="token" value="{$token}">
<input type="file" name="filetoupload">
<input type="hidden" name="MAX_FILE_SIZE" value="{$maxfilesize}">
<input type="submit" name="import_file" value="Import" class="bigbutton"><br>
<input type="checkbox" name="private" id="private"><label for="private">&nbsp;Import all links as private</label><br>
<input type="checkbox" name="overwrite" id="overwrite"><label for="overwrite">&nbsp;Overwrite existing links</label>
</form>
</div>
{include="page.header"}
<div id="uploaddiv">
Import Netscape HTML bookmarks (as exported from Firefox/Chrome/Opera/Delicious/Diigo...) (Max: {$maxfilesize} bytes).
<form method="POST" action="?do=import" enctype="multipart/form-data"
name="uploadform" id="uploadform">
<input type="hidden" name="token" value="{$token}">
<input type="file" name="filetoupload">
<input type="hidden" name="MAX_FILE_SIZE" value="{$maxfilesize}">
<input type="submit" name="import_file" value="Import" class="bigbutton"><br>
<input type="checkbox" name="private" id="private">
<label for="private">&nbsp;Import all links as private</label><br>
<input type="checkbox" name="overwrite" id="overwrite">
<label for="overwrite">&nbsp;Overwrite existing links</label>
</form>
</div>
</div>
{include="page.footer"}
</body>
</html>
</html>

0 comments on commit 35dae13

Please sign in to comment.