-
Notifications
You must be signed in to change notification settings - Fork 152
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Make crawlers able to crawl internal hyper links
- Loading branch information
Showing
4 changed files
with
191 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| <?php | ||
| /* | ||
| * @author xuruiqi | ||
| * @date 2015-10-26 | ||
| * @copyright reetsee.com | ||
| * @desc 字典树的简单实现，没有做内存优化 | ||
| * A simple implementation of trie without improvements on memory | ||
| */ | ||
| class Phpfetcher_Util_Trie { | ||
| protected $_arrTrieRoot = array(); | ||
|
|
||
| public function __construct($arrStrings = array()) { | ||
| $this->_arrTrieRoot = array( | ||
| 'children' => array(), | ||
| 'count' => 0, | ||
| ); | ||
| foreach ($arrStrings as $str) { | ||
| $this->insert($str); | ||
| } | ||
| } | ||
|
|
||
| public function insert($str) { | ||
| try { | ||
| $str = strval($str); | ||
| $intLen = strlen($str); | ||
| $arrCurNode = &$this->_arrTrieRoot; | ||
|
|
||
| for ($i = 0; $i < $intLen; ++$i) { | ||
| if (!isset($arrCurNode['children'][$str[$i]])) { | ||
| $arrCurNode['children'][$str[$i]] = array( | ||
| 'children' => array(), | ||
| 'count' => 0, | ||
| ); | ||
| } | ||
| $arrCurNode = &$arrCurNode['children'][$str[$i]]; | ||
| } | ||
|
|
||
| $arrCurNode['count'] += 1; | ||
| unset($arrCurNode); | ||
|
|
||
| } catch (Exception $e) { | ||
| Phpfetcher_Log::fatal($e->getMessage()); | ||
| return false; | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| public function delete($str) { | ||
| $arrCurNode = &$this->_locateNode($str); | ||
| if (!is_null($arrCurNode) && $arrCurNode['count'] > 0) { | ||
| $arrCurNode['count'] -= 1; | ||
| } | ||
| unset($arrCurNode); | ||
| return true; | ||
| } | ||
|
|
||
| public function has($str) { | ||
| $arrTargetNode = &$this->_locateNode($str); | ||
| $bolRes = false; | ||
| if (!is_null($arrTargetNode) && $arrTargetNode['count'] > 0) { | ||
| $bolRes = true; | ||
| } | ||
| unset($arrTargetNode); | ||
| return $bolRes; | ||
| } | ||
|
|
||
| protected function &_locateNode($str) { | ||
| $str = strval($str); | ||
| $intLen = strlen($str); | ||
| $arrCurNode = &$this->_arrTrieRoot; | ||
|
|
||
| for ($i = 0; $i < $intLen; ++$i) { | ||
| if (!isset($arrCurNode['children'][$str[$i]])) { | ||
| return null; | ||
| } | ||
| $arrCurNode = &$arrCurNode['children'][$str[$i]]; | ||
| } | ||
|
|
||
| return $arrCurNode; | ||
| } | ||
|
|
||
| //public function startsWith($str) { | ||
| // $str = strval($str); | ||
| // //TODO | ||
| //} | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| <?php | ||
| $test_include_path = dirname(__FILE__) . '/../'; | ||
| set_include_path(get_include_path() . PATH_SEPARATOR . $test_include_path); | ||
|
|
||
| require_once('phpfetcher.php'); | ||
|
|
||
| function print_trie(&$trie) { | ||
| echo "has ftp:" . var_export($trie->has("ftp"), true) . "\n"; | ||
| echo "has http:" . var_export($trie->has("http"), true) . "\n"; | ||
| echo "has https:" . var_export($trie->has("https"), true) . "\n"; | ||
| echo "\n"; | ||
| } | ||
|
|
||
| $arrSchemes = array( | ||
| "http", | ||
| "https", | ||
| "ftp", | ||
| ); | ||
| $trie = new Phpfetcher_Util_Trie($arrSchemes); | ||
| print_trie($trie); | ||
|
|
||
| echo "delete 'abc'\n"; | ||
| $trie->delete("abc"); | ||
| print_trie($trie); | ||
|
|
||
| echo "delete 'ftp'\n"; | ||
| $trie->delete("ftp"); | ||
| print_trie($trie); | ||
|
|
||
| echo "delete 'http'\n"; | ||
| $trie->delete("http"); | ||
| print_trie($trie); | ||
|
|
||
| echo "insert 'ftp'\n"; | ||
| $trie->insert("ftp"); | ||
| print_trie($trie); | ||
|
|
||
| echo "delete 'https'\n"; | ||
| $trie->delete("https"); | ||
| print_trie($trie); | ||
|
|
||
| echo "insert 'http'\n"; | ||
| $trie->insert("http"); | ||
| print_trie($trie); |