Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
added stub of oai-pmh importer (not working yet)
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6437 6c8d7289-2bf4-0310-a012-ef5d649a1542
- Loading branch information
orbiter
committed
Oct 30, 2009
1 parent
77c99e5
commit 30f108f
Showing
7 changed files
with
238 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> | ||
| <html xmlns="http://www.w3.org/1999/xhtml"> | ||
| <head> | ||
| <title>YaCy '#[clientname]#': OAI-PMH Import</title> | ||
| #%env/templates/metas.template%# | ||
| #(import)#::<meta http-equiv="REFRESH" content="10" />#(/import)# | ||
| </head> | ||
| <body id="IndexImportOAIPMH"> | ||
| #%env/templates/header.template%# | ||
| #%env/templates/submenuIntegration.template%# | ||
| <h2>OAI-PMH Import</h2> | ||
|
|
||
| #(import)# | ||
| <p>#(status)#No import thread is running, you can start a new thread here::Bad input data: #[message]# #(/status)#</p> | ||
| <form action="IndexImportOAIPMH_p.html" method="get"> | ||
| <fieldset> | ||
| <legend>OAI-PMH Import: set a OAI-PMH URL</legend> | ||
| <input name="oaipmhurl" type="text" value="" size="80" /> | ||
| <input name="submit" type="submit" value="Import from a OAI-PMH source" /> | ||
| </fieldset> | ||
| </form> | ||
| :: | ||
| <form><fieldset><legend>Import Process</legend> | ||
| <dl> | ||
| <dt>Thread:</dt><dd>#[thread]#</dd> | ||
| <dt>Source:</dt><dd>#[source]#</dd> | ||
| <dt>Processed:</dt><dd>#[count]# Wiki Entries</dd> | ||
| <dt>Speed:</dt><dd>#[speed]# articles per second</dd> | ||
| <dt>Running Time:</dt><dd>#[runningHours]# hours, #[runningMinutes]# minutes</dd> | ||
| <dt>Remaining Time:</dt><dd>#[remainingHours]# hours, #[remainingMinutes]# minutes</dd> | ||
| </dl> | ||
| </fieldset></form> | ||
| #(/import)# | ||
|
|
||
| #%env/templates/footer.template%# | ||
| </body> | ||
| </html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| // IndexImportOAIPMH.java | ||
| // ------------------------- | ||
| // (C) 2009 by Michael Peter Christen; mc@yacy.net | ||
| // first published 04.05.2009 on http://yacy.net | ||
| // Frankfurt, Germany | ||
| // | ||
| // $LastChangedDate: 2009-10-11 23:29:18 +0200 (So, 11 Okt 2009) $ | ||
| // $LastChangedRevision: 6400 $ | ||
| // $LastChangedBy: orbiter $ | ||
| // | ||
| // This program is free software; you can redistribute it and/or modify | ||
| // it under the terms of the GNU General Public License as published by | ||
| // the Free Software Foundation; either version 2 of the License, or | ||
| // (at your option) any later version. | ||
| // | ||
| // This program is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU General Public License for more details. | ||
| // | ||
| // You should have received a copy of the GNU General Public License | ||
| // along with this program; if not, write to the Free Software | ||
| // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
|
|
||
| import java.io.File; | ||
| import java.net.MalformedURLException; | ||
|
|
||
| import net.yacy.document.importer.OAIPMHImporter; | ||
| import net.yacy.kelondro.data.meta.DigestURI; | ||
|
|
||
| import de.anomic.http.server.RequestHeader; | ||
| import de.anomic.search.Switchboard; | ||
| import de.anomic.server.serverObjects; | ||
| import de.anomic.server.serverSwitch; | ||
|
|
||
| public class IndexImportOAIPMH_p { | ||
|
|
||
| public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { | ||
| final serverObjects prop = new serverObjects(); | ||
| final Switchboard sb = (Switchboard) env; | ||
|
|
||
| if (OAIPMHImporter.job != null && OAIPMHImporter.job.isAlive()) { | ||
| // one import is running, no option to insert anything | ||
| prop.put("import", 1); | ||
| prop.put("import_thread", "running"); | ||
| prop.put("import_source", OAIPMHImporter.job.source()); | ||
| prop.put("import_count", OAIPMHImporter.job.count()); | ||
| prop.put("import_speed", OAIPMHImporter.job.speed()); | ||
| prop.put("import_runningHours", (OAIPMHImporter.job.runningTime() / 60) / 60); | ||
| prop.put("import_runningMinutes", (OAIPMHImporter.job.runningTime() / 60) % 60); | ||
| prop.put("import_remainingHours", (OAIPMHImporter.job.remainingTime() / 60) / 60); | ||
| prop.put("import_remainingMinutes", (OAIPMHImporter.job.remainingTime() / 60) % 60); | ||
| } else { | ||
| prop.put("import", 0); | ||
| if (post == null) { | ||
| prop.put("import_status", 0); | ||
| } else { | ||
| if (post.containsKey("file")) { | ||
| String oaipmhurl = post.get("oaipmhurl"); | ||
| DigestURI url = null; | ||
| try { | ||
| url = new DigestURI(oaipmhurl, null); | ||
| OAIPMHImporter.job = new OAIPMHImporter(sb.loader, url); | ||
| OAIPMHImporter.job.start(); | ||
| prop.put("import", 1); | ||
| prop.put("import_thread", "started"); | ||
| prop.put("import_dump", OAIPMHImporter.job.source()); | ||
| prop.put("import_count", 0); | ||
| prop.put("import_speed", 0); | ||
| prop.put("import_runningHours", 0); | ||
| prop.put("import_runningMinutes", 0); | ||
| prop.put("import_remainingHours", 0); | ||
| prop.put("import_remainingMinutes", 0); | ||
| } catch (MalformedURLException e) { | ||
| e.printStackTrace(); | ||
| prop.put("import", 0); | ||
| prop.put("import_status", 1); | ||
| prop.put("import_status_message", e.getMessage()); | ||
| } | ||
| } | ||
| return prop; | ||
| } | ||
| } | ||
| return prop; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package net.yacy.document.importer; | ||
|
|
||
| public interface Importer extends Runnable { | ||
|
|
||
|
|
||
| public String source(); | ||
|
|
||
| public int count(); | ||
|
|
||
| /** | ||
| * return the number of articles per second | ||
| * @return | ||
| */ | ||
| public int speed(); | ||
|
|
||
| /** | ||
| * return the time this import is already running | ||
| * @return | ||
| */ | ||
| public long runningTime(); | ||
|
|
||
|
|
||
| /** | ||
| * return the remaining seconds for the completion of all records in milliseconds | ||
| * @return | ||
| */ | ||
| public long remainingTime(); | ||
|
|
||
|
|
||
|
|
||
| public boolean isAlive(); | ||
|
|
||
| public void start(); | ||
|
|
||
| /** | ||
| * the run method from runnable | ||
| */ | ||
| public void run(); | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters