- added ViewProfile / Impressum (default on) to local peer's robots.txt

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3874 6c8d7289-2bf4-0310-a012-ef5d649a1542
yacy · Jun 12, 2007 · 669f840 · 669f840
1 parent 9470090
commit 669f840
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 2 deletions.
diff --git a/htroot/ConfigRobotsTxt_p.html b/htroot/ConfigRobotsTxt_p.html
@@ -41,6 +41,8 @@ <h2>Exclude Web-Spiders</h2>
           <dd><input type="checkbox" id="homepage" name="homepage"#(homepage.checked)#:: checked="checked"#(/homepage.checked)# /></dd>
           <dt><label for="fileshare">File Share</label></dt>
           <dd><input type="checkbox" id="fileshare" name="fileshare"#(fileshare.checked)#:: checked="checked"#(/fileshare.checked)# /></dd>
+          <dt><label for="profile">Impressum</label></dt>
+          <dd><input type="checkbox" id="profile" name="profile"#(profile.checked)#:: checked="checked"#(/profile.checked)# /></dd>
           <dt>&nbsp;</dt>
           <dd><input type="submit" name="save" value="Save restrictions" /></dd>
         </dl>

diff --git a/htroot/ConfigRobotsTxt_p.java b/htroot/ConfigRobotsTxt_p.java
@@ -78,6 +78,7 @@ public static servletProperties respond(httpHeader header, serverObjects post, s
                 rbc.setStatusDisallowed(post.containsKey(httpdRobotsTxtConfig.STATUS));
                 rbc.setSurftipsDisallowed(post.containsKey(httpdRobotsTxtConfig.SURFTIPS));
                 rbc.setWikiDisallowed(post.containsKey(httpdRobotsTxtConfig.WIKI));
+                rbc.setProfileDisallowed(post.containsKey(httpdRobotsTxtConfig.PROFILE));
                 env.setConfig(plasmaSwitchboard.ROBOTS_TXT, rbc.toString());
             }
         }
@@ -94,6 +95,7 @@ public static servletProperties respond(httpHeader header, serverObjects post, s
         prop.put(httpdRobotsTxtConfig.STATUS + ".checked", (rbc.isStatusDisallowed()) ? 1 : 0);
         prop.put(httpdRobotsTxtConfig.SURFTIPS + ".checked", (rbc.isSurftipsDisallowed()) ? 1 : 0);
         prop.put(httpdRobotsTxtConfig.WIKI + ".checked", (rbc.isWikiDisallowed()) ? 1 : 0);
+        prop.put(httpdRobotsTxtConfig.PROFILE + ".checked", (rbc.isProfileDisallowed()) ? 1 : 0);
         return prop;
     }
 }
diff --git a/htroot/robots.java b/htroot/robots.java
@@ -28,6 +28,7 @@ public static servletProperties respond(httpHeader header, serverObjects post, s
             if (rbc.isStatusDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.STATUS, 1);
             if (rbc.isSurftipsDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.SURFTIPS, 1);
             if (rbc.isWikiDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.WIKI, 1);
+            if (rbc.isProfileDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.PROFILE, 1);
 
             if (rbc.isLockedDisallowed() || rbc.isDirsDisallowed()) {
                 final ArrayList[] p = getFiles(env.getConfig(plasmaSwitchboard.HTROOT_PATH, plasmaSwitchboard.HTROOT_PATH_DEFAULT));
@@ -60,8 +61,9 @@ private static ArrayList[] getFiles(String htrootPath) {
             if (file.isDirectory()) {
                 htrootDirs.add(htroots[i]);
             } else if (
-                    (dot = htroots[i].lastIndexOf('.')) < 2 ||
-                    htroots[i].charAt(dot - 2) == '_' && htroots[i].charAt(dot - 1) == 'p'
+                    ((dot = htroots[i].lastIndexOf('.')) < 2 ||
+                    htroots[i].charAt(dot - 2) == '_' && htroots[i].charAt(dot - 1) == 'p') &&
+                    !(htroots[i].endsWith("java") || htroots[i].endsWith("class"))
             ) {
                 htrootFiles.add(htroots[i]);
             }

diff --git a/htroot/robots.txt b/htroot/robots.txt
@@ -51,6 +51,12 @@ Disallow: /share#(/fileshare)#
 # surftips
 Disallow: /Surftips.html#(/surftips)#
 
+#(profile)#::
+# view profile
+Disallow: /ViewProfile.html
+Disallow: /ViewProfile.vcf
+Disallow: /ViewProfile.xml#(/profile)#
+
 ::
 # all
 Disallow: /

diff --git a/source/de/anomic/http/httpdRobotsTxtConfig.java b/source/de/anomic/http/httpdRobotsTxtConfig.java
@@ -63,6 +63,7 @@ public final class httpdRobotsTxtConfig {
     public static final String LOCKED = "locked";
     public static final String DIRS = "dirs";
     public static final String NETWORK = "network";
+    public static final String PROFILE = "profile";
     public static final String ALL = "all";
 
     private boolean allDisallowed = false;
@@ -77,6 +78,7 @@ public final class httpdRobotsTxtConfig {
     private boolean networkDisallowed = false;
     private boolean surftipsDisallowed = false;
     private boolean bookmarksDisallowed = false;
+    private boolean profileDisallowed = true;
 
     public httpdRobotsTxtConfig() {  }
 
@@ -95,6 +97,7 @@ public httpdRobotsTxtConfig(String[] active) {
             if (active[i].equals(NETWORK)) { this.networkDisallowed = true; continue; }
             if (active[i].equals(LOCKED)) { this.lockedDisallowed = true; continue; }
             if (active[i].equals(DIRS)) { this.dirsDisallowed = true; continue; }
+            if (active[i].equals(PROFILE)) { this.profileDisallowed = true; continue; }
             if (active[i].equals(ALL)) { this.allDisallowed = true; continue; }
         }
     }
@@ -119,6 +122,7 @@ public String toString() {
         if (this.statusDisallowed) sb.append(STATUS).append(",");
         if (this.surftipsDisallowed) sb.append(SURFTIPS).append(",");
         if (this.wikiDisallowed) sb.append(WIKI).append(",");
+        if (this.profileDisallowed) sb.append(PROFILE).append(",");
         return sb.toString();
     }
 
@@ -217,4 +221,12 @@ public boolean isWikiDisallowed() {
     public void setWikiDisallowed(boolean wikiDisallowed) {
         this.wikiDisallowed = wikiDisallowed;
     }
+
+    public boolean isProfileDisallowed() {
+        return profileDisallowed || this.allDisallowed;
+    }
+
+    public void setProfileDisallowed(boolean profileDisallowed) {
+        this.profileDisallowed = profileDisallowed;
+    }
 }