Merged master into postprocessing branch

addon/YaCy.app/Contents/Info.plist

@@ -19,7 +19,7 @@
 <key>CFBundleAllowMixedLocalizations</key>
 <string>true</string>
 <key>CFBundleExecutable</key>
-<string>startYACY.sh</string>
+<string>startYACYMacOS.sh</string>
 <key>CFBundleDevelopmentRegion</key>
 <string>English</string>
 <key>CFBundlePackageType</key>

addon/YaCy.app/Contents/MacOS/startYACYMacOS.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+# Launcher for YaCy in a MacOS bundle : 
+# rely on the generic startYACY.sh, but specifies the user home relative path for YaCy data
+# This data directory is set in conforming to OS X File System Programming Guide 
+# see : https://developer.apple.com/library/ios/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html
+
+"`dirname $0`"/startYACY.sh -s "'Library/Application Support/net.yacy.YaCy'"

build.xml

@@ -764,7 +764,8 @@
   	</copy>
   	<copy file="${addon}/YaCy.app/Contents/Info.plist" tofile="${release_mac}/YaCy.app/Contents/Info.plist" filtering="true" overwrite="true" />
   	<move file="${release_main}" tofile="${release_mac}/YaCy.app/Contents/MacOS" verbose="false" />
-  	<!-- startYACY.sh will be the main entry point : we set permissions to make it an executable file -->
+  	<!-- startYACY.sh and startYACYMacOS.sh will be the main entry points : we set permissions to make it executable files -->
+  	<chmod file="${release_mac}/YaCy.app/Contents/MacOS/startYACYMacOS.sh" perm="755"/>
   	<chmod file="${release_mac}/YaCy.app/Contents/MacOS/startYACY.sh" perm="755"/>
   	<exec executable="hdiutil">
   	  	<arg line="create -srcfolder ${release_mac}/YaCy.app ${release}/yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.dmg"/>

docker/Dockerfile

@@ -18,6 +18,10 @@ WORKDIR /opt
 # - Compile with ant
 # - remove unnecessary and size consuming .git directory
 # - remove ant and git packages
+
+# Possible alternative : copy directly your current sources an remove git clone command from the following RUN
+# COPY . /opt/yacy_search_server/
+
 RUN apt-get update && \
 	apt-get install -yq ant git && \
 	git clone https://github.com/yacy/yacy_search_server.git && \
@@ -30,14 +34,17 @@ RUN apt-get update && \
 # Set initial admin password : "docker" (encoded with custom yacy md5 function net.yacy.cora.order.Digest.encodeMD5Hex())
 RUN sed -i "/adminAccountBase64MD5=/c\adminAccountBase64MD5=MD5:e672161ffdce91be4678605f4f4e6786" /opt/yacy_search_server/defaults/yacy.init
 
+# Intially enable HTTPS : this is the most secure option for remote administrator authentication
+RUN sed -i "/server.https=false/c\server.https=true" /opt/yacy_search_server/defaults/yacy.init
+
 # Create user and group yacy : this user will be used to run YaCy main process
 RUN adduser --system --group --no-create-home --disabled-password yacy
 
 # Set ownership of yacy install directory to yacy user/group
 RUN chown yacy:yacy -R /opt/yacy_search_server
 
-# Expose port 8090
-EXPOSE 8090
+# Expose HTTP and HTTPS default ports
+EXPOSE 8090 8443
 
 # Set data volume : yacy data and configuration will persist aven after container stop or destruction
 VOLUME ["/opt/yacy_search_server/DATA"]

docker/Dockerfile.alpine

@@ -51,6 +51,10 @@ WORKDIR /opt
 # - compile with apache ant
 # - remove unnecessary and size consuming .git directory
 # - delete git package and ant binary install
+
+# Possible alternative : copy directly your current sources an remove git clone command from the following RUN
+# COPY . /opt/yacy_search_server/
+
 RUN apk update && \
 	apk add --no-cache git && \
 	git clone https://github.com/yacy/yacy_search_server.git && \
@@ -62,14 +66,17 @@ RUN apk update && \
 # Set initial admin password : "docker" (encoded with custom yacy md5 function net.yacy.cora.order.Digest.encodeMD5Hex())
 RUN sed -i "/adminAccountBase64MD5=/c\adminAccountBase64MD5=MD5:e672161ffdce91be4678605f4f4e6786" /opt/yacy_search_server/defaults/yacy.init
 
+# Intially enable HTTPS : this is the most secure option for remote administrator authentication
+RUN sed -i "/server.https=false/c\server.https=true" /opt/yacy_search_server/defaults/yacy.init
+
 # Create user and group yacy : this user will be used to run YaCy main process
 RUN addgroup yacy && adduser -S -G yacy -H -D yacy
 
 # Set ownership of yacy install directory to yacy user/group
 RUN chown yacy:yacy -R /opt/yacy_search_server
 
-# Expose port 8090
-EXPOSE 8090
+# Expose HTTP and HTTPS default ports
+EXPOSE 8090 8443
 
 # Set data volume : yacy data and configuration will persist aven after container stop or destruction
 VOLUME ["/opt/yacy_search_server/DATA"]

docker/Readme.md

@@ -18,6 +18,11 @@ Using yacy_search_server/docker/Dockerfile :
 	cd yacy_search_server/docker
 	docker build .
 
+To build the Alpine variant :
+
+	cd yacy_search_server/docker
+	docker build -f Dockerfile.alpine .
+
 ## Image variants
 
 `luccioman/yacy:latest`
@@ -49,12 +54,12 @@ You can retrieve the container IP address with `docker inspect`.
 
 #### Easier to handle
 
-	docker run --name yacy -p 8090:8090 --log-opt max-size=100m --log-opt max-file=2 luccioman/yacy
+	docker run --name yacy -p 8090:8090 -p 8443:8443 --log-opt max-size=200m --log-opt max-file=2 luccioman/yacy
 
 ##### Options detail
 
 * --name : allow easier management of your container (without it, docker automatically generate a new name at each startup).
-* -p : map host port and container port, allowing web interface access through the usual http://localhost:8090.
+* -p 8090:8090 -p 8443:8443 : map host ports to YaCy container ports, allowing web interface access through the usual http://localhost:8090 and https://localhost:8443 (you can set a different mapping, for example -p 443:8443 if you prefer to use the default HTTPS port on your host)
 * --log-opt max-size : limit maximum docker log file size for this container
 * --log-opt max-file : limit number of docker rotated log files for this container
 
@@ -76,9 +81,47 @@ Note that you can list all docker volumes with :
 
 	docker volume ls
 
-#### As background process
+#### Start as background process
 
 	docker run -d luccioman/yacy
+
+### HTTPS support
+
+This images are default configured with HTTPS enabled, and use a default certificate stored in defaults/freeworldKeystore. You should use your own certificate. In order to do it, you can proceed as follow.
+
+#### Self-signed certificate
+
+A self-signed certificate will provide encrypted communications with your YaCy server, but browsers will still complain about an invalid security certificate with the error "SEC_ERROR_UNKNOWN_ISSUER". If it is sufficient for you, you can permanently add and exception to your browser.
+
+This kind of certificate can be generated and added to your YaCy Docker container with the following :
+
+	keytool -keystore /var/lib/docker/volumes/[your_yacy_volume]/_data/SETTINGS/yacykeystore -genkey -keyalg RSA -alias yacycert
+
+Then edit YaCy config file. For example with the nano text editor :
+
+	nano /var/lib/docker/volumes/[your_yacy_volume]/_data/SETTINGS/yacy.conf
+
+And configure the keyStoreXXXX properties accordingly :
+
+	keyStore=/opt/yacy_search_server/DATA/SETTINGS/yacykeystore
+	keyStorePassword=yourpassword
+
+#### Import an existing certificate:
+
+Importing a certificate validated by a certification authority (CA) will ensure you have full HTTPS support with no security errors when accessing your YaCy peer. You can import an existing certificate in pkcs12 format.
+
+First copy it to the YaCy Docker container volume :
+
+	cp [yourStore].pkcs12 /var/lib/docker/volumes/[your_yacy_volume]/_data/SETTINGS/[yourStore].pkcs12
+
+Then edit YaCy config file. For example with the nano text editor :
+
+	nano /var/lib/docker/volumes/[your_yacy_volume]/_data/SETTINGS/yacy.conf
+
+And configure the pkcs12XXX properties accordingly :
+
+	pkcs12ImportFile=/opt/yacy_search_server/DATA/SETTINGS/[yourStore].pkcs12
+	pkcs12ImportPwd=yourpassword
 
 ### Next starts
 
@@ -109,7 +152,7 @@ OR
 
 Create new container based on pulled image, using volume data from old container :
 
-	docker create --name [tmp-container_name] -p 8090:8090 --volumes-from=[container_name] --log-opt max-size=100m --log-opt max-file=2 luccioman/yacy:latest
+	docker create --name [tmp-container_name] -p 8090:8090 -p 8443:8443 --volumes-from=[container_name] --log-opt max-size=100m --log-opt max-file=2 luccioman/yacy:latest
 
 Stop old container :
 

docker/docker-cloud.yml

@@ -2,5 +2,6 @@ yacy:
   image: 'luccioman/yacy:latest'
   ports:
     - '8090:8090'
+    - '8443:8443'
   restart: on-failure
   autoredeploy: true

htroot/Autocrawl_p.java

@@ -1,5 +1,4 @@
 import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.crawler.data.CrawlProfile.CrawlAttribute;
 import net.yacy.search.Switchboard;
 import net.yacy.search.SwitchboardConstants;
 import net.yacy.server.serverObjects;

htroot/News.html

@@ -43,6 +43,10 @@ <h2>Published&nbsp;News</h2>
         A change in the personal profile will create a news entry. You can see recently made changes of
         profile entries on the Network page, where that profile change is visualized with a '*' beside the 'P' (profile) - selector.
       </li>
+      <li>
+        Publishing of added or modified translation for the user interface. Other peers may include it in their local translation list.
+        To publish a translation, use the integrated <a href="Translator_p.html">translation editor</a> to add a translation and publish it afterwards.
+      </li>
     </ul>
     <p>
       More news services will follow.

htroot/TransNews_p.html

@@ -6,7 +6,7 @@
   </head>
   <body>
     #%env/templates/header.template%#
-
+    #%env/templates/submenuComputation.template%#
     <h2 class="yacy">Translation News for Language #[currentlang]#</h2>
 
     <form method="post" enctype="multipart/form-data">

htroot/TransNews_p.java

@@ -42,7 +42,6 @@
 import net.yacy.server.serverSwitch;
 import net.yacy.utils.crypt;
 import net.yacy.utils.translation.TranslationManager;
-import net.yacy.utils.translation.TranslatorXliff;
 
 public class TransNews_p {
 
@@ -90,32 +89,13 @@ public static serverObjects respond(final RequestHeader header, final serverObje
                                 continue;
                             }
                             if (NewsPool.CATEGORY_TRANSLATION_ADD.equals(rtmp.category())) {
-                                //String tmplng = rtmp.attribute("language", null);
+                                String tmplng = rtmp.attribute("language", null);
                                 String tmpfile = rtmp.attribute("file", null);
                                 String tmpsource = rtmp.attribute("source", null);
-                                String tmptarget = rtmp.attribute("target", null);
+                                //String tmptarget = rtmp.attribute("target", null);
 
-                                if (sb.peers.mySeed().hash.equals(rtmp.originator())) {
-                                    /*
-                                        if (tmplng != null && tmplng.equals(currentlang)) {
-                                            sendit = false;
-                                            break;
-                                        }*/
-                                    if (tmpfile != null && tmpfile.equals(file)) {
-                                        sendit = false;
-                                        break;
-                                    }
-                                    if (tmpsource != null && tmpsource.equals(sourcetxt)) {
-                                        sendit = false;
-                                        break;
-                                    }
-                                    if (tmptarget != null && tmptarget.equals(targettxt)) {
-                                        sendit = false;
-                                        break;
-                                    }
-                                }
                                 // if news with file and source exist (maybe from other peer) - skip sending another msg (to avoid confusion)
-                                if ((tmpfile != null && tmpfile.equals(file))
+                                if ((tmplng != null && tmplng.equals(currentlang)) && (tmpfile != null && tmpfile.equals(file))
                                         && (tmpsource != null && tmpsource.equals(sourcetxt))) {
                                     sendit = false;
                                     break;
@@ -189,8 +169,8 @@ public static serverObjects respond(final RequestHeader header, final serverObje
         final HashMap<String, Integer> positiveHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
         accumulateVotes(sb, negativeHashes, positiveHashes, NewsPool.INCOMING_DB);
         final ScoreMap<String> ranking = new ConcurrentScoreMap<String>(); // score cluster for url hashes
-        final HashMap<String, NewsDB.Record> Translation = new HashMap<String, NewsDB.Record>(); // a mapping from an url hash to a kelondroRow.Entry with display properties
-        accumulateTranslations(sb, Translation, ranking, negativeHashes, positiveHashes, NewsPool.INCOMING_DB);
+        final HashMap<String, NewsDB.Record> translation = new HashMap<String, NewsDB.Record>(); // a mapping from an url hash to a kelondroRow.Entry with display properties
+        accumulateTranslations(sb, translation, ranking, negativeHashes, positiveHashes, NewsPool.INCOMING_DB);
 
         // read out translation-news array and create property entries
         final Iterator<String> k = ranking.keys(false);
@@ -207,7 +187,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
                 continue;
             }
 
-            row = Translation.get(refid);
+            row = translation.get(refid);
             if (row == null) {
                 continue;
             }
@@ -220,6 +200,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
                 continue;
             }
 
+            if (!lang.equals(currentlang)) continue;
 
             String existingtarget = null; //transMgr.getTranslation(filename, source);
             Map<String, String> tmpMap = localTrans.get(filename);

htroot/WebStructurePicture_p.java

@@ -39,7 +39,6 @@
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.cora.sorting.ClusteredScoreMap;
 import net.yacy.cora.util.CommonPattern;
-import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.peers.graphics.WebStructureGraph;
 import net.yacy.search.Switchboard;
 import net.yacy.server.serverObjects;

htroot/env/templates/submenuComputation.template

@@ -33,6 +33,7 @@
     <ul class="SubMenu">
       <li><a href="Surftips.html" class="MenuItemLink">Surftips</a></li>
       <li><a href="Wiki.html?display=1" class="MenuItemLink">Local Peer Wiki</a></li>
+      <li><a href="TransNews_p.html" class="MenuItemLink">UI Translations</a></li>
     </ul>
   </div>
 </div>

htroot/yacy/search.java

@@ -45,7 +45,6 @@
 import net.yacy.cora.protocol.Domains;
 import net.yacy.cora.protocol.HeaderFramework;
 import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.cora.sorting.ConcurrentScoreMap;
 import net.yacy.cora.sorting.ScoreMap;
 import net.yacy.cora.sorting.WeakPriorityBlockingQueue;
 import net.yacy.cora.storage.HandleSet;
@@ -380,7 +379,9 @@ public static serverObjects respond(final RequestHeader header, final serverObje
 
             // prepare reference hints
             final long timer = System.currentTimeMillis();
-            final ScoreMap<String> topicNavigator = sb.index.connectedRWI() ? theSearch.getTopics(5, 100) : new ConcurrentScoreMap<String>();
+            //final ScoreMap<String> topicNavigator = sb.index.connectedRWI() ? theSearch.getTopics(5, 100) : new ConcurrentScoreMap<String>();
+            final ScoreMap<String> topicNavigator = theSearch.getTopics(); // as there is currently no index interaction in getTopics(), we can use it by default
+
             final StringBuilder refstr = new StringBuilder(6000);
             final Iterator<String> navigatorIterator = topicNavigator.keys(false);
             int i = 0;

htroot/yacy/seedlist.java

@@ -24,6 +24,7 @@
 import java.util.Set;
 
 import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.util.JSONObject;
 import net.yacy.peers.Seed;
 import net.yacy.search.Switchboard;
 import net.yacy.server.serverCore;
@@ -91,22 +92,22 @@ public static serverObjects respond(final RequestHeader header, final serverObje
                 Set<String> ips = seed.getIPs();
                 if (ips == null || ips.size() == 0) continue;
                 prop.putJSON("peers_" + count + "_map_0_k", Seed.HASH);
-                prop.put("peers_" + count + "_map_0_v", '"' + serverObjects.toJSON(seed.hash) + '"');
+                prop.put("peers_" + count + "_map_0_v", JSONObject.quote(seed.hash));
                 prop.put("peers_" + count + "_map_0_c", 1);
                 Map<String, String> map = seed.getMap();
                 int c = 1;
                 if (!addressonly) {
                     for (Map.Entry<String, String> m: map.entrySet()) {
                         prop.putJSON("peers_" + count + "_map_" + c + "_k", m.getKey());
-                        prop.put("peers_" + count + "_map_" + c + "_v", '"' + serverObjects.toJSON(m.getValue()) + '"');
+                        prop.put("peers_" + count + "_map_" + c + "_v", JSONObject.quote(m.getValue()));
                         prop.put("peers_" + count + "_map_" + c + "_c", 1);
                         c++;
                     }
                 }
                 // construct a list of ips
                 StringBuilder a = new StringBuilder();
                 a.append('[');
-                for (String ip: ips) a.append('"').append(serverObjects.toJSON(seed.getPublicAddress(ip))).append('"').append(',');
+                for (String ip: ips) a.append(JSONObject.quote(seed.getPublicAddress(ip))).append(',');
                 a.setCharAt(a.length()-1, ']');
                 prop.putJSON("peers_" + count + "_map_" + c + "_k", "Address");
                 prop.put("peers_" + count + "_map_" + c + "_v", a.toString());

locales/fr.lng

@@ -1212,8 +1212,14 @@ Category==Categorie
 Received==Reçu
 Distributed==Distribué
 Attributes==Attributs
-"#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#"==#(page)#::Traiter les nouvelles sélectionnées::Supprimer les nouvelles sélectionnées::Annuler la publication des nouvelles sélectionnées::Supprimer les nouvelles sélectionées#(/page)#
-"#(page)#::Process All News::Delete All News::Abort Publication of All News::Delete All News#(/page)#"==#(page)#::Traiter toutes les nouvelles::Supprimer toutes les nouvelles::Annuler la publication de toutes les nouvelles::Supprimer toutes les nouvelles#(/page)#
+Process Selected News==Traiter les nouvelles sélectionnées
+Delete Selected News==Supprimer les nouvelles sélectionnées
+Abort Publication of Selected News==Annuler la publication des nouvelles sélectionnées
+Delete Selected News==Supprimer les nouvelles sélectionées
+Process All News==Traiter toutes les nouvelles
+Delete All News==Supprimer toutes les nouvelles
+Abort Publication of All News==Annuler la publication de toutes les nouvelles
+Delete All News==Supprimer toutes les nouvelles
 #-----------------------------
 
 #File: Performance_p.html