Skip to content
Permalink
Browse files
Site search is 80% complete now
  • Loading branch information
LadyCailin committed Mar 28, 2020
1 parent b063add commit d7b8e99e0a6bce9273aa20c8208587a54fcb7afe
Show file tree
Hide file tree
Showing 11 changed files with 732 additions and 134 deletions.
@@ -0,0 +1,53 @@
package com.laytonsmith.tools.docgen.localization;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.json.simple.JSONObject;

/**
*
*/
public class MasterSearchIndex {

private final Map<String, List<String>> segments = new HashMap<>();

void addSegment(String toLocation, String segment) {
if(!segments.containsKey(toLocation)) {
segments.put(toLocation, new ArrayList<>());
}
segments.get(toLocation).add(segment);
}

/**
* Creates the index file, ready to be written to the server. The general format of the file is a json file
* with the following schema:
* <p>
* The outermost element is a map. Each element of the map is a string containing the full search segment. The
* value of each element is an array of locations where this string occurs.
* <p>
* The intended search pattern is to loop through the segments finding segments where the strings exist, eliminating
* those index elements where the search text isn't found. Of the remaining segments, the next problem is to sort
* these by relevance. Priority should be given to segments that have a higher match percentage, so if the search
* string is "test" and the remaining segments are "Hello Test!", "Testing", and "Test", then they should be ranked
* as 1. "Test", 2. "Testing", and 3. "Hello Test!".
* <p>
* In any case, this logic is implemented client side.
* @return
*/
public String getIndex() {
Map<String, List<String>> index = new HashMap<>();
for(Map.Entry<String, List<String>> entry : segments.entrySet()) {
String location = entry.getKey();
for(String string : entry.getValue()) {
if(!index.containsKey(string)) {
index.put(string, new ArrayList<>());
}
index.get(string).add(location);
}
}
return JSONObject.toJSONString(index);
}

}
@@ -31,6 +31,7 @@ public class TranslationMaster {
private final File translationDb;
private final TranslationSummary translationSummary;
private final ProgressIterator progressCallback;
private final MasterSearchIndex masterSearchIndex = new MasterSearchIndex();

/**
* Creates an object that represents a full translation database, located at the given local location.
@@ -176,9 +177,18 @@ public boolean hasMasterTranslation(Locale locale, String key) {
return allLocales.get(locale).master.containsKey(key);
}

/**
* Given a page, filters and segments the input string to create translation segments. In addition, creates
* a search index based on the input segments. For English, this index can be used as is, but for other languages,
* this index must also be localized. The index segments will all be in the translation database, however, so
* this process can be automatic. In any case, this function does not handle that.
* @param toLocation
* @param inputString
*/
public void createTranslationMemory(String toLocation, String inputString) {
Set<String> segments = TranslationMaster.findSegments(inputString);
for(String segment : segments) {
masterSearchIndex.addSegment(toLocation, segment);
int id;
if(!translationSummary.containsTranslation(segment)) {
id = getNewId();
@@ -364,6 +374,10 @@ public TranslationMemory generateNewTranslation(Locale locale, String englishKey
return new TranslationMemory(englishKey, locale, "", "", "", id);
}

public MasterSearchIndex getSearchIndex() {
return this.masterSearchIndex;
}

private static final String[] SEGMENT_SEP = new String[]{
"==+",
"\n\n",
@@ -62,26 +62,50 @@ public void setAutomaticTranslation(String automaticTranslation) {
this.automaticTranslation = automaticTranslation;
}

/**
* The English Key is the "master" value for this translation.
* @return
*/
public String getEnglishKey() {
return englishKey;
}

/**
* The locale that this {@code TranslationMemory} is associated with.
* @return
*/
public Locale getLocale() {
return locale;
}

/**
* The local specific comment for this translation.
* @return
*/
public String getComment() {
return comment;
}

/**
* The unique id for this translation.
* @return
*/
public int getId() {
return translationId;
}

/**
* The machine translation, or null if not set.
* @return
*/
public String getMachineTranslation() {
return this.automaticTranslation;
}

/**
* The human translation, or null if not set.
* @return
*/
public String getTranslation() {
return this.translation;
}
@@ -41,6 +41,9 @@
import com.laytonsmith.tools.docgen.DocGenTemplates;
import com.laytonsmith.tools.docgen.DocGenTemplates.Generator;
import com.laytonsmith.tools.docgen.DocGenTemplates.Generator.GenerateException;
import com.laytonsmith.tools.docgen.localization.Locale;
import com.laytonsmith.tools.docgen.localization.MasterSearchIndex;
import com.laytonsmith.tools.docgen.localization.TranslationMemory;
import com.laytonsmith.tools.docgen.templates.Template;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -115,11 +118,13 @@ public static void run(boolean generatePrefs, boolean useLocalCache, File sitede
defaults.add(new Preferences.Preference(USERNAME, "", Preferences.Type.STRING, "The username to scp with"));
defaults.add(new Preferences.Preference(HOSTNAME, "", Preferences.Type.STRING, "The hostname to scp to (not the"
+ " hostname of the site). If"
+ " the hostname is \"localhost\", this triggers special handling, which skips the upload, and simply"
+ " the hostname is \"localhost\", or \"127.0.0.1\" this triggers special handling,"
+ " which skips the upload, and simply"
+ " saves to the specified location on local disk. This should work with all OSes, otherwise the host"
+ " that this connects to must support ssh, though it does not necessarily have to be a unix"
+ " based system."
+ " If the value is localhost, the values " + USERNAME + ", " + PORT + ", and " + PASSWORD + " are"
+ " If the value is localhost/127.0.0.1, the values " + USERNAME + ", " + PORT + ", and " + PASSWORD
+ " are"
+ " irrelevant, and not used. This should NOT begin with a protocol, i.e. http://"));
defaults.add(new Preferences.Preference(PORT, "22", Preferences.Type.INT, "The port to use for SCP"));
defaults.add(new Preferences.Preference(DIRECTORY, "/var/www/docs", Preferences.Type.STRING,
@@ -229,7 +234,7 @@ public static void run(boolean generatePrefs, boolean useLocalCache, File sitede
if(!siteBase.startsWith("https://") && !siteBase.startsWith("http://")) {
configErrors.add("SiteBase must begin with either http:// or https://");
}
if(!"localhost".equals(hostname)) {
if(!"localhost".equalsIgnoreCase(hostname) && !"127.0.0.1".equals(hostname)) {
if(port < 0 || port > 65535) {
configErrors.add("Port must be a number between 0 and 65535.");
}
@@ -322,7 +327,7 @@ public static void run(boolean generatePrefs, boolean useLocalCache, File sitede
}
}
DeploymentMethod deploymentMethod;
if("localhost".equals(hostname)) {
if("localhost".equalsIgnoreCase(hostname) || "127.0.0.1".equals(hostname)) {
deploymentMethod = new LocalDeploymentMethod(directory + "/");
} else {
/**
@@ -388,6 +393,11 @@ public void run() {
} catch (Throwable ex) {
writeLog("Could not write out translations!", ex);
}
try {
writeSearchIndex();
} catch (Throwable ex) {
writeLog("Could not write out search index!", ex);
}
uploadQueue.shutdown();
} else {
uploadQueue.submit(this);
@@ -1053,6 +1063,25 @@ private void writeMasterTranslations() throws IOException {
}
}

/**
* If the masterMemories is not null, this writes out the search index file. In any case, writes out a small page
* (which can be loaded on each page load)
* called "searchIndexExists.json" which states whether or not the index exists, so the search bar can be
* shown only if the index is present.
* @throws IOException
*/
private void writeSearchIndex() throws IOException {
if(masterMemories != null) {
writeStatus("Generating and uploading search index");
MasterSearchIndex index = masterMemories.getSearchIndex();
writeFromString(index.getIndex(), "searchIndex.json");
// For now, just hardcode false, until it's ready to be switched on.
writeFromString("{\"searchIndexExists\":false}", "searchIndexExists.json");
} else {
writeFromString("{\"searchIndexExists\":false}", "searchIndexExists.json");
}
}

/**
* Pages deployed: All pages under the siteDeploy/resources folder. They are transferred as is, including
* recursively looking through the folder structure. Binary files are also supported. index.js (after template
@@ -92,18 +92,19 @@ Create a configuration file somewhere on your file system, call it site-deploy-l
directory=REPLACEME

# The base url of the docs. This should begin with http:// or https://
docs-base=http://localhost:8080/docs/
docs-base=http://127.0.0.1:8080/docs/

# The hostname to scp to (not the hostname of the site). If the hostname is "localhost", this triggers special handling,
# The hostname to scp to (not the hostname of the site). If the hostname is "localhost", or "127.0.0.1"
# this triggers special handling,
# which skips the upload, and simply saves to the specified location on local disk. This should work with all OSes,
# otherwise the host that this connects to must support ssh, though it does not necessarily have to be a unix based
# system. If the value is localhost, the values username, port, and use-password are irrelevant, and not used. This
# should NOT begin with a protocol, i.e. http://
hostname=localhost
hostname=127.0.0.1


# The base url of the site (where "home" should be). This should begin with http:// or https://
site-base=http://localhost:8080/
site-base=http://127.0.0.1:8080/

use-password=false
username=
@@ -133,7 +134,7 @@ On the first run, this may take a while, so just sit back and relax.
From the command prompt, ''cd'' to one folder above where you specified in the ''directory'' value in the config (that
is, the root of the website) and run

<%PRE|npx http-server -p 8080 -c-1 -o "http://localhost:8080/index.html"%>
<%PRE|npx http-server -p 8080 -c-1 -o "index.html"%>

Alternatively, you can install the http-server package, and leave off the ''npx'' command.

0 comments on commit d7b8e99

Please sign in to comment.