Skip to content
Permalink
Browse files

new search script

  • Loading branch information
adamdruppe committed Mar 10, 2018
1 parent e06c98a commit b81cceff12a269e79c00937cea6f111faa36191b
Showing with 134 additions and 25 deletions.
  1. +5 −0 Makefile
  2. +6 −3 doc2.d
  3. +99 −15 locate.d
  4. +24 −7 search-docs.js
@@ -4,6 +4,11 @@ all:
#dmd diff.d terminal.d $(LIBDPARSE)
dmd -m64 doc2.d latex.d cgi.d comment.d stemmer.d dom.d script.d jsvar.d html.d color.d -J. $(LIBDPARSE) -g # -version=std_parser_verbose
#dmd -of/var/www/dpldocs.info/locate locate.d dom.d stemmer.d cgi -J. -version=fastcgi -m64 -debug
locate:
dmd -of/var/www/dpldocs.info/locate locate.d dom.d stemmer.d cgi -J. -version=fastcgi -m64 -debug

vps_locate:
ldc2 -oq -O3 -m64 locate.d dom.d stemmer.d cgi -J. -d-version=scgi -d-version=vps -g

ldc:
ldc2 -oq -O3 -m64 doc2.d latex.d cgi.d comment.d stemmer.d dom.d script.d jsvar.d html.d color.d -J. $(LIBDPARSE) -g # -version=std_parser_verbose
9 doc2.d
@@ -3116,6 +3116,7 @@ void main(string[] args) {
if(copyStandardFiles) {
copyStandardFileTo(outputDirectory ~ "style.css", "style.css");
copyStandardFileTo(outputDirectory ~ "script.js", "script.js");
copyStandardFileTo(outputDirectory ~ "search-docs.js", "search-docs.js");
}

/*
@@ -3388,6 +3389,10 @@ void main(string[] args) {
cgi.setResponseContentType("text/javascript");
cgi.write(readText(findStandardFile("script.js")), true);
return;
} else if(file == "search-docs.js") {
cgi.setResponseContentType("text/javascript");
cgi.write(readText(findStandardFile("search-docs.js")), true);
return;
} else {
if(file.length == 0) {
if("index" !in pseudoFiles)
@@ -3538,9 +3543,7 @@ void main(string[] args) {
index.writeln("</script>");

// write the script that runs the search
index.writeln("<script>");
index.write(import("search-docs.js"));
index.writeln("</script>");
index.writeln("<script src=\"search-docs.js\"></script>");

// and close the skeleton
index.writeln("</body></html>");
114 locate.d
@@ -1,5 +1,7 @@
// FIXME: add +proj and -proj to adjust project results

module adrdox.locate;

// # dpldocs: if one request, go right to it. and split camel case and ry rearranging words. File.size returned nothing

import ps = PorterStemmer;
@@ -15,8 +17,20 @@ class ProjectSearcher {
this(string path, string name, int projectAdjustment) {
this.projectName = name;
this.projectAdjustment = projectAdjustment;

string text;
if(path[$ - 3 .. $] == ".gz") {
auto com = std.file.read(path);
import std.zlib;
auto uc = new UnCompress;
text = cast(string) uc.uncompress(com);
text ~= cast(string) uc.flush();
} else {
text = readText(path);
}

auto index = new Document();
index.parseUtf8(readText(path), true, true);
index.parseUtf8(text, true, true);

if(auto s = index.querySelector("script#search-index-container")) {
index = new Document();
@@ -153,18 +167,76 @@ class ProjectSearcher {

}

ProjectSearcher[] projectSearchers;
__gshared ProjectSearcher[] projectSearchers;

shared static this() {
version(vps) {
version(embedded_httpd)
processPoolSize = 2;

static this() {
projectSearchers ~= new ProjectSearcher("experimental-docs/search-results.html", "", 5);
//projectSearchers ~= new ProjectSearcher("experimental-docs/std.xml", "Standard Library", 5);
//projectSearchers ~= new ProjectSearcher("experimental-docs/arsd.xml", "arsd", 4);
projectSearchers ~= new ProjectSearcher("experimental-docs/vibe.xml", "Vibe.d", 0);
projectSearchers ~= new ProjectSearcher("experimental-docs/dmd.xml", "DMD", 0);
import std.file;

foreach(dirName; dirEntries("/dpldocs/", SpanMode.shallow)) {
string filename;
filename = dirName ~ "/master/adrdox-generated/search-results.html.gz";
if(!exists(filename)) {
filename = null;
foreach(fn; dirEntries(dirName, "search-results.html.gz", SpanMode.depth)) {
filename = fn;
break;
}
}

if(filename.length) {
projectSearchers ~= new ProjectSearcher(filename, dirName["/dpldocs/".length .. $], 0);
import std.stdio; writeln("Loading ", filename);
}
}

import std.stdio;
writeln("Ready");

} else {
projectSearchers ~= new ProjectSearcher("experimental-docs/search-results.html", "", 5);
//projectSearchers ~= new ProjectSearcher("experimental-docs/std.xml", "Standard Library", 5);
//projectSearchers ~= new ProjectSearcher("experimental-docs/arsd.xml", "arsd", 4);
projectSearchers ~= new ProjectSearcher("experimental-docs/vibe.xml", "Vibe.d", 0);
projectSearchers ~= new ProjectSearcher("experimental-docs/dmd.xml", "DMD", 0);
}
}

void searcher(Cgi cgi) {
auto search = cgi.request("q", cgi.queryString);

version(vps) {
string path = cgi.requestUri;

auto q = path.indexOf("?");
if(q != -1) {
path = path[0 .. q];
}

if(path.length && path[0] == '/')
path = path[1 .. $];



if(path == "script.js") {
import std.file;
cgi.setResponseContentType("text/javascript");
cgi.write(std.file.read("/dpldocs-build/script.js"), true);
return;

}

if(path == "style.css") {
import std.file;
cgi.setResponseContentType("text/css");
cgi.write(std.file.read("/dpldocs-build/style.css"), true);
return;
}
}

auto search = cgi.request("q", cgi.request("searchTerm", cgi.queryString));

ProjectSearcher.Magic[] magic;
foreach(searcher; projectSearchers)
@@ -184,16 +256,25 @@ void searcher(Cgi cgi) {
}

auto document = new Document();
document.parseUtf8(import("skeleton.html"), true, true);
version(vps) {
import std.file;
document.parseUtf8(readText("/dpldocs-build/skeleton.html"), true, true);
document.title = "Dub Documentation Search";
} else
document.parseUtf8(import("skeleton.html"), true, true);
document.title = "Search Results";

auto form = document.requireElementById!Form("search");
form.setValue("searchTerm", search);

auto l = document.requireSelector("link");
l.href = "/experimental-docs/" ~ l.href;
l = document.requireSelector("script[src]");
l.src = "/experimental-docs/" ~ l.src;
version(vps) {
// intentionally blank
} else {
auto l = document.requireSelector("link");
l.href = "/experimental-docs/" ~ l.href;
l = document.requireSelector("script[src]");
l.src = "/experimental-docs/" ~ l.src;
}

auto pc = document.requireSelector("#page-content");
pc.addChild("h1", "Search Results");
@@ -219,7 +300,10 @@ void searcher(Cgi cgi) {
foreach(idx, item; magic) {
Element decl = item.decl;
if(decl is null) continue; // should never happen
auto link = "http://dpldocs.info/experimental-docs/" ~ decl.requireSelector("link").innerText;
version(vps)
auto link = "http://"~item.searcher.projectName~".dpldocs.info/" ~ decl.requireSelector("link").innerText;
else
auto link = "http://dpldocs.info/experimental-docs/" ~ decl.requireSelector("link").innerText;
auto fqn = getFqn(decl);
if(fqn in alreadyPresent)
continue;
@@ -34,18 +34,35 @@ function adrdox_search(searchTerm) {
if(searchTerm.length == 0)
return;

searchTerm = searchTerm.replace(/\++/g, ' ');

if(searchTerm == window.currentSearchTerm)
return;

window.currentSearchTerm = searchTerm;

location.hash = "#!" + encodeURIComponent(searchTerm);

var hits = searchDocument.querySelectorAll("adrdox > index > term[value=\""+/*stemmer*/(searchTerm)+"\"] > result");
hits = Array.prototype.slice.call(hits);
var hitsObj = {};

var terms = searchTerm.split(" ");
for(var i = 0; i < terms.length; i++) {
var t = terms[i];
var hitschild = searchDocument.querySelectorAll("adrdox > index > term[value=\""+stemmer(t)+"\"] > result");
for(var a = 0; a < hitschild.length; a++) {
if(hitsObj[hitschild[a].getAttribute("decl")])
hitsObj[hitschild[a].getAttribute("decl")] += Number(hitschild[a].getAttribute("score"));
else
hitsObj[hitschild[a].getAttribute("decl")] = Number(hitschild[a].getAttribute("score"));
}
}

var hits = [];
for(name in hitsObj)
hits.push( { decl: name, score: hitsObj[name] } );

hits.sort(function(a, b) {
return Number(b.getAttribute("score")) - Number(a.getAttribute("score"));
return Number(b.score) - Number(a.score);
});

var container = document.getElementById("page-content");
@@ -56,7 +73,7 @@ function adrdox_search(searchTerm) {
container.appendChild(resultsElement);

for(var a = 0; a < hits.length; a++) {
var decl = searchDocument.querySelector("adrdox > listing decl[id=\""+hits[a].getAttribute("decl")+"\"]");
var decl = searchDocument.querySelector("adrdox > listing decl[id=\""+hits[a].decl+"\"]");
if(!decl) continue;

var dt = document.createElement("dt");
@@ -76,7 +93,7 @@ function adrdox_search(searchTerm) {

dt.appendChild(link);
dt.className = "search-result";
dt.setAttribute("data-score", hits[a].getAttribute("score"));
dt.setAttribute("data-score", hits[a].score);
resultsElement.appendChild(dt);

var dd = document.createElement("dd");
@@ -90,7 +107,7 @@ function adrdox_search(searchTerm) {
}

window.onhashchange = function() {
adrdox_search(location.hash.substring(2));
adrdox_search(decodeURIComponent(location.hash.substring(2)));
};

window.onload = function() {
@@ -107,7 +124,7 @@ window.onload = function() {
if(searchTerm)
location.href = location.href.substring(0, location.href.indexOf("?"));
searchTerm = location.hash.substring(2);
adrdox_search(searchTerm);
adrdox_search(decodeURIComponent(searchTerm));

var search = document.getElementById("search");
if(!search)

0 comments on commit b81ccef

Please sign in to comment.
You can’t perform that action at this time.