Skip to content
This repository has been archived by the owner on Oct 26, 2023. It is now read-only.

Commit

Permalink
Added cache and getting more images
Browse files Browse the repository at this point in the history
  • Loading branch information
@rtrevinnoc
rtrevinnoc committed Feb 7, 2021
1 parent fb8cb8b commit 6633d5b
Show file tree
Hide file tree
Showing 8 changed files with 76 additions and 41 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,4 @@ nohup.out
peer_registry/
ex.py
docs/build/
external_image_cache/
2 changes: 2 additions & 0 deletions docs/source/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ It is suggested to start with this configuration template, which is essentially
SECOND_NOTICE = "Proudly Hosted on <a href='https://uberspace.de/en/'>Uberspace</a>"
DONATE = "<a href='https://www.buymeacoffee.com/searchatfuture'>DONATE</a>"
COLABORATE = "<a href='https://github.com/rtrevinnoc/FUTURE'>COLABORATE</a>"
CACHE_TIMEOUT = 15
CACHE_THRESHOLD = 100
After you have configurated your FUTURE instance, but before you can start the server, you will be required to add a minimum of ~25 urls to your local index, by executing:

Expand Down
Empty file added event_log.txt
View file
Empty file.
50 changes: 33 additions & 17 deletions future.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,17 @@
from forms import *
# from werkzeug.middleware.proxy_fix import ProxyFix
from werkzeug.contrib.fixers import ProxyFix
from flask_caching import Cache
from werkzeug.utils import secure_filename
from base64 import b64decode
from symspellpy.symspellpy import SymSpell, Verbosity
from bs4 import BeautifulSoup
from config import HOST_NAME, PEER_PORT, CONTACT, MAINTAINER, FIRST_NOTICE, SECOND_NOTICE, DONATE, COLABORATE
from config import HOST_NAME, PEER_PORT, CONTACT, MAINTAINER, FIRST_NOTICE, SECOND_NOTICE, DONATE, COLABORATE, CACHE_TIMEOUT, CACHE_THRESHOLD

bson.loads = bson.BSON.decode
bson.dumps = bson.BSON.encode

global port, hostIP, hostname, listOfPeers, app, hnswImagesLookup, imageDBIndex, analyticsDBIndex, spellChecker, dirname, numberOfURLs, goodSearxInstances, headersForSearx
global port, hostIP, hostname, listOfPeers, app, hnswImagesLookup, imageDBIndex, analyticsDBIndex, spellChecker, dirname, numberOfURLs, goodSearxInstances, headersForSearx, cache
port = int(PEER_PORT)
hostIP = requests.get(
"https://api.ipify.org?format=json").json()["ip"] + ":" + str(port)
Expand Down Expand Up @@ -75,6 +76,7 @@
spellChecker.load_dictionary(
"./frequency_dictionary_en_82_765.txt", 0, 1
) # LAST TWO PARAMETERS ARE (COLUMN TERM, FREQUENCY TERM) LOCATIONS IN DICTIONARY FILE
cache = Cache(app, config={'CACHE_TYPE': 'filesystem','CACHE_DEFAULT_TIMEOUT': CACHE_TIMEOUT, 'CACHE_THRESHOLD': CACHE_THRESHOLD, 'CACHE_DIR': './external_image_cache'})


def sendRegisterRequestToPeer(url):
Expand Down Expand Up @@ -127,7 +129,7 @@ def sendAnswerRequestToPeer(url, query, queryVector, queryLanguage,
print("#######################")
if peer == hostIP or peer == hostname:
print("Same as origin")
return {"urls": []} #, "images": []}
return {"urls": []}
else:
try:
r = requests.get("http://" + peer + "/_answerPeer",
Expand All @@ -143,7 +145,6 @@ def sendAnswerRequestToPeer(url, query, queryVector, queryLanguage,
print("Obtained with http")
return {
"urls": list(zip(result["urls"], result["url_scores"]))
# "images": list(zip(result["images"], result["images_scores"]))
}
except:
try:
Expand All @@ -160,12 +161,10 @@ def sendAnswerRequestToPeer(url, query, queryVector, queryLanguage,
print("Obtained with https")
return {
"urls": list(zip(result["urls"], result["url_scores"]))
# "images":
# list(zip(result["images"], result["images_scores"]))
}
except:
print("Could not connect with peer")
return {"urls": []} #, "images": []}
return {"urls": []}


def sendImagesAnswerRequestToPeer(url, query, queryVector, queryLanguage,
Expand Down Expand Up @@ -688,6 +687,7 @@ def fetchSearxVideos():


@app.route('/_retrieveImage')
@cache.cached(timeout=15, query_string=True)
def _retrieveImage():
url = request.args.get("url", "", type=str)
if url.startswith("//"):
Expand Down Expand Up @@ -880,40 +880,56 @@ def _updateAnswer():
queryLanguage = inferLanguage(query)

urls = loadMoreUrls(q_vec, queryLanguage, numberOfURLs, page)
images = loadMoreImages(q_vec, 50, page)

listOfDataFromPeers = asyncio.run(
getDataFromPeers(query, q_vec, queryLanguage, numberOfURLs, page))
if len(listOfDataFromPeers) > 0:
listOfUrlsFromHost = list(zip(urls["urls"], urls["scores"]))
listOfImagesFromHost = list(zip(images["images"], images["scores"]))
listOfUrlsFromPeers = [pack["urls"] for pack in listOfDataFromPeers][0]
bigListOfUrls = listOfUrlsFromHost + listOfUrlsFromPeers
bigListOfUrls.sort(key=lambda x: x[1])
bigListOfUrls = [url[0] for url in bigListOfUrls]
else:
bigListOfUrls = urls["urls"]

return jsonify(
result={
"urls":
list({frozenset(item.items()): item
for item in bigListOfUrls}.values())
})

@app.route("/_updateImages", methods=["GET", "POST"])
def _updateImages():
query = request.args.get("query", 0, type=str)
page = request.args.get("page", 0, type=int)
q_vec = getSentenceMeanVector(query)
queryLanguage = inferLanguage(query)

images = loadMoreImages(q_vec, 15, page)

listOfDataFromPeers = asyncio.run(
getImagesFromPeers(query, q_vec, queryLanguage, numberOfURLs, page))
if len(listOfDataFromPeers) > 0:
listOfImagesFromHost = list(zip(images["images"], images["scores"]))
listOfImagesFromPeers = [
pack["images"] for pack in listOfDataFromPeers
][0]
bigListOfUrls = listOfUrlsFromHost + listOfUrlsFromPeers
bigListOfImages = list(
set(listOfImagesFromHost + listOfImagesFromPeers))
bigListOfUrls.sort(key=lambda x: x[1])
bigListOfImages.sort(key=lambda x: x[1])
bigListOfUrls = [url[0] for url in bigListOfUrls]
bigListOfImages = [
image[0] for image in bigListOfImages if image[0] != ''
]
else:
bigListOfUrls = urls["urls"]
bigListOfImages = images["images"]

return jsonify(
result={
"urls":
list({frozenset(item.items()): item
for item in bigListOfUrls}.values()),
"images":
bigListOfImages
})


@app.route("/_midnightcypher")
def _midnightcypher():
query = request.args.get("query", 0, type=str)
Expand Down
4 changes: 2 additions & 2 deletions static/css/color/index.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,12 @@ body {
background-color: #aaaaaa;
}

#small_summary, #load_more_items {
#small_summary, .load_more_items {
background-color: #1C1C1C;
color: #bfbfbf;
}

#load_more_items {
.load_more_items {
text-align: center;
}

Expand Down
8 changes: 4 additions & 4 deletions static/css/default/INDEX.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@ label {
white-space: nowrap;
}

#small_summary, #load_more_items {
#small_summary, .load_more_items {
border-radius: 8px;
font-size: 2vh;
padding: 12px;
}

#load_more_items {
.load_more_items {
margin-bottom: 10px;
max-width: 200px;
margin: 0 auto;
Expand Down Expand Up @@ -98,7 +98,7 @@ p#gathered {
margin-bottom: -2vh;
}

#images_content {
#images_list {
display: grid;
grid-gap: 10px;
grid-template-columns: repeat(auto-fill, minmax(calc(24.1vh + 3.3vw),1fr));
Expand Down Expand Up @@ -556,7 +556,7 @@ p.body, p.link_paragraph, p.link_paragraph2 {
margin-bottom: 2vh;
}

#images_content {
#images_list {
grid-template-columns: repeat(3, 1fr);
}

Expand Down
46 changes: 29 additions & 17 deletions static/js/index_animations.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ $(function() {
$("#links_list").empty()
$("#links_description").empty()
$("#welcome").fadeOut("fast");
images.empty()
$("images_list").empty()
videos.empty()
maps.empty()
summary.empty()
Expand Down Expand Up @@ -92,7 +92,7 @@ $(function() {
$("#links_list").empty()
$("#links_description").empty()
$("#welcome").fadeOut("fast");
images.empty()
$("images_list").empty()
videos.empty()
maps.empty()
summary.empty()
Expand Down Expand Up @@ -148,7 +148,7 @@ $(function() {
$("#links_list").empty()
$("#links_description").empty()
$("#welcome").fadeOut("fast");
images.empty()
$("images_list").empty()
videos.empty()
maps.empty()
summary.empty()
Expand Down Expand Up @@ -324,6 +324,7 @@ $(function() {
var start_time = date.getTime();
$(".hex").addClass("rotate");
counter = 0
current_links_page = 1

$.getJSON($SCRIPT_ROOT + '/_fetchSearxResults', {
query: input
Expand All @@ -348,8 +349,6 @@ $(function() {
}, function(data) {
response = data.result

var current_page = 1

if (searchbar.val() != response["corrected"]) {
searchbar.val(response["corrected"])
searchbar.animate({
Expand Down Expand Up @@ -385,14 +384,15 @@ $(function() {
function get_images(input) {
$(".hex").addClass("rotate");
counter = 0
current_images_page = 1

$.getJSON($SCRIPT_ROOT + '/_fetchSearxImages', {
query: input
}, function(data) {
searx_response = data.result

searx_response["images"].reverse().forEach(function(image) {
images.prepend('<div class="grid-item"><a href=' + image["parentUrl"] + '><img class="image-item" src="/_retrieveImage?url=' + image["url"] + '" alt="Not available"></a></div>')
$("#images_list").prepend('<div class="grid-item"><a href=' + image["parentUrl"] + '><img class="image-item" src="/_retrieveImage?url=' + image["url"] + '" alt="Not available"></a></div>')
});

counter += 1
Expand All @@ -407,7 +407,7 @@ $(function() {
response = data.result

response["images"].forEach(function(image) {
images.append('<div class="grid-item"><a href=' + image["parentUrl"] + '><img class="image-item" src="/_retrieveImage?url=' + image["url"] + '" alt="Not available"></a></div>')
$("#images_list").append('<div class="grid-item"><a href=' + image["parentUrl"] + '><img class="image-item" src="/_retrieveImage?url=' + image["url"] + '" alt="Not available"></a></div>')
});

counter += 1
Expand Down Expand Up @@ -531,21 +531,33 @@ $(function() {
}
})

$('#load_more_items').click(function(e) {
$('#load_more_links').click(function(e) {
$(".hex").addClass("rotate");
$.getJSON($SCRIPT_ROOT + '/_updateAnswer', {
query: searchbar.val(),
page: (current_page + 1)
query: current_query,
page: (current_links_page + 1)
}, function(data) {
$(".hex").removeClass("rotate")
response = data.result
response["urls"].forEach(function(url) {
$('<div class="url_item"><p class="link_paragraph"><span class="domain"><a href="' + url["url"] + '">' + url["header"] + '</a></span></p><p class="link_paragraph2"><span class="link"><a href="' + url["url"] + '">' + url["url"] + '</a></span></p><p class="body searchable">' + url["body"] + '<p></div>').insertBefore("#load_more_items");
$("#links_list").append('<div class="url_item"><p class="link_paragraph"><span class="domain"><a href="' + url["url"] + '">' + url["header"] + '</a></span></p><p class="link_paragraph2"><span class="link"><a href="' + url["url"] + '">' + url["url"] + '</a></span></p><p class="body searchable">' + url["body"] + '<p></div>');
});
current_links_page += 1;
});
});

$('#load_more_images').click(function(e) {
$(".hex").addClass("rotate");
$.getJSON($SCRIPT_ROOT + '/_updateImages', {
query: current_query,
page: (current_images_page + 1)
}, function(data) {
$(".hex").removeClass("rotate")
response = data.result
response["images"].forEach(function(image) {
$("#images_list").append('<div class="grid-item"><a href=' + image["parentUrl"] + '><img class="image-item" src="/_retrieveImage?url=' + image["url"] + '" alt="Not available"></a></div>')
});
//response["images"].forEach(function(image) {
//images.append('<div class="grid-item"><a href=' + image["parentUrl"] + '><img class="image-item" src="/_retrieveImage?url=' + image["url"] + '" alt="Not available"></a></div>')
//});
current_page = current_page + 1;
current_images_page += 1;
});
});

Expand Down Expand Up @@ -610,7 +622,7 @@ $(function() {
$("#links_list").empty()
$("#links_description").empty()
$("#welcome").fadeOut("fast");
images.empty()
$("images_list").empty()
videos.empty()
maps.empty()
summary.empty()
Expand All @@ -629,7 +641,7 @@ $(function() {
$("#links_list").empty()
$("#links_description").empty()
$("#welcome").fadeOut("fast");
images.empty()
$("images_list").empty()
videos.empty()
maps.empty()
summary.empty()
Expand Down
6 changes: 5 additions & 1 deletion templates/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,13 @@ <h1 id="title"><a href="/">FUTURE</a></h1>
</div>
<div id="links_list">
</div>
<div id="load_more_items"><span>Load more items<span></div>
<div class="load_more_items" id="load_more_links"><span>Load more items<span></div>
</div>
<div id="images_content">
<div id="images_list">
</div>
<br>
<div class="load_more_items" id="load_more_images"><span>Load more items<span></div>
</div>
<div id="maps_content">
</div>
Expand Down

0 comments on commit 6633d5b

Please sign in to comment.