Skip to content
This repository has been archived by the owner on Oct 26, 2023. It is now read-only.

Commit

Permalink
Improved result gathering from distributed network
Browse files Browse the repository at this point in the history
  • Loading branch information
@rtrevinnoc
rtrevinnoc committed Feb 13, 2021
1 parent 6c72168 commit a50ef5e
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 20 deletions.
7 changes: 6 additions & 1 deletion docs/source/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,17 @@
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
==================================
Welcome to FUTURE's documentation!
==================================

Contents
========

.. toctree::
:maxdepth: 2
:caption: Contents:

api

FUTURE is a decentralized, open-source and privacy focused search engine.
It is capable of running completely standalone, but it usually complements its own results with others sourced from meta-search at public Searx instances.
Expand Down
54 changes: 35 additions & 19 deletions future.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ def sendRegisterRequestToPeer(url):


def sendAnswerRequestToPeer(url, query, queryVector, queryLanguage,
numberOfURLs, numberOfPage):
numberOfURLs, numberOfPage, minimumScore):
peer = url
queryVector = json.dumps(queryVector.tolist())
print("#######################")
Expand All @@ -145,7 +145,8 @@ def sendAnswerRequestToPeer(url, query, queryVector, queryLanguage,
'q_vec': queryVector,
'queryLanguage': queryLanguage,
'numberOfURLs': numberOfURLs,
'numberOfPage': numberOfPage
'numberOfPage': numberOfPage,
'minimumScore': minimumScore
},
timeout=10)
result = r.json()["result"]
Expand All @@ -159,7 +160,8 @@ def sendAnswerRequestToPeer(url, query, queryVector, queryLanguage,
'q_vec': queryVector,
'queryLanguage': queryLanguage,
'numberOfURLs': numberOfURLs,
'numberOfPage': numberOfPage
'numberOfPage': numberOfPage,
'minimumScore': minimumScore
},
timeout=10)
result = r.json()["result"]
Expand All @@ -173,7 +175,7 @@ def sendAnswerRequestToPeer(url, query, queryVector, queryLanguage,


def sendImagesAnswerRequestToPeer(url, query, queryVector, queryLanguage,
numberOfURLs, numberOfPage):
numberOfURLs, numberOfPage, minimumScore):
peer = url
queryVector = json.dumps(queryVector.tolist())
print("#######################")
Expand All @@ -191,7 +193,8 @@ def sendImagesAnswerRequestToPeer(url, query, queryVector, queryLanguage,
'q_vec': queryVector,
'queryLanguage': queryLanguage,
'numberOfURLs': numberOfURLs,
'numberOfPage': numberOfPage
'numberOfPage': numberOfPage,
'minimumScore': minimumScore
},
timeout=10)
result = r.json()["result"]
Expand All @@ -207,7 +210,8 @@ def sendImagesAnswerRequestToPeer(url, query, queryVector, queryLanguage,
'q_vec': queryVector,
'queryLanguage': queryLanguage,
'numberOfURLs': numberOfURLs,
'numberOfPage': numberOfPage
'numberOfPage': numberOfPage,
'minimumScore': minimumScore
},
timeout=10)
result = r.json()["result"]
Expand Down Expand Up @@ -258,15 +262,15 @@ def sendImagesAnswerRequestToPeer(url, query, queryVector, queryLanguage,


async def getDataFromPeers(query, queryVector, queryLanguage, numberOfURLs,
numberOfPage):
numberOfPage, minimumScore):
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
loop = asyncio.get_event_loop()
futures = [
loop.run_in_executor(
executor,
functools.partial(sendAnswerRequestToPeer, peer, query,
queryVector, queryLanguage, numberOfURLs,
numberOfPage)) for peer in listOfPeers
numberOfPage, minimumScore)) for peer in listOfPeers
]
listOfResponses = []
for response in await asyncio.gather(*futures):
Expand All @@ -275,15 +279,15 @@ async def getDataFromPeers(query, queryVector, queryLanguage, numberOfURLs,


async def getImagesFromPeers(query, queryVector, queryLanguage, numberOfURLs,
numberOfPage):
numberOfPage, minimumScore):
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
loop = asyncio.get_event_loop()
futures = [
loop.run_in_executor(
executor,
functools.partial(sendImagesAnswerRequestToPeer, peer, query,
queryVector, queryLanguage, numberOfURLs,
numberOfPage)) for peer in listOfPeers
numberOfPage, minimumScore)) for peer in listOfPeers
]
listOfResponses = []
for response in await asyncio.gather(*futures):
Expand Down Expand Up @@ -387,9 +391,10 @@ def answer(query: str, page: int) -> jsonify:
}

urls = loadMoreUrls(q_vec, queryLanguage, numberOfURLs, page)
minimumScore = min(urls["scores"])

listOfDataFromPeers = asyncio.run(
getDataFromPeers(query, q_vec, queryLanguage, numberOfURLs, page))
getDataFromPeers(query, q_vec, queryLanguage, numberOfURLs, page, minimumScore))
if len(listOfDataFromPeers) > 0:
listOfUrlsFromHost = list(zip(urls["urls"], urls["scores"]))
listOfUrlsFromPeers = list(
Expand Down Expand Up @@ -445,9 +450,10 @@ def answerImages(query: str, page: int) -> jsonify:
return {"images": []}

images = loadMoreImages(q_vec, 10, page)
minimumScore = min(images["scores"])

listOfDataFromPeers = asyncio.run(
getImagesFromPeers(query, q_vec, queryLanguage, numberOfURLs, page))
getImagesFromPeers(query, q_vec, queryLanguage, numberOfURLs, page, minimumScore))
if len(listOfDataFromPeers) > 0:
listOfImagesFromHost = list(zip(images["images"], images["scores"]))
listOfImagesFromPeers = list(
Expand Down Expand Up @@ -666,16 +672,18 @@ def fetchSearxVideos():
def _retrieveImage():
url = request.args.get("url", "", type=str)
if url.startswith("//"):
url = "http:" + url
image = requests.get(url, allow_redirects=True)
try:
image = requests.get("http:"url, allow_redirects=True)
except:
image = requests.get("https:"url, allow_redirects=True)

pic = Image.open(io.BytesIO(image.content))
pic.thumbnail((480, 480), Image.LANCZOS)

img_io = io.BytesIO()
pic.save(img_io, 'PNG', optimize=True)
img_io.seek(0)
return send_file(img_io, mimetype='image/jpeg')
return send_file(img_io, mimetype='image/png')


@app.route('/sw.js', methods=['GET'])
Expand Down Expand Up @@ -836,8 +844,12 @@ def _answerPeer():
queryLanguage = request.args.get("queryLanguage", 0, type=str)
numberOfURLs = request.args.get("numberOfURLs", 0, type=int)
numberOfPage = request.args.get("numberOfPage", 1, type=int)
return jsonify(result=answerPeer(query, q_vec, queryLanguage, numberOfURLs,
numberOfPage))
minimumScore = request.args.get("minimumScore", 0, type=float)
result = answerPeer(query, q_vec, queryLanguage, numberOfURLs, numberOfPage)
if max(result["url_scores"]) >= minimumScore:
return jsonify(result=result)
else:
raise Exception("No relevant results.")


@app.route("/_answerPeerImages")
Expand All @@ -848,8 +860,12 @@ def _answerPeerImages():
queryLanguage = request.args.get("queryLanguage", 0, type=str)
numberOfURLs = request.args.get("numberOfURLs", 0, type=int)
numberOfPage = request.args.get("numberOfPage", 1, type=int)
return jsonify(result=answerPeerImages(query, q_vec, queryLanguage,
numberOfURLs, numberOfPage))
minimumScore = request.args.get("minimumScore", 0, type=float)
result = answerPeerImages(query, q_vec, queryLanguage, numberOfURLs, numberOfPage)
if max(result["images_scores"]) >= minimumScore:
return jsonify(result=result)
else:
raise Exception("No relevant results.")


if __name__ == "__main__":
Expand Down

0 comments on commit a50ef5e

Please sign in to comment.