add server

Makefile

@@ -15,4 +15,18 @@ download:
 	gcloud compute scp "keras--tf-gpu":~/images_similarity/submission submission/ --recurse --compress --zone europe-west1-d
 
 start:
-	gcloud compute instances start keras--tf-gpu --zone=europe-west1-d
+	gcloud compute instances start keras--tf-gpu --zone=europe-west1-d
+
+ssh_site:
+	gcloud compute --project "mltest-180907" ssh --zone "europe-west1-d" "kawaii-search"
+
+copy_site:
+	gcloud compute ssh --command "rm -rf ~/images_similarity/src" --project "mltest-180907" --zone "europe-west1-d" "kawaii-search"
+	gcloud compute ssh --command "rm -rf ~/images_similarity/templates" --project "mltest-180907" --zone "europe-west1-d" "kawaii-search"
+	gcloud compute scp templates "kawaii-search":~/images_similarity/templates --recurse --zone europe-west1-d --compress
+	gcloud compute scp src "kawaii-search":~/images_similarity/src --recurse --zone europe-west1-d --compress
+
+copy_site_data:
+	gcloud compute scp submission/images_order_-70232735.csv "kawaii-search":~/images_similarity/submission/images_order_-70232735.csv --zone europe-west1-d
+	gcloud compute scp data/photos_-70232735.csv "kawaii-search":~/images_similarity/data/photos_-70232735.csv --zone europe-west1-d
+	gcloud compute scp submission/images_vec_-70232735.npz "kawaii-search":~/images_similarity/submission/images_vec_-70232735.npz --zone europe-west1-d

requirements.txt

@@ -1,7 +1,9 @@
+pandas
 flask
 keras
 numpy
 matplotlib
 pillow
 scipy
-sklearn
+sklearn
+h5py

src/database.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+
+from settings import config
+
+
+def pictures_repo(owner_id):
+    df = pd.read_csv(config.info_path(owner_id))
+
+    def get_path(small_path):
+        fname = small_path.split('/')[-1]
+        parts = fname.split('_')
+        id, date = int(parts[0]), int(parts[1])
+        res = df[(df['id'] == id) & (df['date'] == date)]
+        if res.shape[0] != 0:
+            post = '{}_{}'.format(res['owner_id'].values[0], res['id'].values[0])
+            return res['big'].values[0], vk_link(post)
+        else:
+            return
+
+    return get_path
+
+
+def vk_link(post):
+    return "https://vk.com/feed?z=photo{}".format(post)

src/get_images.py

@@ -18,11 +18,11 @@ def get_limit(session, owner_id):
     return 50000
 
 
-def save(path, iter, limit):
+def save(path, iter_, limit):
     done = 0.0
     with open(path, "a") as fd:
         fd.write(','.join(KEYS) + "\n")
-        for i in iter:
+        for i in iter_:
             done += 1
             fd.write(','.join(line(i)) + "\n")
             if done % 100 == 0:

src/imlucky.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+from database import pictures_repo
+from predict_dummy import load_predictor, random
+
+owner_id = -70232735
+
+print("Loading imlucky")
+pred = load_predictor(owner_id)
+rf = random(owner_id)
+repo = pictures_repo(owner_id)
+print("Imlucky loaded")
+
+def imlucky():
+    f = rf()
+    files = pred(f, 12)
+    return [(repo(file), d) for file, d in files]

src/predict.py

@@ -22,8 +22,8 @@ def _vectorize(path, model):
     return pred.ravel()
 
 
-def _similar(vec, knn, filenames):
-    dist, indices = knn.kneighbors(vec.reshape(1, -1), n_neighbors=6)
+def _similar(vec, knn, filenames, n_neighbors=6):
+    dist, indices = knn.kneighbors(vec.reshape(1, -1), n_neighbors=n_neighbors)
     dist, indices = dist.flatten(), indices.flatten()
     return [(filenames[indices[i]], dist[i]) for i in range(len(indices))]
 
@@ -37,14 +37,14 @@ def load_predictor(owner_id):
     knn = NearestNeighbors(metric='cosine', algorithm='brute')
     knn.fit(vecs)
 
-    def similarity(file_path):
+    def similarity(file_path, n_neighbors=6):
         vec = _vectorize(file_path, model)
-        return _similar(vec, knn, filenames)
+        return _similar(vec, knn, filenames, n_neighbors)
 
     return similarity
 
 
-def random_file(owner_id):
+def random(owner_id):
     filenames = open(config.images_order(owner_id), 'r').readline().split(',')
 
     def rf():

src/predict_dummy.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+import numpy as np
+from sklearn.neighbors import NearestNeighbors
+
+from settings import config
+from sparce import load_sparse_matrix
+
+
+def _similar(vec, knn, filenames, n_neighbors=6):
+    dist, indices = knn.kneighbors(vec.reshape(1, -1), n_neighbors=n_neighbors)
+    dist, indices = dist.flatten(), indices.flatten()
+    return [(filenames[indices[i]], dist[i]) for i in range(len(indices))]
+
+
+def load_predictor(owner_id):
+    print("load predictor")
+    filenames = open(config.images_order(owner_id), 'r').readline().split(',')
+    vecs = load_sparse_matrix(config.vectors_path(owner_id))
+    knn = NearestNeighbors(metric='cosine', algorithm='brute')
+    knn.fit(vecs)
+
+    def similarity(vec, n_neighbors=6):
+        return _similar(vec, knn, filenames, n_neighbors)
+
+    print("Predictor loaded")
+    return similarity
+
+
+def random(owner_id):
+    print("Preparing random generator")
+    vecs = load_sparse_matrix(config.vectors_path(owner_id)).toarray()
+    s = vecs.shape[0]
+
+    def rf():
+        return vecs[np.random.randint(0, s)]
+
+    print("Random generator is ready")
+    return rf

src/server.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+from flask import Flask, render_template
+
+from imlucky import imlucky
+
+app = Flask(__name__, template_folder='../templates', static_folder='../data')
+
+
+@app.route('/', methods=['GET'])
+def index():
+    return render_template('index.html')
+
+
+@app.route('/', methods=['POST'])
+def imlucky_action():
+    files = imlucky()
+    return render_template('index.html', files=files)
+
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0', port=5000)

src/settings/config.py

@@ -2,21 +2,22 @@
 
 pool_size = 20
 
-def images_path(onwer_id):
-    return 'data/images_{}'.format(onwer_id)
 
+def images_path(owner_id):
+    return 'data/images_{}'.format(owner_id)
 
-def images_glob_path(onwer_id):
-    return 'data/images_{}/*.jpg'.format(onwer_id)
 
+def images_glob_path(owner_id):
+    return 'data/images_{}/*.jpg'.format(owner_id)
 
-def images_order(onwer_id):
-    return 'submission/images_order_{}.csv'.format(onwer_id)
 
+def images_order(owner_id):
+    return 'submission/images_order_{}.csv'.format(owner_id)
 
-def info_path(onwer_id):
-    return 'data/photos_{}.csv'.format(onwer_id)
 
+def info_path(owner_id):
+    return 'data/photos_{}.csv'.format(owner_id)
 
-def vectors_path(onwer_id):
-    return 'submission/images_vec_{}.npz'.format(onwer_id)
+
+def vectors_path(owner_id):
+    return 'submission/images_vec_{}.npz'.format(owner_id)

templates/index.html

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Kawaii Search</title>
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta.2/css/bootstrap.min.css"
+          integrity="sha384-PsH8R72JQ3SOdhVi3uxftmaW6Vc51MKb0q5P2rRUpPvrszuE4W1povHYgTpBfshb" crossorigin="anonymous">
+</head>
+<body>
+<div class="container">
+    <h1>Kawaii search</h1>
+    <p><a href="https://github.com/blan4/ImagesSimilarityDemo">GitHub</a></p>
+
+    <section>
+        <form method="POST" enctype="multipart/form-data">
+            <button type="submit" class="btn btn-primary">I'm lucky</button>
+        </form>
+    </section>
+
+    <section>
+        {% for file in files %}
+        <figure style="float: left; margin-right: 20px; margin-bottom: 20px;">
+            <a href="{{file[0][1]}}" target="_blank"><img src="{{ file[0][0] }}" height="300px"></a>
+            <figcaption>{{ "{0:.4f}".format(file[1]) }}</figcaption>
+        </figure>
+        {% endfor %}
+    </section>
+</div>
+</body>
+</html>