Skip to content

Commit

Permalink
Did improvements on get_recommendations:
Browse files Browse the repository at this point in the history
	Added show_details to placements model and engine types get_recommendations method
	Fixed visual similarity validate_config
	Added the item_type property to items model
	Added the property name to slots model
	Added a option to distribute the slots recommendations between them
	Added a new method get_slots to placements model. This method returns the recommendations by slots with fallbacks recommendations separated (this can be used for debug or to get batched recommendations)
	Added fallback logic to placements model
        Appended the item_type to each recommendation when placements get_recommendations
	Did the recommendations items unique
  • Loading branch information
dutradda committed Nov 10, 2016
1 parent 80f1476 commit 40e1a3a
Show file tree
Hide file tree
Showing 11 changed files with 468 additions and 59 deletions.
19 changes: 15 additions & 4 deletions myreco/engines/types/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,25 +58,36 @@ def validate_config(self):
def _validate_config(self, engine):
pass

def get_recommendations(self, session, filters, max_recos, **variables):
def get_recommendations(self, session, filters, max_recos, show_details, **variables):
rec_vector = self._build_rec_vector(session, **variables)

if rec_vector is not None:
[filter_.filter(session, rec_vector, ids) for filter_, ids in filters.items()]
return self._build_rec_list(session, rec_vector, max_recos)
return self._build_rec_list(session, rec_vector, max_recos, show_details)

return []

def _build_rec_vector(self, session, **variables):
pass

def _build_rec_list(self, session, rec_vector, max_recos):
def _build_rec_list(self, session, rec_vector, max_recos, show_details):
items_indices_map = ItemsIndicesMap(self.items_model)
best_indices = self._get_best_indices(rec_vector, max_recos)
best_items_keys = items_indices_map.get_items(session, best_indices)
return [msgpack.loads(item, encoding='utf-8') for item in session.redis_bind.hmget(

if show_details:
return [msgpack.loads(item, encoding='utf-8') for item in session.redis_bind.hmget(
self.items_model.__key__, best_items_keys) if item is not None]

else:
items_ids = []
for key in best_items_keys:
item = self.items_model()
item.set_ids(eval(key))
items_ids.append(item)

return items_ids

def _get_best_indices(self, rec_vector, max_recos):
if max_recos > rec_vector.size:
max_recos = rec_vector.size
Expand Down
4 changes: 2 additions & 2 deletions myreco/engines/types/visual_similarity/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ def _validate_config(self, engine):

if item_id_name not in item_type_schema_props:
raise ValidationError(message.format('item_id_name'),
instance=dict_inst, schema=item_type_schema_props)
instance=engine['configuration'], schema=item_type_schema_props)

elif aggregators_ids_name not in item_type_schema_props:
raise ValidationError(message.format('aggregators_ids_name'),
instance=dict_inst, schema=item_type_schema_props)
instance=engine['configuration'], schema=item_type_schema_props)
4 changes: 3 additions & 1 deletion myreco/items_types/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,9 @@ def _build_items_model(cls, item_type, store):
class_name = cls._build_class_name(item_type['name'], store['name'])
key = build_item_key(item_type['name'], store['id'])
id_names = item_type['schema']['id_names']
return RedisModelBuilder(class_name, key, id_names, None, metaclass=ItemsModelBaseMeta)
items_model = RedisModelBuilder(class_name, key, id_names, None, metaclass=ItemsModelBaseMeta)
items_model.__item_type__ = item_type
return items_model

@classmethod
def _build_items_collections_schema(cls, key, schema, id_names):
Expand Down
121 changes: 109 additions & 12 deletions myreco/placements/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from myreco.items_types.models import build_item_key
from falcon.errors import HTTPNotFound
from sqlalchemy.ext.declarative import AbstractConcreteBase, declared_attr
import random as random_
import sqlalchemy as sa
import hashlib
import json
Expand All @@ -41,6 +42,8 @@ class PlacementsModelBase(AbstractConcreteBase):
small_hash = sa.Column(sa.String(255), unique=True, nullable=False)
name = sa.Column(sa.String(255), unique=True, nullable=False)
ab_testing = sa.Column(sa.Boolean, default=False)
show_details = sa.Column(sa.Boolean, default=True)
distribute_recos = sa.Column(sa.Boolean, default=False)

@declared_attr
def store_id(cls):
Expand Down Expand Up @@ -72,25 +75,49 @@ class PlacementsModelRecommenderBase(PlacementsModelBase):

@classmethod
def get_recommendations(cls, req, resp):
cls._get_recommendations(req, resp)

@classmethod
def get_slots(cls, req, resp):
cls._get_recommendations(req, resp, True)

@classmethod
def _get_recommendations(cls, req, resp, by_slots=False):
placement = cls._get_placement(req, resp)
recommendations = []
session = req.context['session']
input_variables = req.context['parameters']['query_string']
show_details = placement.get('show_details')
distribute_recos = placement.get('distribute_recos')
recos = []
recos_key = 'slots' if by_slots else 'recommendations'

for slot in placement['variations'][0]['slots']:
engine = slot['engine']
items_model = cls._get_items_model(engine)
engine_vars, filters = \
cls._get_variables_and_filters(slot, items_model, input_variables)
engine_type = EngineTypeChooser(engine['type_name']['name'])(engine, items_model)
max_recos = slot['max_recos']
eng_recos = engine_type.get_recommendations(session, filters, max_recos, **engine_vars)
recommendations.extend(eng_recos)

if not recommendations:
slot_recos = {'fallbacks': []}
slot_recos['main'] = \
cls._get_recos_by_slot(slot, input_variables, session, show_details)

cls._get_fallbacks_recos(slot_recos, slot, input_variables, session, show_details)

if by_slots:
slot = {'name': slot['name'], 'item_type': slot['engine']['item_type']['name']}
slot['recommendations'] = slot_recos
recos.append(slot)
else:
cls._get_recos(recos, slot_recos, slot, distribute_recos)

if not recos:
raise HTTPNotFound()

resp.body = json.dumps(recommendations)
if not by_slots:
if distribute_recos:
recos = cls._distribute_recos(recos)

recos = cls._unique_recos(recos)

placement = {'name': placement['name'], 'small_hash': placement['small_hash']}
placement[recos_key] = recos

resp.body = json.dumps(placement)

@classmethod
def _get_placement(cls, req, resp):
Expand All @@ -103,6 +130,18 @@ def _get_placement(cls, req, resp):

return placements[0]

@classmethod
def _get_recos_by_slot(cls, slot, input_variables, session, show_details, max_recos=None):
engine = slot['engine']
items_model = cls._get_items_model(engine)
engine_vars, filters = \
cls._get_variables_and_filters(slot, items_model, input_variables)
engine_type = EngineTypeChooser(engine['type_name']['name'])(engine, items_model)
max_recos = slot['max_recos'] if max_recos is None else max_recos

return \
engine_type.get_recommendations(session, filters, max_recos, show_details, **engine_vars)

@classmethod
def _get_items_model(cls, engine):
items_types_model_key = build_item_key(engine['item_type']['name'])
Expand Down Expand Up @@ -152,6 +191,64 @@ def _get_variable_schema(cls, engine, engine_var):

return var['schema'], filter_input_schema

@classmethod
def _get_fallbacks_recos(cls, slot_recos, slot, input_variables, session, show_details):
if len(slot_recos['main']) != slot['max_recos']:
for fallback in slot['fallbacks']:
fallbacks_recos_size = \
sum([len(fallback) for fallback in slot_recos['fallbacks']])
max_recos = slot['max_recos'] - len(slot_recos['main']) - fallbacks_recos_size
if max_recos == 0:
break

fallback_recos = cls._get_recos_by_slot(
fallback, input_variables, session, show_details, max_recos)
slot_recos['fallbacks'].append(fallback_recos)

@classmethod
def _get_recos(cls, recos, slot_recos, slot, distribute_recos):
fallbacks_recos = []
[fallbacks_recos.extend(recos) for recos in slot_recos['fallbacks']]
slot_recos = slot_recos['main'] + fallbacks_recos

for reco in slot_recos:
reco['type'] = slot['engine']['item_type']['name']

if distribute_recos:
recos.append(slot_recos)
else:
recos.extend(slot_recos)

@classmethod
def _distribute_recos(cls, recos_list, random=True):
total_length = sum([len(recos) for recos in recos_list])
total_items = []
initial_pos = 0

for recos in recos_list:
if not recos:
continue

step = int(total_length / len(recos))

if random:
initial_pos = random_.randint(0, step-1)

positions = range(initial_pos, total_length, step)
zip_items_positions = zip(recos, positions)
total_items.extend(zip_items_positions)

random_.shuffle(total_items)
sorted_items = sorted(total_items, key=(lambda each: each[1]))

return [i[0] for i in sorted_items]

@classmethod
def _unique_recos(cls, recos):
unique = list()
[unique.append(reco) for reco in recos if not unique.count(reco)]
return unique

@classmethod
def _build_id_names_schema(cls, engine):
return {
Expand Down
10 changes: 10 additions & 0 deletions myreco/placements/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,12 @@
"responses": {"200": {"description": "Got"}}
}
},
"/placements/{small_hash}/slots": {
"get": {
"operationId": "get_slots",
"responses": {"200": {"description": "Got"}}
}
},
"definitions": {
"schema_without_required": {
"type": "object",
Expand All @@ -73,6 +79,8 @@
"store_id": {"type": "integer"},
"name": {"type": "string"},
"ab_testing": {"type": "boolean"},
"show_details": {"type": "boolean"},
"distribute_recos": {"type": "boolean"},
"variations": {"$ref": "#/definitions/variations"}
}
},
Expand All @@ -87,6 +95,8 @@
"store_id": {"type": "integer"},
"name": {"type": "string"},
"ab_testing": {"type": "boolean"},
"show_details": {"type": "boolean"},
"distribute_recos": {"type": "boolean"},
"variations": {"$ref": "#/definitions/variations"}
}
}
Expand Down
1 change: 1 addition & 0 deletions myreco/slots/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ class SlotsModelBase(AbstractConcreteBase):

id = sa.Column(sa.Integer, primary_key=True)
max_recos = sa.Column(sa.Integer, nullable=False)
name = sa.Column(sa.String(255), nullable=False)

@declared_attr
def engine_id(cls):
Expand Down
4 changes: 3 additions & 1 deletion myreco/slots/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
"additionalProperties": false,
"properties": {
"max_recos": {"type": "integer"},
"name": {"type": "string"},
"store_id": {"type": "integer"},
"engine_id": {"type": "integer"},
"fallbacks": {"$ref": "#/definitions/fallbacks"},
Expand All @@ -77,9 +78,10 @@
"items": {
"type": "object",
"additionalProperties": false,
"required": ["engine_id", "store_id", "engine_variables", "max_recos"],
"required": ["engine_id", "store_id", "engine_variables", "max_recos", "name"],
"properties": {
"max_recos": {"type": "integer"},
"name": {"type": "string"},
"store_id": {"type": "integer"},
"engine_id": {"type": "integer"},
"fallbacks": {"$ref": "#/definitions/fallbacks"},
Expand Down
2 changes: 1 addition & 1 deletion myreco/version.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,4 @@
# SOFTWARE.


VERSION = '0.9.1'
VERSION = '0.10.0'
Original file line number Diff line number Diff line change
Expand Up @@ -794,6 +794,7 @@ def filters_updater_app(redis, session):

slot = {
'max_recos': 10,
'name': 'test',
'store_id': 1,
'engine_id': 1,
'engine_variables': [{
Expand Down
Loading

0 comments on commit 40e1a3a

Please sign in to comment.