New recommender, different layouts, points system, sharing

app/forms.py

@@ -62,6 +62,7 @@ class SurveyForm(FlaskForm):
     education = RadioField('Wat is uw hoogst behaalde opleidingsniveau?',
         choices=[('1', 'Geen onderwijs / basisonderwijs / lagere school'),
             ('2', 'LBO / VBO / VMBO (kader- en beroepsgerichte leerweg)'), ('3', 'MAVO / eerste 3 jaar HAVO en VWO / VMBO (theoretische en gemengde leerweg)'),('4', 'MBO'), ('Grad', 'HAVO en VWO bovenbouw / WO-propedeuse'),('5', 'HBO / WO-bachelor of kandidaats '), ('6', 'WO-doctoraal of master')],validators=[DataRequired()])
+    pers2_q4 = RadioField('', choices=[(1,'helemaal<br/>mee oneens'),(2,'2    '),(3,'3    '),(4,'4    '),(5,'5    '), (6, '6    '), (7, 'helemaal<br/>mee eens')])
     submit = SubmitField('Afzenden')
 
 class rating(FlaskForm):

app/models.py

@@ -7,6 +7,7 @@
 from time import time
 import jwt
 from app import app
+from sqlalchemy_utils import aggregated
 
 class User(UserMixin, db.Model):
     id = db.Column(db.Integer, primary_key=True)
@@ -15,6 +16,10 @@ class User(UserMixin, db.Model):
     password_hash = db.Column(db.String(128))
     group = db.Column(db.Integer)
     first_login = db.Column(db.DateTime, index = True, default = datetime.utcnow)
+    points_stories = db.relationship('Points_stories', backref = 'user', lazy = 'dynamic')
+    points_invites = db.relationship('Points_invites', backref = 'user', lazy = 'dynamic')
+    points_ratings = db.relationship('Points_ratings', backref = 'user', lazy = 'dynamic')
+    points_logins = db.relationship('Points_logins', backref = 'user', lazy = 'dynamic')
     categories = db.relationship('Category', backref = 'user', lazy = 'dynamic')
     displayed_news = db.relationship('News', backref = 'user', lazy = 'dynamic')
     selected_news = db.relationship('News_sel', backref = 'user', lazy = 'dynamic')
@@ -39,6 +44,23 @@ def verify_reset_password_token(token):
     def __repr__(self):
         return '<User {}>'.format(self.username)
 
+    @aggregated('logins_sum', db.Column(db.Integer))
+    def sum_logins(self):
+        return db.func.sum(Points_logins.points_logins)
+    logins_sum = db.relationship('Points_logins')
+    @aggregated('ratings_sum', db.Column(db.Numeric(5,1)))
+    def sum_ratings(self):
+        return db.func.sum(Points_ratings.points_ratings)
+    ratings_sum = db.relationship('Points_ratings')
+    @aggregated('invites_sum', db.Column(db.Integer))
+    def sum_invites(self):
+        return db.func.sum(Points_invites.points_invites)
+    invites_sum = db.relationship('Points_invites')
+    @aggregated('stories_sum', db.Column(db.Integer))
+    def sum_stories(self):
+        return db.func.sum(Points_stories.points_stories)
+    stories_sum = db.relationship('Points_stories')
+
 class Category(db.Model):
     id = db.Column(db.Integer, primary_key =True)
     Binnenland = db.Column(db.Integer)
@@ -67,9 +89,43 @@ class News_sel(db.Model):
     starttime = db.Column(db.DateTime, index = True, default = datetime.utcnow)
     endtime = db.Column(db.DateTime, index = True, default = datetime.utcnow)
     time_spent = db.Column(db.Interval)
-    rating = db.Column(db.Integer)
+    rating = db.Column(db.Integer, default = 0)
     user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
 
+class User_invite(db.Model):
+    id = db.Column(db.Integer, primary_key = True)
+    timestamp = db.Column(db.DateTime, index = True, default = datetime.utcnow)
+    stories_read = db.Column(db.Integer, default = 0)
+    times_logged_in = db.Column(db.Integer, default = 0)
+    user_host = db.Column(db.Integer, db.ForeignKey('user.id'))
+    user_guest = db.Column(db.String(64), db.ForeignKey('user.username'))
+
+class Points_stories(db.Model):
+    id = db.Column(db.Integer, primary_key = True)
+    points_stories = db.Column(db.Integer, default = 0)
+    timestamp = db.Column(db.DateTime, index = True, default = datetime.utcnow)
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+
+class Points_invites(db.Model):
+    id = db.Column(db.Integer, primary_key = True)
+    user_guest_new = db.Column(db.String(64), db.ForeignKey('user_invite.user_guest'))
+    points_invites = db.Column(db.Integer, default = 0)
+    timestamp = db.Column(db.DateTime, index = True, default = datetime.utcnow)
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+
+class Points_ratings(db.Model):
+    id = db.Column(db.Integer, primary_key = True)
+    points_ratings = db.Column(db.Numeric(5,1), default = 0.0)
+    timestamp = db.Column(db.DateTime, index = True, default = datetime.utcnow)
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+
+class Points_logins(db.Model):
+    id = db.Column(db.Integer, primary_key = True)
+    points_logins = db.Column(db.Integer, default = 0)
+    timestamp = db.Column(db.DateTime, index = True, default = datetime.utcnow)
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+
 @login.user_loader
 def load_user(id):
     return User.query.get(int(id))
+

app/recommender.py

@@ -157,7 +157,82 @@ def past_behavior(self):
         final_list = recommender_selection + random_selection
         final_list = random.sample(final_list, len(final_list))
         return(final_list)
-
+
+    def past_behavior_topic(self):
+        '''
+         Recommends articles based on the topics of the stories the user has selected in the past (topics indicated by classifier)
+        '''
+        #Get the topics of the stories the user selected in the past and randomly select up to three of them
+        selected_articles = self.get_selected()
+        topics = [item['_source']['topic'] for item in selected_articles]
+        if len(topics) >= 3:
+            topic_list = random.sample(topics, 3)
+        else:
+            topic_list = topics
+        if len(topic_list) == 1: 
+            num_category_select = 6
+        elif len(topic_list) == 2: 
+            num_category_select = 3
+        elif len(topic_list) == 3: 
+            num_category_select = 2
+        else:
+            num_category_select = 0
+
+        #Retrieve new articles
+        new_articles = [self.doctype_last(s) for s in list_of_sources]
+        new_articles = [a for b in new_articles for a in b]
+
+        category_selection = []
+        for category in topic_list:
+            topic_selection = []
+            for item in new_articles:
+                if item['_source']['topic'] == category:
+                    topic_selection.append(item["_id"])
+            if len(topic_selection) > num_category_select:
+                topic_selection = random.sample(topic_selection, num_category_select)
+            for item in topic_selection:
+                category_selection.append(item)
+        if len(category_selection) < self.num_recommender:
+            newtry = self.num_more
+            new_articles = [self.doctype_last(s, num = newtry) for s in list_of_sources]
+            new_articles = [a for b in new_articles for a in b]
+            category_selection = []
+            for category in topic_list:
+                topic_selection = []
+                for item in new_articles:
+                    if item['_source']['topic'] == category:
+                        topic_selection.append(item['_id'])
+                    if len(topic_selection) > num_category_select:
+                        topic_selection = random.sample(topic_selection, num_category_select)
+                    for item in topic_selection:
+                        category_selection.append(item)
+
+      #Mark the selected articles as recommended, select random articles from the non-recommended articles (and get more if not enough unseen articles available), put the two lists together, randomize the ordering and return them        
+        recommender_selection = [a for a in new_articles if a["_id"] in category_selection]
+        for article in recommender_selection:
+            article['recommended'] = 1
+        num_random = self.num_select - len(recommender_selection)
+        random_list = [a for a in new_articles if a["_id"] not in category_selection]
+        try:
+            random_selection = random.sample(random_list, num_random)
+            for article in random_selection:
+                article['recommended'] = 0
+        except ValueError:
+            try:
+                newtry = self.num_more
+                articles = [self.doctype_last(s, num = newtry) for s in list_of_sources]
+                all_articles = [a for b in articles for a in b]
+                random_list = [a for a in all_articles if a["_id"] not in category_selection] 
+                random_selection = random.sample(random_list, self.num_select)
+            except:
+                random_selection = "not enough stories"
+                return(random_selection)
+        for article in random_selection:
+            article['recommended'] = 0
+        final_list = random_selection + recommender_selection
+        final_list = random.sample(final_list, len(final_list))
+        return(final_list)
+
     def category_selection_classifier(self):
         '''
         Uses a classifier to determine the topic categories of each article
@@ -181,7 +256,6 @@ def category_selection_classifier(self):
             num_category_select = 2
         else:
             num_category_select = 0
-
         #For each selected category retrieve the articles that fit this category (and randomly select if the list is longer than needed) and fill the rest with random articles (could also be more than normally as some topics might not appear in the article selection often enough)
         category_selection = []
         for category in sel_categories: