From b63632d9d4edf918489b7b99d58740c9c7eb7d93 Mon Sep 17 00:00:00 2001 From: Natay Aberra Date: Fri, 4 Jun 2021 08:04:11 -0400 Subject: [PATCH 01/12] update --- biostar/forum/ajax.py | 15 ++++++++++- biostar/forum/management/commands/tasks.py | 30 ++++++++++++++++----- biostar/forum/models.py | 31 ++++++++++++++++++++++ biostar/forum/signals.py | 17 ++++++++++-- 4 files changed, 84 insertions(+), 9 deletions(-) diff --git a/biostar/forum/ajax.py b/biostar/forum/ajax.py index 18d8afebe..7e943192a 100644 --- a/biostar/forum/ajax.py +++ b/biostar/forum/ajax.py @@ -20,7 +20,7 @@ from biostar.accounts.models import Profile, User from . import auth, util, forms, tasks, search, views, const, moderate -from .models import Post, Vote, Subscription, delete_post_cache, SharedLink +from .models import Post, Vote, Subscription, delete_post_cache, SharedLink, Diff @@ -470,6 +470,19 @@ def email_disable(request, uid): return ajax_success(msg='Disabled messages') +def view_diff(request, uid): + """ + View diffs made to a post. + """ + + # View changes made in a post + post = Post.objects.filter(uid=uid).first() + + diff = Diff.objects.filter(post=post) + + return + + def similar_posts(request, uid): """ Return a feed populated with posts similar to the one in the request. diff --git a/biostar/forum/management/commands/tasks.py b/biostar/forum/management/commands/tasks.py index a1563edf0..ab365ebf9 100644 --- a/biostar/forum/management/commands/tasks.py +++ b/biostar/forum/management/commands/tasks.py @@ -6,7 +6,7 @@ from biostar import VERSION from django.core.management.base import BaseCommand from biostar.forum import models, util, tasks -from biostar.forum.models import Post +from biostar.forum.models import Post, Diff from django.conf import settings from biostar.accounts.models import User from biostar.utils.decorators import timeit @@ -17,8 +17,8 @@ BACKUP_DIR = os.path.join(settings.BASE_DIR, 'export', 'backup') -BUMP, UNBUMP, AWARD = 'bump', 'unbump', 'award' -CHOICES = [BUMP, UNBUMP, AWARD] +CHOICES = ['bump', 'unbump', 'award', 'diffs'] +BUMP, UNBUMP, AWARD, DIFFS = CHOICES def bump(uids, **kwargs): @@ -77,19 +77,37 @@ def awards(limit=50, **kwargs): return +def create_diffs(nbatch=100000, **kwargs): + """ + Bulk create diffs + """ + + targets = Post.objects.all() + + def batch(): + for post in targets: + diff = Diff(post=post, initial=post.content, current=post.content, created=post.creation_date, + edited=post.lastedit_date) + + yield diff + + models.Diff.objects.bulk_create(objs=batch(), batch_size=nbatch) + class Command(BaseCommand): help = 'Preform action on list of posts.' def add_arguments(self, parser): parser.add_argument('--uids', '-u', type=str, required=False, default='', help='List of uids') - parser.add_argument('--action', '-a', type=str, required=True, choices=CHOICES, default='',help='Action to take.') - parser.add_argument('--limit', dest='limit', type=int, default=100, help='Limit how many users/posts to process.'), + parser.add_argument('--action', '-a', type=str, required=True, choices=CHOICES, default='', + help='Action to take.') + parser.add_argument('--limit', dest='limit', type=int, default=100, + help='Limit how many users/posts to process.'), def handle(self, *args, **options): action = options['action'] - opts = {BUMP: bump, UNBUMP: unbump, AWARD: awards} + opts = {BUMP: bump, UNBUMP: unbump, AWARD: awards, DIFFS: create_diffs} func = opts[action] # print() diff --git a/biostar/forum/models.py b/biostar/forum/models.py index 5028b47ca..48e0a431e 100644 --- a/biostar/forum/models.py +++ b/biostar/forum/models.py @@ -631,6 +631,37 @@ def save(self, *args, **kwargs): def uid(self): return self.pk + +class Diff(models.Model): + + # Initial content state + initial = models.TextField(default='') + + # Current content state + current = models.TextField(default='') + + # Date the initial content was created + created = models.DateTimeField(auto_now_add=True) + + # Date the current content was created + edited = models.DateTimeField(auto_now=True) + + # Post this diff belongs to + post = models.ForeignKey(Post, on_delete=models.CASCADE) + + def save(self, *args, **kwargs): + + self.initial = self.initial or self.post.content + self.current = self.current or self.initial + + self.created = self.created or self.post.lastedit_date + self.edited = self.edited or self.created + + super(Diff, self).save(*args, **kwargs) + + + + class Log(models.Model): """ Represents moderation actions diff --git a/biostar/forum/signals.py b/biostar/forum/signals.py index 1e2b5ffe8..899e481fd 100644 --- a/biostar/forum/signals.py +++ b/biostar/forum/signals.py @@ -1,10 +1,10 @@ import logging -from django.db.models.signals import post_save +from django.db.models.signals import post_save, pre_save from django.dispatch import receiver from taggit.models import Tag from django.db.models import F, Q from biostar.accounts.models import Profile, Message, User -from biostar.forum.models import Post, Award, Subscription, SharedLink +from biostar.forum.models import Post, Award, Subscription, SharedLink, Diff from biostar.forum import tasks, auth, util @@ -43,6 +43,19 @@ def send_award_message(sender, instance, created, **kwargs): return +@receiver(post_save, sender=Post) +def create_diffs(sender, instance, created, **kwargs): + """ + Create a diff for given post + """ + + # Create initial diff using post content + diff, created = Diff.objects.get_or_create(post=instance) + + # Update existing diff to current post content + Diff.objects.filter(pk=diff.pk).update(current=instance.content, editted=instance.lastedit_date) + + @receiver(post_save, sender=Profile) def ban_user(sender, instance, created, **kwargs): """ From e10860a65c3a4517dec975b1f1ca775066f8c398 Mon Sep 17 00:00:00 2001 From: Natay Aberra Date: Fri, 4 Jun 2021 10:19:35 -0400 Subject: [PATCH 02/12] update --- biostar/forum/forms.py | 3 +++ biostar/forum/models.py | 12 +++++------- biostar/forum/signals.py | 4 ++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/biostar/forum/forms.py b/biostar/forum/forms.py index d9186ad31..17c7b9322 100644 --- a/biostar/forum/forms.py +++ b/biostar/forum/forms.py @@ -147,7 +147,10 @@ def edit(self): raise forms.ValidationError("Only the author or a moderator can edit a post.") data = self.cleaned_data self.post.title = data.get('title') + # TODO: make trasaction safe + # Compute/save diff here self.post.content = data.get("content") + self.post.type = data.get('post_type') self.post.tag_val = data.get('tag_val') self.post.lastedit_user = self.user diff --git a/biostar/forum/models.py b/biostar/forum/models.py index 48e0a431e..6edddd162 100644 --- a/biostar/forum/models.py +++ b/biostar/forum/models.py @@ -635,22 +635,20 @@ def uid(self): class Diff(models.Model): # Initial content state - initial = models.TextField(default='') - - # Current content state - current = models.TextField(default='') + diff = models.TextField(default='') # Date the initial content was created created = models.DateTimeField(auto_now_add=True) - # Date the current content was created - edited = models.DateTimeField(auto_now=True) - # Post this diff belongs to post = models.ForeignKey(Post, on_delete=models.CASCADE) + # Person who created the diff + author = '' + def save(self, *args, **kwargs): + self.initial = self.initial or self.post.content self.current = self.current or self.initial diff --git a/biostar/forum/signals.py b/biostar/forum/signals.py index 899e481fd..77f7211f0 100644 --- a/biostar/forum/signals.py +++ b/biostar/forum/signals.py @@ -49,6 +49,10 @@ def create_diffs(sender, instance, created, **kwargs): Create a diff for given post """ + # TODO: testout + if created: + return + # Create initial diff using post content diff, created = Diff.objects.get_or_create(post=instance) From 39b0e5cbf55ca6c962bb2ad216f940dbe2095ee8 Mon Sep 17 00:00:00 2001 From: Natay Aberra Date: Tue, 8 Jun 2021 12:32:58 -0400 Subject: [PATCH 03/12] update --- biostar/forum/ajax.py | 42 ++++++-------- biostar/forum/auth.py | 58 +++++++++++++++---- biostar/forum/forms.py | 36 ++++++++---- biostar/forum/herald.py | 1 + biostar/forum/migrations/0021_diff.py | 26 +++++++++ biostar/forum/models.py | 11 +--- biostar/forum/signals.py | 17 ------ biostar/forum/static/forum.css | 10 +++- biostar/forum/static/forum.js | 39 +++++++++++-- .../forum/templates/forms/form_inplace.html | 3 +- biostar/forum/templates/post_view.html | 1 - .../forum/templates/widgets/post_actions.html | 7 +++ biostar/forum/urls.py | 1 + biostar/forum/views.py | 10 +--- biostar/utils/decorators.py | 4 +- 15 files changed, 180 insertions(+), 86 deletions(-) create mode 100644 biostar/forum/migrations/0021_diff.py diff --git a/biostar/forum/ajax.py b/biostar/forum/ajax.py index 7e943192a..5af6e9e40 100644 --- a/biostar/forum/ajax.py +++ b/biostar/forum/ajax.py @@ -3,6 +3,7 @@ import json from ratelimit.decorators import ratelimit from urllib import request as builtin_request +from difflib import Differ # import requests from urllib.parse import urlencode from datetime import datetime, timedelta @@ -237,21 +238,6 @@ def get_fields(request, post=None): return fields -def set_post(post, user, fields): - # Set the fields for this post. - if post.is_toplevel: - post.title = fields.get('title', post.title) - post.type = fields.get('post_type', post.type) - post.tag_val = fields.get('tag_val', post.tag_val) - - post.lastedit_user = user - post.lastedit_date = util.now() - post.content = fields.get('content', post.content) - post.save() - - return post - - @ajax_limited(key=RATELIMIT_KEY, rate=EDIT_RATE) @ajax_error_wrapper(method="POST", login_required=True) def ajax_edit(request, uid): @@ -279,7 +265,7 @@ def ajax_edit(request, uid): form = forms.PostShortForm(post=post, user=user, data=fields) if form.is_valid(): - post = set_post(post, user, form.cleaned_data) + form.edit() return ajax_success(msg='Edited post', redirect=post.get_absolute_url()) else: msg = [field.errors for field in form if field.errors] @@ -321,7 +307,7 @@ def ajax_comment_create(request): if form.is_valid(): # Create the comment. - post = Post.objects.create(type=Post.COMMENT, content=content, author=user, parent=parent) + post = form.save() return ajax_success(msg='Created post', redirect=post.get_absolute_url()) else: msg = [field.errors for field in form if field.errors] @@ -470,17 +456,25 @@ def email_disable(request, uid): return ajax_success(msg='Disabled messages') -def view_diff(request, uid): +@ajax_error_wrapper(method="POST", login_required=True) +def view_diff(request, pk): """ - View diffs made to a post. + View specific diffs made to a post. """ - # View changes made in a post - post = Post.objects.filter(uid=uid).first() - - diff = Diff.objects.filter(post=post) + # View changes made by this user. + user = request.user + diffobj = Diff.objects.filter(pk=pk).first() + + if not diffobj: + return ajax_error(msg='Post dot not have a diff') + + # Change new line chars to breakline tags. + diff = diffobj.diff + diff = diff.replace('\n', '
') - return + # Return newly created diff + return ajax_success(msg='Disabled messages', diff=diff) def similar_posts(request, uid): diff --git a/biostar/forum/auth.py b/biostar/forum/auth.py index c7d223869..d5e44beaa 100644 --- a/biostar/forum/auth.py +++ b/biostar/forum/auth.py @@ -3,7 +3,8 @@ import re import urllib.parse as urlparse from datetime import timedelta - +from difflib import Differ, SequenceMatcher, HtmlDiff, unified_diff +import bs4 from django.contrib import messages from django.contrib.auth import get_user_model from django.core.cache import cache @@ -21,7 +22,7 @@ from biostar.utils.helpers import get_ip from . import util, awards from .const import * -from .models import Post, Vote, Subscription, Badge, delete_post_cache, Log, SharedLink +from .models import Post, Vote, Subscription, Badge, delete_post_cache, Log, SharedLink, Diff User = get_user_model() @@ -67,7 +68,8 @@ def delete_cache(prefix, user): import datetime -ICONS = [ "monsterid", "robohash", "wavatar", "retro"] +ICONS = ["monsterid", "robohash", "wavatar", "retro"] + def gravatar_url(email, style='mp', size=80, force=None): global ICONS @@ -208,18 +210,19 @@ def create_post_from_json(**json_data): return -def create_post(author, title, content, request, root=None, parent=None, ptype=Post.QUESTION, tag_val="", nodups=True): +def create_post(author, title, content, request=None, root=None, parent=None, ptype=Post.QUESTION, tag_val="", + nodups=True): # Check if a post with this exact content already exists. post = Post.objects.filter(content=content, author=author).order_by('-creation_date').first() # How many seconds since the last post should we disallow duplicates. - time_frame = 60 - if nodups and post: - # Check to see if this post was made within given timeframe - delta_secs = (util.now() - post.creation_date).seconds - if delta_secs < time_frame: + frame = 60 + delta = (util.now() - post.creation_date).seconds if post else frame + + if nodups and delta < frame: + if request: messages.warning(request, "Post with this content was created recently.") - return post + return post post = Post.objects.create(title=title, content=content, root=root, parent=parent, type=ptype, tag_val=tag_val, author=author) @@ -228,6 +231,41 @@ def create_post(author, title, content, request, root=None, parent=None, ptype=P return post +def diff_ratio(text1, text2): + + # Do not match on spaces + s = SequenceMatcher(lambda char: re.match(r'\w+', char), text1, text2) + return round(s.ratio(), 5) + + +def compute_diff(text, post, user): + """ + Compute and return Diff object for diff between text and post.content + """ + + # Skip on post creation + if not post: + return + + ratio = diff_ratio(text1=text, text2=post.content) + + # Skip no changes detected + if ratio == 1: + return + + # Compute diff between text and post. + content = post.content.splitlines(keepends=True) + text = text.splitlines(keepends=True) + + diff = unified_diff(content, text) + diff = ''.join(diff) + + # Create diff object for this user. + dobj = Diff.objects.create(diff=diff, post=post, author=user) + + return dobj + + def merge_profiles(main, alias): """ Merge alias profile into main diff --git a/biostar/forum/forms.py b/biostar/forum/forms.py index 17c7b9322..35e598b0a 100644 --- a/biostar/forum/forms.py +++ b/biostar/forum/forms.py @@ -11,7 +11,7 @@ from snowpenguin.django.recaptcha2.widgets import ReCaptchaWidget from biostar.accounts.models import User from .models import Post, SharedLink -from biostar.forum import models, auth +from biostar.forum import models, auth, util from .const import * @@ -146,13 +146,16 @@ def edit(self): if self.user != self.post.author and not self.user.profile.is_moderator: raise forms.ValidationError("Only the author or a moderator can edit a post.") data = self.cleaned_data + self.post.title = data.get('title') - # TODO: make trasaction safe - # Compute/save diff here - self.post.content = data.get("content") - + content = data.get('content', self.post.content) + # Calculate diff and save to db + auth.compute_diff(text=content, post=self.post, user=self.user) + self.post.content = content + self.post.type = data.get('post_type') self.post.tag_val = data.get('tag_val') + self.post.lastedit_date = util.now() self.post.lastedit_user = self.user self.post.save() return self.post @@ -190,13 +193,12 @@ def clean_content(self): class PostShortForm(forms.Form): MIN_LEN, MAX_LEN = 10, 10000 - parent_uid = forms.CharField(widget=forms.HiddenInput(), min_length=2, max_length=32, required=False) - content = forms.CharField(widget=forms.Textarea, - min_length=MIN_LEN, max_length=MAX_LEN, strip=False) + content = forms.CharField(widget=forms.Textarea, min_length=MIN_LEN, max_length=MAX_LEN, strip=False) - def __init__(self, user=None, post=None, *args, **kwargs): + def __init__(self, post, user=None, request=None, ptype=Post.COMMENT, *args, **kwargs): self.user = user self.post = post + self.ptype = ptype super().__init__(*args, **kwargs) self.fields['content'].strip = False @@ -210,9 +212,23 @@ def clean(self): raise forms.ValidationError("You need to be logged in.") return cleaned_data + def edit(self): + # Set the fields for this post. + self.post.lastedit_user = self.user + self.post.lastedit_date = util.now() + content = self.cleaned_data.get('content', self.post.content) + auth.compute_diff(text=content, post=self.post, user=self.user) + self.post.content = content + self.post.save() + + def save(self): + content = self.cleaned_data.get('content', self.post.content) + post = auth.create_post(parent=self.post, author=self.user, content=content, ptype=self.ptype, + root=self.post.root, title=self.post.title) + return post -class MergeProfiles(forms.Form): +class MergeProfiles(forms.Form): main = forms.CharField(label='Main user email', max_length=100, required=True) alias = forms.CharField(label='Alias email to merge to main', max_length=100, required=True) diff --git a/biostar/forum/herald.py b/biostar/forum/herald.py index 4974b2b9c..475d4c6a2 100644 --- a/biostar/forum/herald.py +++ b/biostar/forum/herald.py @@ -177,6 +177,7 @@ def herald_list(request): # Add the Link attribute. link = form.cleaned_data['url'] text = form.cleaned_data['text'] + # Create the herald_list objects. herald = SharedLink.objects.create(author=user, text=text, url=link) messages.success(request, 'Submitted for review.') diff --git a/biostar/forum/migrations/0021_diff.py b/biostar/forum/migrations/0021_diff.py new file mode 100644 index 000000000..925ed9c85 --- /dev/null +++ b/biostar/forum/migrations/0021_diff.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2 on 2021-06-06 12:43 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('forum', '0020_sharedlink_status'), + ] + + operations = [ + migrations.CreateModel( + name='Diff', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('diff', models.TextField(default='')), + ('created', models.DateTimeField(auto_now_add=True)), + ('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('post', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='forum.post')), + ], + ), + ] diff --git a/biostar/forum/models.py b/biostar/forum/models.py index 6edddd162..2165a7912 100644 --- a/biostar/forum/models.py +++ b/biostar/forum/models.py @@ -644,22 +644,15 @@ class Diff(models.Model): post = models.ForeignKey(Post, on_delete=models.CASCADE) # Person who created the diff - author = '' + author = models.ForeignKey(User, on_delete=models.CASCADE) def save(self, *args, **kwargs): - - self.initial = self.initial or self.post.content - self.current = self.current or self.initial - - self.created = self.created or self.post.lastedit_date - self.edited = self.edited or self.created + self.created = self.created or util.now() super(Diff, self).save(*args, **kwargs) - - class Log(models.Model): """ Represents moderation actions diff --git a/biostar/forum/signals.py b/biostar/forum/signals.py index 77f7211f0..6d987699d 100644 --- a/biostar/forum/signals.py +++ b/biostar/forum/signals.py @@ -43,23 +43,6 @@ def send_award_message(sender, instance, created, **kwargs): return -@receiver(post_save, sender=Post) -def create_diffs(sender, instance, created, **kwargs): - """ - Create a diff for given post - """ - - # TODO: testout - if created: - return - - # Create initial diff using post content - diff, created = Diff.objects.get_or_create(post=instance) - - # Update existing diff to current post content - Diff.objects.filter(pk=diff.pk).update(current=instance.content, editted=instance.lastedit_date) - - @receiver(post_save, sender=Profile) def ban_user(sender, instance, created, **kwargs): """ diff --git a/biostar/forum/static/forum.css b/biostar/forum/static/forum.css index 247dc5586..5c3b3d474 100644 --- a/biostar/forum/static/forum.css +++ b/biostar/forum/static/forum.css @@ -116,6 +116,14 @@ blockquote.twitter-tweet { background: white; } +.diff-plus{ + background: #aee4ae; +} + +.diff-minus{ + background: #ffc5c5; +} + #topicbar { border: none; border-radius: 0; @@ -549,7 +557,7 @@ i.icon { padding-bottom: 10px; } -.draggable { +.draggable, .view-diffs { cursor: pointer; font-size: 85%; } diff --git a/biostar/forum/static/forum.js b/biostar/forum/static/forum.js index 1b738c0c1..fbd36c355 100644 --- a/biostar/forum/static/forum.js +++ b/biostar/forum/static/forum.js @@ -12,6 +12,30 @@ function captcha() { } } +function view_diffs(pk, elem) { + // View post diff given associated Diff.pk + + $.ajax("/view/diffs/" + pk + '/', { + type: 'POST', + dataType: 'json', + ContentType: 'application/json', + + success: function (data) { + if (data.status === 'error') { + popup_message(elem, data.msg, data.status); + return + } + // Success + elem.html(data.diff); + elem.addClass('ui segment'); + }, + error: function (xhr, status, text) { + error_message(elem, xhr, status, text) + } + }); + + +} function apply_vote(vote_elem) { @@ -100,7 +124,7 @@ function moderate(uid, container, url) { } -function disable_emails(user_id, elem){ +function disable_emails(user_id, elem) { var url = '/email/disable/{0}/'.format(user_id); $.ajax(url, { @@ -195,7 +219,6 @@ function activate_prism(elem) { } - function herald_update(hpk, status, elem) { $.ajax('/herald/update/' + hpk + '/', @@ -348,7 +371,10 @@ $(document).ready(function () { on: 'hover', content: 'Drag and Drop' }); - + $(".view-diffs").popup({ + on: 'hover', + content: 'View Changes' + }); $("[data-value='bookmark']").popup({ on: 'hover', content: 'Bookmark ' @@ -359,7 +385,7 @@ $(document).ready(function () { content: 'Accept' }); - $("[data-value='decline']").popup({ + $("[data-value='decline']").popup({ on: 'hover', content: 'Decline' }); @@ -419,7 +445,12 @@ $(document).ready(function () { $(this).on('click', ".herald-sub", function (event) { herald_subscribe($(this)) }); + $(this).on('click', ".view-diffs", function (event) { + var pk = $(this).data('value'); + var elem = $('#diff-cont'); + view_diffs(pk, elem); + }); $('pre').addClass('language-bash'); $('code').addClass('language-bash'); Prism.highlightAll(); diff --git a/biostar/forum/templates/forms/form_inplace.html b/biostar/forum/templates/forms/form_inplace.html index 0d2c00c7e..f8d983a05 100644 --- a/biostar/forum/templates/forms/form_inplace.html +++ b/biostar/forum/templates/forms/form_inplace.html @@ -56,9 +56,10 @@ - Cancel + Cancel + {% if post and not new %} Delete diff --git a/biostar/forum/templates/post_view.html b/biostar/forum/templates/post_view.html index eb5f5da0c..98cf8effa 100644 --- a/biostar/forum/templates/post_view.html +++ b/biostar/forum/templates/post_view.html @@ -52,7 +52,6 @@ {% endif %} -