basic tag/group handling with filters+notifications

changedetectionio/__init__.py

@@ -317,25 +317,21 @@ def rss():
             return "Access denied, bad token", 403
 
         from . import diff
-        limit_tag = request.args.get('tag')
+        limit_tag = request.args.get('tag', '').lower().strip()
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
 
         # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
         for uuid, watch in datastore.data['watching'].items():
-
-            if limit_tag != None:
-                # Support for comma separated list of tags.
-                for tag_in_watch in watch['tag'].split(','):
-                    tag_in_watch = tag_in_watch.strip()
-                    if tag_in_watch == limit_tag:
-                        watch['uuid'] = uuid
-                        sorted_watches.append(watch)
-
-            else:
-                watch['uuid'] = uuid
-                sorted_watches.append(watch)
+            if limit_tag and not limit_tag in watch['tags']:
+                    continue
+            watch['uuid'] = uuid
+            sorted_watches.append(watch)
 
         sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
 
@@ -392,9 +388,17 @@ def rss():
     @app.route("/", methods=['GET'])
     @login_optionally_required
     def index():
+        global datastore
         from changedetectionio import forms
 
-        limit_tag = request.args.get('tag')
+        limit_tag = request.args.get('tag', '').lower().strip()
+
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
+
+
         # Redirect for the old rss path which used the /?rss=true
         if request.args.get('rss'):
             return redirect(url_for('rss', tag=limit_tag))
@@ -414,30 +418,15 @@ def index():
         sorted_watches = []
         search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
-
-            if limit_tag:
-                # Support for comma separated list of tags.
-                if not watch.get('tag'):
+            if limit_tag and not limit_tag in watch['tags']:
                     continue
-                for tag_in_watch in watch.get('tag', '').split(','):
-                    tag_in_watch = tag_in_watch.strip()
-                    if tag_in_watch == limit_tag:
-                        watch['uuid'] = uuid
-                        if search_q:
-                            if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                                sorted_watches.append(watch)
-                        else:
-                            sorted_watches.append(watch)
 
-            else:
-                #watch['uuid'] = uuid
-                if search_q:
-                    if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
-                        sorted_watches.append(watch)
-                else:
+            if search_q:
+                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
                     sorted_watches.append(watch)
+            else:
+                sorted_watches.append(watch)
 
-        existing_tags = datastore.get_all_tags()
         form = forms.quickWatchForm(request.form)
         page = request.args.get(get_page_parameter(), type=int, default=1)
         total_count = len(sorted_watches)
@@ -452,6 +441,7 @@ def index():
                                  # Don't link to hosting when we're on the hosting environment
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
+                                 datastore=datastore,
                                  form=form,
                                  guid=datastore.data['app_guid'],
                                  has_proxies=datastore.proxy_list,
@@ -463,7 +453,7 @@ def index():
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=existing_tags,
+                                 tags=datastore.data['settings']['application'].get('tags'),
                                  watches=sorted_watches
                                  )
 
@@ -606,9 +596,13 @@ def edit_page(uuid):
 
         # proxy_override set to the json/text list of the items
         form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
+                               data=default
                                )
 
+        # For the form widget tag uuid lookup
+        form.tags.datastore = datastore # in _value
+
+
         form.fetch_backend.choices.append(("system", 'System settings default'))
 
         # form.browser_steps[0] can be assumed that we 'goto url' first
@@ -659,6 +653,16 @@ def edit_page(uuid):
                 extra_update_obj['filter_text_replaced'] = True
                 extra_update_obj['filter_text_removed'] = True
 
+            # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
+            tag_uuids = []
+            if form.data.get('tags'):
+                # Sometimes in testing this can be list, dont know why
+                if type(form.data.get('tags')) == list:
+                    extra_update_obj['tags'] = form.data.get('tags')
+                else:
+                    for t in form.data.get('tags').split(','):
+                        tag_uuids.append(datastore.add_tag(name=t))
+                    extra_update_obj['tags'] = tag_uuids
 
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
@@ -713,7 +717,7 @@ def edit_page(uuid):
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
-                                     has_extra_headers_file=watch.has_extra_headers_file or datastore.has_extra_headers_file,
+                                     has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
@@ -1110,8 +1114,8 @@ def get_backup():
                 os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
             ) as f:
                 for uuid in datastore.data["watching"]:
-                    url = datastore.data["watching"][uuid]["url"]
-                    tag = datastore.data["watching"][uuid]["tag"]
+                    url = datastore.data["watching"][uuid].get('url')
+                    tag = datastore.data["watching"][uuid].get('tags', {})
                     f.write("{} {}\r\n".format(url, tag))
 
             # Add it to the Zip
@@ -1199,7 +1203,7 @@ def form_quick_watch_add():
 
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         processor = request.form.get('processor', 'text_json_diff')
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused, 'processor': processor})
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
 
         if new_uuid:
             if add_paused:
@@ -1267,9 +1271,11 @@ def form_watch_checknow():
         elif tag != None:
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
-                if (tag != None and tag in watch['tag']):
+                if (tag != None and tag in watch.get('tags', {})):
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
+                        update_q.put(
+                            queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
+                        )
                         i += 1
 
         else:
@@ -1357,6 +1363,17 @@ def form_watch_list_checkbox_operations():
                     datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
             flash("{} watches set to use default notification settings".format(len(uuids)))
 
+        elif (op == 'assign-tag'):
+            op_extradata = request.form.get('op_extradata')
+            tag_uuid = datastore.add_tag(name=op_extradata)
+            if op_extradata and tag_uuid:
+                for uuid in uuids:
+                    uuid = uuid.strip()
+                    if datastore.data['watching'].get(uuid):
+                        datastore.data['watching'][uuid]['tags'].append(tag_uuid)
+
+            flash("{} watches assigned tag".format(len(uuids)))
+
         return redirect(url_for('index'))
 
     @app.route("/api/share-url", methods=['GET'])
@@ -1366,7 +1383,6 @@ def form_share_put_watch():
            the share-link can be imported/added"""
         import requests
         import json
-        tag = request.args.get('tag')
         uuid = request.args.get('uuid')
 
         # more for testing
@@ -1419,6 +1435,8 @@ def form_share_put_watch():
     import changedetectionio.blueprint.price_data_follower as price_data_follower
     app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
 
+    import changedetectionio.blueprint.tags as tags
+    app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()

changedetectionio/api/api_v1.py

@@ -218,6 +218,11 @@ def post(self):
                 return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
         extras = copy.deepcopy(json_data)
+
+        # Because we renamed 'tag' to 'tags' but dont want to change the API (can do this in v2 of the API)
+        if extras.get('tag'):
+            extras['tags'] = extras.get('tag')
+
         del extras['url']
 
         new_uuid = self.datastore.add_watch(url=url, extras=extras)
@@ -259,13 +264,16 @@ def get(self):
         """
         list = {}
 
-        tag_limit = request.args.get('tag', None)
-        for k, watch in self.datastore.data['watching'].items():
-            if tag_limit:
-                if not tag_limit.lower() in watch.all_tags:
-                    continue
+        tag_limit = request.args.get('tag', '').lower()
+
+
+        for uuid, watch in self.datastore.data['watching'].items():
+            # Watch tags by name (replace the other calls?)
+            tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
+            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
+                continue
 
-            list[k] = {'url': watch['url'],
+            list[uuid] = {'url': watch['url'],
                        'title': watch['title'],
                        'last_checked': watch['last_checked'],
                        'last_changed': watch.last_changed,

changedetectionio/blueprint/tags/README.md

@@ -0,0 +1,9 @@
+# Groups tags
+
+## How it works
+
+Watch has a list() of tag UUID's, which relate to a config under application.settings.tags
+
+The 'tag' is actually a watch, because they basically will eventually share 90% of the same config.
+
+So a tag is like an abstract of a watch

changedetectionio/blueprint/tags/__init__.py

@@ -0,0 +1,131 @@
+from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio import login_optionally_required
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    tags_blueprint = Blueprint('tags', __name__, template_folder="templates")
+
+    @tags_blueprint.route("/list", methods=['GET'])
+    @login_optionally_required
+    def tags_overview_page():
+        from .form import SingleTag
+        add_form = SingleTag(request.form)
+        output = render_template("groups-overview.html",
+                                 form=add_form,
+                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
+                                 )
+
+        return output
+
+    @tags_blueprint.route("/add", methods=['POST'])
+    @login_optionally_required
+    def form_tag_add():
+        from .form import SingleTag
+        add_form = SingleTag(request.form)
+
+        if not add_form.validate():
+            for widget, l in add_form.errors.items():
+                flash(','.join(l), 'error')
+            return redirect(url_for('tags.tags_overview_page'))
+
+        title = request.form.get('name').strip()
+
+        if datastore.tag_exists_by_name(title):
+            flash(f'The tag "{title}" already exists', "error")
+            return redirect(url_for('tags.tags_overview_page'))
+
+        datastore.add_tag(title)
+        flash("Tag added")
+
+
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def mute(uuid):
+        if datastore.data['settings']['application']['tags'].get(uuid):
+            datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def delete(uuid):
+        removed = 0
+        # Delete the tag, and any tag reference
+        if datastore.data['settings']['application']['tags'].get(uuid):
+            del datastore.data['settings']['application']['tags'][uuid]
+
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                removed += 1
+                watch['tags'].remove(uuid)
+
+        flash(f"Tag deleted and removed from {removed} watches")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def unlink(uuid):
+        unlinked = 0
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                unlinked += 1
+                watch['tags'].remove(uuid)
+
+        flash(f"Tag unlinked removed from {unlinked} watches")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def form_tag_edit(uuid):
+        from changedetectionio import forms
+
+        if uuid == 'first':
+            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
+
+        default = datastore.data['settings']['application']['tags'].get(uuid)
+
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               )
+        form.datastore=datastore # needed?
+
+        output = render_template("edit-tag.html",
+                                 data=default,
+                                 form=form,
+                                 settings_application=datastore.data['settings']['application'],
+                                 )
+
+        return output
+
+
+    @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
+    @login_optionally_required
+    def form_tag_edit_submit(uuid):
+        from changedetectionio import forms
+        if uuid == 'first':
+            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
+
+        default = datastore.data['settings']['application']['tags'].get(uuid)
+
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               )
+        # @todo subclass form so validation works
+        #if not form.validate():
+#            for widget, l in form.errors.items():
+#                flash(','.join(l), 'error')
+#           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
+
+        datastore.data['settings']['application']['tags'][uuid].update(form.data)
+        datastore.needs_write_urgent = True
+        flash("Updated")
+
+        return redirect(url_for('tags.tags_overview_page'))
+
+
+    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
+    def form_tag_delete(uuid):
+        return redirect(url_for('tags.tags_overview_page'))
+    return tags_blueprint