Merge pull request #3648 from GeotrekCE/feat_merge_path_command

Add command to identify and merge some paths #3607

docs/changelog.rst

@@ -28,6 +28,7 @@ CHANGELOG
 
 - Update ``check_ign_keys`` script to match new IGN urls
 - Update ``base.py`` configuration for layers
+- Add ``merge_segmented_paths`` command to find and merge paths (#3607)
 
 **Bug fixes**
 

docs/install/import.rst

@@ -851,6 +851,52 @@ You have to run ``sudo geotrek remove_duplicate_paths``
 During the process of the command, every topology on a duplicate path will be set on the original path, and the duplicate path will be deleted.
 
 
+Merge segmented paths
+----------------------
+
+A path network is most optimized when there is only one path between intersections.
+If the path database includes many fragmented paths, they could be merged to improve performances.
+
+You can run ``sudo geotrek merge_segmented_paths``. 
+
+.. danger::
+    This command can take several hours to run. During the process, every topology on a path will be set on the path it is merged with, but it would still be more efficient (and safer) to run it before creating topologies. 
+
+Before :
+::
+
+       p1      p2      p3      p5     p6     p7      p8     p9     p14
+    +-------+------+-------+------+-------+------+-------+------+------+
+                   |                             |
+                   |  p4                         |  p13
+                   |                             |
+                   +                             +-------
+                   |                             |       |
+                   |  p10                        |   p16 |
+             p11   |                             |       |
+            +------+------+ p15                  --------
+                   |
+                   |  p12
+                   |
+
+After :
+::
+
+           p1                     p6                       p14
+    +--------------+-----------------------------+---------------------+
+                   |                             |
+                   |                             |  p13
+                   |                             |
+                   |  p10                        +-------
+                   |                             |       |
+                   |                             |   p16 |
+             p11   |                             |       |
+            +------+------+ p15                  --------
+                   |
+                   |  p12
+                   |
+
+
 Unset structure on categories
 -----------------------------
 

geotrek/core/management/commands/merge_segmented_paths.py

@@ -0,0 +1,136 @@
+from datetime import datetime
+from time import sleep
+
+from django.core.management.base import BaseCommand
+from django.db import connection, transaction
+
+from geotrek.core.models import Path
+
+
+class Command(BaseCommand):
+    help = 'Find and merge Paths that are splitted in several segments\n'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--sleeptime', '-d', action='store', dest='sleeptime', default=0.25,
+                            help="Time to wait between merges (SQL triggers take time)")
+
+    def extract_neighbourgs_graph(self, number_of_neighbourgs, extremities=[]):
+        # Get all neighbours for each path
+        neighbours = dict()
+        with connection.cursor() as cursor:
+            cursor.execute('''select id1, array_agg(id2) from
+                        (select p1.id as id1, p2.id as id2
+                        from core_path p1, core_path p2
+                        where st_touches(p1.geom, p2.geom) and (p1.id != p2.id)
+                        group by p1.id, p2.id
+                        order by p1.id) a
+                        group by id1;''')
+
+            for path_id, path_neighbours_ids in cursor.fetchall():
+                if path_id not in extremities and len(path_neighbours_ids) == number_of_neighbourgs:
+                    neighbours[path_id] = path_neighbours_ids
+        return neighbours
+
+    def try_merge(self, a, b):
+        if {a, b} in self.discarded:
+            self.stdout.write(f"├ Already discarded {a} and {b}")
+            return False
+        success = 2
+        try:
+            patha = Path.include_invisible.get(pk=a)
+            pathb = Path.include_invisible.get(pk=b)
+            with transaction.atomic():
+                success = patha.merge_path(pathb)
+        except Exception:
+            self.stdout.write(f"├ Cannot merge {a} and {b}")
+            self.discarded.append({a, b})
+            return False
+        if success == 2 or success == 0:
+            self.stdout.write(f"├ Cannot merge {a} and {b}")
+            self.discarded.append({a, b})
+            return False
+        else:
+            self.stdout.write(f"├ Merged {b} into {a}")
+            sleep(self.sleeptime)
+            return True
+
+    def merge_paths_with_one_neighbour(self):
+        self.stdout.write("┌ STEP 1")
+        neighbours_graph = self.extract_neighbourgs_graph(1)
+        successes = 0
+        fails = 0
+        while len(neighbours_graph) > fails:
+            fails = 0
+            for path, neighbours in neighbours_graph.items():
+                success = self.try_merge(path, neighbours[0])
+                if success:
+                    successes += 1
+                else:
+                    fails += 1
+            neighbours_graph = self.extract_neighbourgs_graph(1)
+        return successes
+
+    def merge_paths_with_two_neighbours(self):
+        self.stdout.write("┌ STEP 2")
+        successes = 0
+        neighbours_graph = self.extract_neighbourgs_graph(2)
+        mergeables = list(neighbours_graph.keys())
+        fails = 0
+        while len(mergeables) > fails:
+            fails = 0
+            for (a, neighbours) in neighbours_graph.items():
+                b = neighbours[0]
+                success = self.try_merge(a, b)
+                if success:
+                    successes += 1
+                else:
+                    fails += 1
+            neighbours_graph = self.extract_neighbourgs_graph(2)
+            mergeables = neighbours_graph.keys()
+        return successes
+
+    def merge_paths_with_three_neighbours(self):
+        self.stdout.write("┌ STEP 3")
+        successes = 0
+        neighbours_graph = self.extract_neighbourgs_graph(3)
+        mergeables = list(neighbours_graph.keys())
+        extremities = []
+        while len(mergeables) > len(extremities):
+            for (a, neighbours) in neighbours_graph.items():
+                failed_neighbours = 0
+                for n in neighbours:
+                    success = self.try_merge(a, n)
+                    if success:
+                        successes += 1
+                    else:
+                        failed_neighbours += 1
+                    if failed_neighbours == 3:
+                        extremities.append(a)
+            neighbours_graph = self.extract_neighbourgs_graph(3, extremities=extremities)
+            mergeables = list(neighbours_graph.keys())
+        return successes
+
+    def handle(self, *args, **options):
+        self.sleeptime = options.get('sleeptime')
+        total_successes = 0
+        self.discarded = []
+        paths_before = Path.include_invisible.count()
+
+        self.stdout.write("\n")
+        self.stdout.write(str(datetime.now()))
+
+        first_step_successes = self.merge_paths_with_one_neighbour()
+        self.stdout.write(f"└ {first_step_successes} merges")
+        total_successes += first_step_successes
+
+        second_step_successes = self.merge_paths_with_two_neighbours()
+        self.stdout.write(f"└ {second_step_successes} merges")
+        total_successes += second_step_successes
+
+        third_step_successes = self.merge_paths_with_three_neighbours()
+        self.stdout.write(f"└ {third_step_successes} merges")
+        total_successes += third_step_successes
+
+        paths_after = Path.include_invisible.count()
+        self.stdout.write(f"\n--- RAN {total_successes} MERGES - FROM {paths_before} TO {paths_after} PATHS ---\n")
+        self.stdout.write(str(datetime.now()))

geotrek/core/tests/test_commands.py

@@ -626,3 +626,110 @@ def test_split_reorder_fail(self):
         output = StringIO()
         call_command('reorder_topologies', stdout=output)
         self.assertIn(f'Topologies with errors :\nTREK id: {topo.pk}\n', output.getvalue())
+
+
+class MergePathsTest(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        geom_1 = LineString((0, 0), (1, 1))
+        cls.p1 = Path.objects.create(geom=geom_1)
+        geom_2 = LineString((1, 1), (2, 2))
+        cls.p2 = Path.objects.create(geom=geom_2)
+        geom_3 = LineString((2, 2), (3, 3))
+        cls.p3 = Path.objects.create(geom=geom_3)
+        geom_4 = LineString((2, 2), (6, 1))
+        cls.p4 = Path.objects.create(geom=geom_4)
+        geom_5 = LineString((3, 3), (4, 4))
+        cls.p5 = Path.objects.create(geom=geom_5)
+        geom_6 = LineString((4, 4), (5, 5))
+        cls.p6 = Path.objects.create(geom=geom_6)
+        geom_7 = LineString((5, 5), (6, 6))
+        cls.p7 = Path.objects.create(geom=geom_7)
+        geom_8 = LineString((6, 6), (7, 7))
+        cls.p8 = Path.objects.create(geom=geom_8)
+        geom_9 = LineString((7, 7), (8, 8))
+        cls.p9 = Path.objects.create(geom=geom_9)
+        geom_10 = LineString((6, 1), (4, 1))
+        cls.p10 = Path.objects.create(geom=geom_10)
+        geom_11 = LineString((4, 1), (4, 0))
+        cls.p11 = Path.objects.create(geom=geom_11)
+        geom_12 = LineString((4, 1), (6, 1))
+        cls.p12 = Path.objects.create(geom=geom_12)
+        geom_13 = LineString((6, 6), (7, 5))
+        cls.p13 = Path.objects.create(geom=geom_13)
+        geom_14 = LineString((8, 8), (9, 9))
+        cls.p14 = Path.objects.create(geom=geom_14)
+        geom_15 = LineString((5, 3), (4, 1))
+        cls.p15 = Path.objects.create(geom=geom_15)
+        geom_16 = LineString((7, 5), (8, 5), (8, 6), (7, 6), (7, 5))
+        cls.p16 = Path.objects.create(geom=geom_16)
+
+    @override_settings(PATH_SNAPPING_DISTANCE=0, PATH_MERGE_SNAPPING_DISTANCE=0)
+    def test_find_and_merge_paths(self):
+        # Before call
+        #    p1      p2      p3      p5     p6     p7      p8     p9     p14
+        # +-------+------+-------+------+-------+------+-------+------+------+
+        #                |                             |
+        #                |  p4                         |  p13
+        #                |                             |
+        #                +                             +-------
+        #                |                             |       |
+        #                |  p10                        |   p16 |
+        #          p11   |                             |       |
+        #         +------+------+ p15                  --------
+        #                |
+        #                |  p12
+        #                |
+        #                +
+        self.assertEqual(Path.objects.count(), 16)
+        output = StringIO()
+        call_command('merge_segmented_paths', stdout=output)
+        # After call
+        #        p1                     p6                       p14
+        # +--------------+-----------------------------+---------------------+
+        #                |                             |
+        #                |  p4                         |  p13
+        #                |                             |
+        #                +                             +-------
+        #                |                             |       |
+        #                |  p10                        |   p16 |
+        #          p11   |                             |       |
+        #         +------+------+ p15                  --------
+        #                |
+        #                |  p12
+        #                |
+        #
+        output_str = (f"┌ STEP 1\n"
+                      f"├ Merged {self.p2.pk} into {self.p1.pk}\n"
+                      f"├ Merged {self.p9.pk} into {self.p14.pk}\n"
+                      f"├ Cannot merge {self.p16.pk} and {self.p13.pk}\n"
+                      f"├ Merged {self.p8.pk} into {self.p14.pk}\n"
+                      f"├ Already discarded {self.p16.pk} and {self.p13.pk}\n"
+                      f"└ 3 merges\n"
+                      f"┌ STEP 2\n"
+                      f"├ Cannot merge {self.p1.pk} and {self.p3.pk}\n"
+                      f"├ Merged {self.p3.pk} into {self.p5.pk}\n"
+                      f"├ Merged {self.p5.pk} into {self.p6.pk}\n"
+                      f"├ Cannot merge {self.p14.pk} and {self.p7.pk}\n"
+                      f"└ 2 merges\n"
+                      f"┌ STEP 3\n"
+                      f"├ Cannot merge {self.p6.pk} and {self.p1.pk}\n"
+                      f"├ Cannot merge {self.p6.pk} and {self.p4.pk}\n"
+                      f"├ Merged {self.p7.pk} into {self.p6.pk}\n"
+                      f"├ Cannot merge {self.p7.pk} and {self.p6.pk}\n"
+                      f"├ Cannot merge {self.p7.pk} and {self.p13.pk}\n"
+                      f"├ Already discarded {self.p7.pk} and {self.p14.pk}\n"
+                      f"├ Cannot merge {self.p10.pk} and {self.p4.pk}\n"
+                      f"├ Cannot merge {self.p10.pk} and {self.p11.pk}\n"
+                      f"├ Cannot merge {self.p10.pk} and {self.p15.pk}\n"
+                      f"├ Cannot merge {self.p12.pk} and {self.p4.pk}\n"
+                      f"├ Cannot merge {self.p12.pk} and {self.p11.pk}\n"
+                      f"├ Cannot merge {self.p12.pk} and {self.p15.pk}\n"
+                      f"├ Already discarded {self.p13.pk} and {self.p7.pk}\n"
+                      f"├ Cannot merge {self.p13.pk} and {self.p14.pk}\n"
+                      f"├ Already discarded {self.p13.pk} and {self.p16.pk}\n"
+                      f"└ 1 merges\n"
+                      f"\n"
+                      f"--- RAN 6 MERGES - FROM 16 TO 10 PATHS ---\n")
+        self.assertEqual(Path.objects.count(), 10)
+        self.assertIn(output_str, output.getvalue())