From 27dd4dcbb2ae32f1f15c7b1cf4cb324b273bd5dd Mon Sep 17 00:00:00 2001
From: Jochen Topf <jochen@topf.org>
Date: Tue, 7 Feb 2023 14:21:04 +0100
Subject: [PATCH] Flush COPY between nodes/ways and ways/relations respectively

In most cases database tables are filled based on OSM object types, so
after we have read all nodes nothing will be added to node-type tables
any more etc. So it makes sense to explicitly flush the COPY making sure
this data is written to the database. If more data is later added, maybe
because of second-stage processing, it doesn't matter, the COPY will be
reopened.
---
 src/db-copy-mgr.hpp | 14 +++++++++-----
 src/flex-table.hpp  |  2 ++
 src/osmdata.cpp     | 12 ++++++++++--
 src/output-flex.cpp | 14 ++++++++++++++
 src/output-flex.hpp |  3 +++
 src/output.hpp      |  3 +++
 6 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/src/db-copy-mgr.hpp b/src/db-copy-mgr.hpp
index b207c7d6b..bf2f129a3 100644
--- a/src/db-copy-mgr.hpp
+++ b/src/db-copy-mgr.hpp
@@ -271,6 +271,14 @@ class db_copy_mgr_t
         m_current->add_deletable(std::forward<ARGS>(args)...);
     }
 
+    void flush()
+    {
+        // finish any ongoing copy operations
+        if (m_current) {
+            m_processor->add_buffer(std::move(m_current));
+        }
+    }
+
     /**
      * Synchronize with worker.
      *
@@ -278,11 +286,7 @@ class db_copy_mgr_t
      */
     void sync()
     {
-        // finish any ongoing copy operations
-        if (m_current) {
-            m_processor->add_buffer(std::move(m_current));
-        }
-
+        flush();
         m_processor->sync_and_wait();
     }
 
diff --git a/src/flex-table.hpp b/src/flex-table.hpp
index 3d0c64722..27472c377 100644
--- a/src/flex-table.hpp
+++ b/src/flex-table.hpp
@@ -247,6 +247,8 @@ class table_connection_t
 
     pg_result_t get_geom_by_id(osmium::item_type type, osmid_t id) const;
 
+    void flush() { m_copy_mgr.flush(); }
+
     void sync() { m_copy_mgr.sync(); }
 
     void new_line() { m_copy_mgr.new_line(m_target); }
diff --git a/src/osmdata.cpp b/src/osmdata.cpp
index 7c29ca3a6..e9b036aca 100644
--- a/src/osmdata.cpp
+++ b/src/osmdata.cpp
@@ -67,7 +67,11 @@ void osmdata_t::node(osmium::Node const &node)
     }
 }
 
-void osmdata_t::after_nodes() { m_mid->after_nodes(); }
+void osmdata_t::after_nodes()
+{
+    m_mid->after_nodes();
+    m_output->after_nodes();
+}
 
 void osmdata_t::way(osmium::Way &way)
 {
@@ -84,7 +88,11 @@ void osmdata_t::way(osmium::Way &way)
     }
 }
 
-void osmdata_t::after_ways() { m_mid->after_ways(); }
+void osmdata_t::after_ways()
+{
+    m_mid->after_ways();
+    m_output->after_ways();
+}
 
 void osmdata_t::relation(osmium::Relation const &rel)
 {
diff --git a/src/output-flex.cpp b/src/output-flex.cpp
index c9a13e3b8..0add7baa4 100644
--- a/src/output-flex.cpp
+++ b/src/output-flex.cpp
@@ -1263,6 +1263,20 @@ void output_flex_t::sync()
     }
 }
 
+void output_flex_t::after_nodes()
+{
+    for (auto &table : m_table_connections) {
+        table.flush();
+    }
+}
+
+void output_flex_t::after_ways()
+{
+    for (auto &table : m_table_connections) {
+        table.flush();
+    }
+}
+
 void output_flex_t::stop()
 {
     for (auto &table : m_table_connections) {
diff --git a/src/output-flex.hpp b/src/output-flex.hpp
index 385f1fdd3..5ab5d7d6f 100644
--- a/src/output-flex.hpp
+++ b/src/output-flex.hpp
@@ -124,6 +124,9 @@ class output_flex_t : public output_t
     void stop() override;
     void sync() override;
 
+    void after_nodes() override;
+    void after_ways() override;
+
     void wait() override;
 
     idset_t const &get_marked_way_ids() override;
diff --git a/src/output.hpp b/src/output.hpp
index 5700c36ec..35a787f61 100644
--- a/src/output.hpp
+++ b/src/output.hpp
@@ -62,6 +62,9 @@ class output_t
     virtual void stop() = 0;
     virtual void sync() = 0;
 
+    virtual void after_nodes() {}
+    virtual void after_ways() {}
+
     virtual void wait() {}
 
     virtual osmium::index::IdSetSmall<osmid_t> const &get_marked_way_ids()