Merge pull request osm2pgsql-dev#6 from MapQuest/mla_threadpend

Added expiry tree merging
alex85k · Sep 16, 2014 · 73e98aa · 73e98aa
2 parents 79680bd + aa7fc5f
commit 73e98aa
Show file tree

Hide file tree

Showing 8 changed files with 526 additions and 45 deletions.
diff --git a/.gitignore b/.gitignore
@@ -51,6 +51,7 @@ tests/test-output-multi-line
 tests/test-output-multi-point
 tests/test-output-multi-point-multi-table
 tests/test-output-multi-polygon
+tests/test-expire-tiles
 tests/*.log
 tests/*.trs
 

diff --git a/Makefile.am b/Makefile.am
@@ -91,7 +91,8 @@ check_PROGRAMS = \
 	tests/test-output-multi-point-multi-table \
 	tests/test-output-multi-polygon \
 	tests/test-pgsql-escape \
-	tests/test-parse-options
+	tests/test-parse-options \
+	tests/test-expire-tiles
 
 tests_test_parse_xml2_SOURCES = tests/test-parse-xml2.cpp 
 tests_test_parse_xml2_LDADD = libosm2pgsql.la
@@ -111,6 +112,8 @@ tests_test_pgsql_escape_SOURCES = tests/test-pgsql-escape.cpp
 tests_test_pgsql_escape_LDADD = libosm2pgsql.la
 tests_test_parse_options_SOURCES = tests/test-parse-options.cpp
 tests_test_parse_options_LDADD = libosm2pgsql.la
+tests_test_expire_tiles_SOURCES = tests/test-expire-tiles.cpp
+tests_test_expire_tiles_LDADD = libosm2pgsql.la
 
 TESTS = $(check_PROGRAMS) tests/regression-test.sh
 TEST_EXTENSIONS = .sh
@@ -164,6 +167,7 @@ tests_test_output_multi_point_multi_table_LDADD += $(GLOBAL_LDFLAGS)
 tests_test_output_multi_polygon_LDADD += $(GLOBAL_LDFLAGS)
 tests_test_pgsql_escape_LDADD += $(GLOBAL_LDFLAGS)
 tests_test_parse_options_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_expire_tiles_LDADD += $(GLOBAL_LDFLAGS)
 nodecachefilereader_LDADD += $(GLOBAL_LDFLAGS)
 
 osm2pgsql_DATA = default.style 900913.sql

diff --git a/expire-tiles.cpp b/expire-tiles.cpp
@@ -125,67 +125,142 @@ void output_dirty_tile(FILE * outfile, int x, int y, int zoom, int min_zoom, int
 	}
 }
 
-void _output_and_destroy_tree(FILE * outfile, struct expire_tiles::tile * tree, int x, int y, int this_zoom, int min_zoom, int &outcount) {
+struct tile_output_file : public expire_tiles::tile_output {
+  tile_output_file(const std::string &expire_tiles_filename)
+    : outcount(0)
+    , outfile(fopen(expire_tiles_filename.c_str(), "a")) {
+    if (outfile == NULL) {
+      fprintf(stderr, "Failed to open expired tiles file (%s).  Tile expiry list will not be written!\n", strerror(errno));
+    }
+  }
+
+  virtual ~tile_output_file() {
+    if (outfile) {
+      fclose(outfile);
+    }
+  }
+
+  virtual void output_dirty_tile(int x, int y, int zoom, int min_zoom) {
+    ::output_dirty_tile(outfile, x, y, zoom, min_zoom, outcount);
+  }
+
+private:
+  int outcount;
+  FILE *outfile;
+};
+
+void _output_and_destroy_tree(expire_tiles::tile_output *output, struct expire_tiles::tile * tree, int x, int y, int this_zoom, int min_zoom) {
 	int	sub_x = x << 1;
 	int	sub_y = y << 1;
-	FILE *	ofile;
+        expire_tiles::tile_output *out;
 
 	if (! tree) return;
 
-	ofile = outfile;
-	if ((tree->complete[0][0]) && outfile) {
-		output_dirty_tile(outfile, sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom, outcount);
-		ofile = NULL;
+	out = output;
+	if ((tree->complete[0][0]) && output) {
+		output->output_dirty_tile(sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom);
+		out = NULL;
 	}
-	if (tree->subtiles[0][0]) _output_and_destroy_tree(ofile, tree->subtiles[0][0], sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom, outcount);
+	if (tree->subtiles[0][0]) _output_and_destroy_tree(out, tree->subtiles[0][0], sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom);
 
-	ofile = outfile;
-	if ((tree->complete[0][1]) && outfile) {
-		output_dirty_tile(outfile, sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom, outcount);
-		ofile = NULL;
+	out = output;
+	if ((tree->complete[0][1]) && output) {
+		output->output_dirty_tile(sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom);
+		out = NULL;
 	}
-	if (tree->subtiles[0][1]) _output_and_destroy_tree(ofile, tree->subtiles[0][1], sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom, outcount);
+	if (tree->subtiles[0][1]) _output_and_destroy_tree(out, tree->subtiles[0][1], sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom);
 
-	ofile = outfile;
-	if ((tree->complete[1][0]) && outfile) {
-		output_dirty_tile(outfile, sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom, outcount);
-		ofile = NULL;
+	out = output;
+	if ((tree->complete[1][0]) && output) {
+		output->output_dirty_tile(sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom);
+		out = NULL;
 	}
-	if (tree->subtiles[1][0]) _output_and_destroy_tree(ofile, tree->subtiles[1][0], sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom, outcount);
+	if (tree->subtiles[1][0]) _output_and_destroy_tree(out, tree->subtiles[1][0], sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom);
 
-	ofile = outfile;
-	if ((tree->complete[1][1]) && outfile) {
-		output_dirty_tile(outfile, sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom, outcount);
-		ofile = NULL;
+	out = output;
+	if ((tree->complete[1][1]) && output) {
+		output->output_dirty_tile(sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom);
+		out = NULL;
 	}
-	if (tree->subtiles[1][1]) _output_and_destroy_tree(ofile, tree->subtiles[1][1], sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom, outcount);
+	if (tree->subtiles[1][1]) _output_and_destroy_tree(out, tree->subtiles[1][1], sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom);
 
 	free(tree);
 }
 
+// merge the two trees, destroying b in the process. returns the
+// number of completed subtrees.
+int _tree_merge(struct expire_tiles::tile **a,
+                struct expire_tiles::tile **b) {
+  if (*a == NULL) {
+    *a = *b;
+    *b = NULL;
+
+  } else if (*b != NULL) {
+    for (int x = 0; x < 2; ++x) {
+      for (int y = 0; y < 2; ++y) {
+        // if b is complete on a subtree, then the merged tree must
+        // be complete too.
+        if ((*b)->complete[x][y]) {
+          (*a)->complete[x][y] = (*b)->complete[x][y];
+          destroy_tree((*a)->subtiles[x][y]);
+          (*a)->subtiles[x][y] = NULL;
+
+          // but if a is already complete, don't bother moving across
+          // anything
+        } else if (!(*a)->complete[x][y]) {
+          int complete = _tree_merge(&((*a)->subtiles[x][y]), &((*b)->subtiles[x][y]));
+
+          if (complete >= 4) {
+            (*a)->complete[x][y] = 1;
+            destroy_tree((*a)->subtiles[x][y]);
+            (*a)->subtiles[x][y] = NULL;
+          }
+        }
+
+        destroy_tree((*b)->subtiles[x][y]);
+        (*b)->subtiles[x][y] = NULL;
+      }
+    }
+  }
+
+  // count the number complete, so we can return it
+  int a_complete = 0;
+  for (int x = 0; x < 2; ++x) {
+    for (int y = 0; y < 2; ++y) {
+      if ((*a != NULL) && ((*a)->complete[x][y])) {
+        ++a_complete;
+      }
+    }
+  }
+
+  return a_complete;
+}
+
 } // anonymous namespace
 
-void expire_tiles::output_and_destroy_tree(FILE * outfile, struct tile * tree) {
-    _output_and_destroy_tree(outfile, tree, 0, 0, 0, Options->expire_tiles_zoom_min, outcount);
+void expire_tiles::output_and_destroy(tile_output *output) {
+    _output_and_destroy_tree(output, dirty, 0, 0, 0, Options->expire_tiles_zoom_min);
+    dirty = NULL;
 }
 
-expire_tiles::~expire_tiles() {
-	FILE *	outfile;
+void expire_tiles::output_and_destroy() {
+  if (Options->expire_tiles_zoom >= 0) {
+    tile_output_file output(Options->expire_tiles_filename);
 
-	if (Options->expire_tiles_zoom < 0) return;
-	outcount = 0;
-	if ((outfile = fopen(Options->expire_tiles_filename.c_str(), "a"))) {
-	    output_and_destroy_tree(outfile, dirty);
-	    fclose(outfile);
-	} else {
-        fprintf(stderr, "Failed to open expired tiles file (%s).  Tile expiry list will not be written!\n", strerror(errno));
-    }
-	dirty = NULL;
+    output_and_destroy(&output);
+  }
+}
+
+expire_tiles::~expire_tiles() {
+  if (dirty != NULL) {
+    destroy_tree(dirty);
+    dirty = NULL;
+  }
 }
 
 expire_tiles::expire_tiles(const struct options_t *options)
     : Options(options), map_width(0), tile_width(0),
-      dirty(NULL), outcount(0)
+      dirty(NULL)
 {
 	if (Options->expire_tiles_zoom < 0) return;
 	map_width = 1 << Options->expire_tiles_zoom;
@@ -441,4 +516,21 @@ int expire_tiles::from_db(table_t* table, osmid_t osm_id) {
     return wkts->get_count();
 }
 
+void expire_tiles::merge_and_destroy(expire_tiles &other) {
+  if (map_width != other.map_width) {
+    throw std::runtime_error((boost::format("Unable to merge tile expiry sets when "
+                                            "map_width does not match: %1% != %2%.")
+                              % map_width % other.map_width).str());
+  }
+
+  if (tile_width != other.tile_width) {
+    throw std::runtime_error((boost::format("Unable to merge tile expiry sets when "
+                                            "tile_width does not match: %1% != %2%.")
+                              % tile_width % other.tile_width).str());
+  }
 
+  _tree_merge(&dirty, &other.dirty);
+
+  destroy_tree(other.dirty);
+  other.dirty = NULL;
+}
diff --git a/expire-tiles.hpp b/expire-tiles.hpp
@@ -24,22 +24,40 @@ struct expire_tiles : public boost::noncopyable {
         struct tile* subtiles[2][2];
     };
 
-    //TODO: a method to coalesce multiple tile trees into this
-    //objects tree then write that coalesced one only once
+    /* customisable tile output. this can be passed into the
+     * `output_and_destroy` function to override output to a file.
+     * this is primarily useful for testing.
+     */
+    struct tile_output {
+        virtual ~tile_output() {}
+        // dirty a tile at x, y & zoom, and all descendants of that
+        // tile at the given zoom if zoom < min_zoom.
+        virtual void output_dirty_tile(int x, int y, int zoom, int min_zoom) = 0;
+    };
+
+    // output the list of expired tiles to a file. note that this
+    // consumes the list of expired tiles destructively.
+    void output_and_destroy();
+
+    // output the list of expired tiles using a `tile_output`
+    // functor. this consumes the list of expired tiles destructively.
+    void output_and_destroy(tile_output *output);
+
+    // merge the list of expired tiles in the other object into this
+    // object, destroying the list in the other object.
+    void merge_and_destroy(expire_tiles &);
 
 private: 
     void expire_tile(int x, int y);
     int normalise_tile_x_coord(int x);
     void from_line(double lon_a, double lat_a, double lon_b, double lat_b);
     void from_xnodes_poly(const struct osmNode * const * xnodes, int * xcount, osmid_t osm_id);
     void from_xnodes_line(const struct osmNode * const * xnodes, int * xcount);
-    void output_and_destroy_tree(FILE * outfile, struct tile * tree);
 
     int map_width;
     double tile_width;
     const struct options_t *Options;
     struct tile *dirty;
-    int outcount;
 };
 
 #endif
diff --git a/output-multi.cpp b/output-multi.cpp
@@ -170,6 +170,7 @@ void output_multi_t::rel_cb_func::finish(int exists) {
 
 void output_multi_t::stop() {
     m_table->stop();
+    m_expire->output_and_destroy();
     m_expire.reset();
 }
 

diff --git a/output-pgsql.cpp b/output-pgsql.cpp
@@ -425,6 +425,7 @@ void output_pgsql_t::stop()
     }
 #endif
 
+    expire->output_and_destroy();
     expire.reset();
 }