Skip to content

Commit

Permalink
[osm2pgsql] Report some more timing information to see where osm2pgsq…
Browse files Browse the repository at this point in the history 
…l spends its time

git-svn-id: http://svn.openstreetmap.org/applications/utils/export/osm2pgsql@26588 b9d5c4c9-76e1-0310-9c85-f3177eceb1e4
  • Loading branch information
apmon committed Aug 28, 2011
1 parent c2c307d commit 8c21f8c
Show file tree
Hide file tree
Showing 6 changed files with 86 additions and 15 deletions.
25 changes: 21 additions & 4 deletions middle-pgsql.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include <string.h>
#include <assert.h>
#include <math.h>
#include <time.h>

#ifdef HAVE_PTHREAD
#include <pthread.h>
Expand Down Expand Up @@ -913,6 +914,8 @@ static void pgsql_iterate_ways(int (*callback)(osmid_t id, struct keyval *tags,
/* The flag we pass to indicate that the way in question might exist already in the database */
int exists = Append;

time_t start, end;
time(&start);
fprintf(stderr, "\nGoing over pending ways\n");

/* Make sure we're out of copy mode */
Expand All @@ -927,8 +930,11 @@ static void pgsql_iterate_ways(int (*callback)(osmid_t id, struct keyval *tags,
struct osmNode *nodes;
int nd_count;

if (count++ %1000 == 0)
fprintf(stderr, "\rprocessing way (%dk)", count/1000);
if (count++ %1000 == 0) {
time(&end);
fprintf(stderr, "\rprocessing way (%dk) at %.2fk/s", count/1000,
(int)(end - start) > 0 ? ((double)count / 1000.0 / (double)(end - start)) : 0);
}

initList(&tags);
if( pgsql_ways_get(id, &tags, &nodes, &nd_count) )
Expand All @@ -943,6 +949,9 @@ static void pgsql_iterate_ways(int (*callback)(osmid_t id, struct keyval *tags,

PQclear(res_ways);
fprintf(stderr, "\n");
time(&end);
if (end - start > 0)
fprintf(stderr, "Pending ways took %ds at a rate of %.2f/s\n",(int)(end - start), ((double)count / (double)(end - start)));
}

static int pgsql_way_changed(osmid_t osm_id)
Expand Down Expand Up @@ -1112,6 +1121,8 @@ static void pgsql_iterate_relations(int (*callback)(osmid_t id, struct member *m
/* The flag we pass to indicate that the way in question might exist already in the database */
int exists = Append;

time_t start, end;
time(&start);
fprintf(stderr, "\nGoing over pending relations\n");

/* Make sure we're out of copy mode */
Expand All @@ -1126,8 +1137,11 @@ static void pgsql_iterate_relations(int (*callback)(osmid_t id, struct member *m
struct member *members;
int member_count;

if (count++ %10 == 0)
fprintf(stderr, "\rprocessing relation (%d)", count);
if (count++ %10 == 0) {
time(&end);
fprintf(stderr, "\rprocessing relation (%d) at %.2f/s", count,
(int)(end - start) > 0 ? ((double)count / (double)(end - start)) : 0);
}

initList(&tags);
if( pgsql_rels_get(id, &members, &member_count, &tags) )
Expand All @@ -1142,6 +1156,9 @@ static void pgsql_iterate_relations(int (*callback)(osmid_t id, struct member *m

PQclear(res_rels);
fprintf(stderr, "\n");
time(&end);
if (end - start > 0)
fprintf(stderr, "Pending relations took %ds at a rate of %.2f/s\n",(int)(end - start), ((double)count / (double)(end - start)));
}

static int pgsql_rel_changed(osmid_t osm_id)
Expand Down
35 changes: 30 additions & 5 deletions osm2pgsql.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,20 @@ void resetMembers(struct osmdata_t *osmdata)

void printStatus(struct osmdata_t *osmdata)
{
fprintf(stderr, "\rProcessing: Node(%" PRIdOSMID "k) Way(%" PRIdOSMID "k) Relation(%" PRIdOSMID ")",
osmdata->count_node/1000, osmdata->count_way/1000, osmdata->count_rel);
time_t now;
time(&now);
time_t end_nodes = osmdata->start_way > 0 ? osmdata->start_way : now;
time_t end_way = osmdata->start_rel > 0 ? osmdata->start_rel : now;
time_t end_rel = now;
fprintf(stderr, "\rProcessing: Node(%" PRIdOSMID "k %.1fk/s) Way(%" PRIdOSMID "k %.2fk/s) Relation(%" PRIdOSMID " %.2f/s)",
osmdata->count_node/1000,
(double)osmdata->count_node/1000.0/((int)(end_nodes - osmdata->start_node) > 0 ? (double)(end_nodes - osmdata->start_node) : 1.0),
osmdata->count_way/1000,
osmdata->count_way > 0 ? (double)osmdata->count_way/1000.0/
((double)(end_way - osmdata->start_way) > 0.0 ? (double)(end_way - osmdata->start_way) : 1.0) : 0.0,
osmdata->count_rel,
osmdata->count_rel > 0 ? (double)osmdata->count_rel/
((double)(end_rel - osmdata->start_rel) > 0.0 ? (double)(end_rel - osmdata->start_rel) : 1.0) : 0.0);
}

int node_wanted(struct osmdata_t *osmdata, double lat, double lon)
Expand Down Expand Up @@ -534,6 +546,9 @@ int main(int argc, char *argv[])
}
}

time_t overall_start, overall_end;
time(&overall_start);

osmdata.out->start(&options);

realloc_nodes(&osmdata);
Expand Down Expand Up @@ -575,10 +590,18 @@ int main(int argc, char *argv[])
xmlMemoryDump();

if (osmdata.count_node || osmdata.count_way || osmdata.count_rel) {
time_t now;
time(&now);
time_t end_nodes = osmdata.start_way > 0 ? osmdata.start_way : now;
time_t end_way = osmdata.start_rel > 0 ? osmdata.start_rel : now;
time_t end_rel = now;
fprintf(stderr, "\n");
fprintf(stderr, "Node stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ")\n", osmdata.count_node, osmdata.max_node);
fprintf(stderr, "Way stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ")\n", osmdata.count_way, osmdata.max_way);
fprintf(stderr, "Relation stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ")\n", osmdata.count_rel, osmdata.max_rel);
fprintf(stderr, "Node stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", osmdata.count_node, osmdata.max_node,
osmdata.count_node > 0 ? (int)(end_nodes - osmdata.start_node) : 0);
fprintf(stderr, "Way stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", osmdata.count_way, osmdata.max_way,
osmdata.count_way > 0 ? (int)(end_way - osmdata.start_way) : 0);
fprintf(stderr, "Relation stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", osmdata.count_rel, osmdata.max_rel,
osmdata.count_rel > 0 ? (int)(end_rel - osmdata.start_rel) : 0);
}
osmdata.out->stop();

Expand All @@ -591,6 +614,8 @@ int main(int argc, char *argv[])
project_exit();
text_exit();
fprintf(stderr, "\n");
time(&overall_end);
fprintf(stderr, "Osm2pgsql took %ds overall\n", (int)(overall_end - overall_start));

return 0;
}
1 change: 1 addition & 0 deletions osmtypes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ struct osmdata_t {
osmid_t count_node, max_node;
osmid_t count_way, max_way;
osmid_t count_rel, max_rel;
time_t start_node, start_way, start_rel;

struct output_t *out;

Expand Down
7 changes: 6 additions & 1 deletion output-pgsql.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include <string.h>
#include <assert.h>
#include <errno.h>
#include <time.h>

#ifdef HAVE_PTHREAD
#include <pthread.h>
Expand Down Expand Up @@ -1359,6 +1360,8 @@ static void *pgsql_out_stop_one(void *arg)
pgsql_exec(sql_conn, PGRES_COMMAND_OK, "COMMIT");
if (!Options->append)
{
time_t start, end;
time(&start);
fprintf(stderr, "Sorting data and creating indexes for %s\n", table->name);
pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ANALYZE %s;\n", table->name);
pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE TABLE %s_tmp AS SELECT * FROM %s ORDER BY way;\n", table->name, table->name);
Expand All @@ -1381,9 +1384,12 @@ static void *pgsql_out_stop_one(void *arg)
}
pgsql_exec(sql_conn, PGRES_COMMAND_OK, "GRANT SELECT ON %s TO PUBLIC;\n", table->name);
pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ANALYZE %s;\n", table->name);
time(&end);
fprintf(stderr, "Indexes on %s created in %ds\n", table->name, (int)(end - start));
}
PQfinish(sql_conn);
table->sql_conn = NULL;

fprintf(stderr, "Completed %s\n", table->name);
free(table->name);
free(table->columns);
Expand Down Expand Up @@ -1498,7 +1504,6 @@ static int pgsql_process_relation(osmid_t id, struct member *members, int member
xcount[count] = 0;
xid[count] = 0;
xrole[count] = NULL;

// At some point we might want to consider storing the retreived data in the members, rather than as seperate arrays
pgsql_out_relation(id, tags, xnodes, xtags, xcount, xid, xrole);

Expand Down
13 changes: 13 additions & 0 deletions parse-pbf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <stdio.h>
#include <string.h>
#include <arpa/inet.h>
#include <time.h>

#include <zlib.h>
#include <bzlib.h>
Expand Down Expand Up @@ -283,6 +284,9 @@ int processOsmDataNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, String
osmdata->max_node = node->id;
}

if (osmdata->count_node == 0) {
time(&osmdata->start_node);
}
osmdata->count_node++;
if (osmdata->count_node%10000 == 0)
printStatus(osmdata);
Expand Down Expand Up @@ -357,6 +361,9 @@ int processOsmDataDenseNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, S
osmdata->max_node = deltaid;
}

if (osmdata->count_node == 0) {
time(&osmdata->start_node);
}
osmdata->count_node++;
if (osmdata->count_node%10000 == 0)
printStatus(osmdata);
Expand Down Expand Up @@ -405,6 +412,9 @@ int processOsmDataWays(struct osmdata_t *osmdata, PrimitiveGroup *group, StringT
osmdata->max_way = way->id;
}

if (osmdata->count_way == 0) {
time(&osmdata->start_way);
}
osmdata->count_way++;
if (osmdata->count_way%1000 == 0)
printStatus(osmdata);
Expand Down Expand Up @@ -481,6 +491,9 @@ int processOsmDataRelations(struct osmdata_t *osmdata, PrimitiveGroup *group, St
osmdata->max_rel = relation->id;
}

if (osmdata->count_rel == 0) {
time(&osmdata->start_rel);
}
osmdata->count_rel++;
if (osmdata->count_rel%10 == 0)
printStatus(osmdata);
Expand Down
20 changes: 15 additions & 5 deletions parse-xml2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <time.h>

#include <libxml/xmlstring.h>
#include <libxml/xmlreader.h>
Expand Down Expand Up @@ -107,6 +108,9 @@ static void StartElement(xmlTextReaderPtr reader, const xmlChar *name, struct os
if (osmdata->osm_id > osmdata->max_node)
osmdata->max_node = osmdata->osm_id;

if (osmdata->count_node == 0) {
time(&osmdata->start_node);
}
osmdata->count_node++;
if (osmdata->count_node%10000 == 0)
printStatus(osmdata);
Expand Down Expand Up @@ -138,14 +142,17 @@ static void StartElement(xmlTextReaderPtr reader, const xmlChar *name, struct os
assert(xid);
osmdata->osm_id = strtoosmid((char *)xid, NULL, 10);
osmdata->action = ParseAction( reader, osmdata );

if (osmdata->osm_id > osmdata->max_way)
osmdata->max_way = osmdata->osm_id;

osmdata->max_way = osmdata->osm_id;

if (osmdata->count_way == 0) {
time(&osmdata->start_way);
}
osmdata->count_way++;
if (osmdata->count_way%1000 == 0)
printStatus(osmdata);
printStatus(osmdata);

osmdata->nd_count = 0;
xmlFree(xid);

Expand All @@ -167,6 +174,9 @@ static void StartElement(xmlTextReaderPtr reader, const xmlChar *name, struct os
if (osmdata->osm_id > osmdata->max_rel)
osmdata->max_rel = osmdata->osm_id;

if (osmdata->count_rel == 0) {
time(&osmdata->start_rel);
}
osmdata->count_rel++;
if (osmdata->count_rel%10 == 0)
printStatus(osmdata);
Expand Down

0 comments on commit 8c21f8c

Please sign in to comment.