Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add new option --ignore-timestamps to osmium derive-changes #88

Closed
wants to merge 1 commit into from
Closed

Add new option --ignore-timestamps to osmium derive-changes #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions man/osmium-derive-changes.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,14 @@ STDOUT.
: Update timestamp of deleted objects to the current time. This is the same
behaviour as Osmosis.

--ignore-timestamps
: Do not use the timestamp to compare two objects. If the objects in one
input file have timestamps but the objects in the other input file do not
have timestamps, all objects will be written to the input file twice
because their metadata changed (fields being added or dropped) although
their coordinates, tags or members did not change. This option prevents the
diff become huge under these circumstances.


@MAN_COMMON_OPTIONS@
@MAN_PROGRESS_OPTIONS@
Expand Down
25 changes: 20 additions & 5 deletions src/command_derive_changes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
#include <osmium/io/reader_with_progress_bar.hpp>
#include <osmium/io/writer.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/object_comparisons.hpp>
#include <osmium/util/verbose_output.hpp>

#include <boost/program_options.hpp>
Expand All @@ -48,6 +48,8 @@ bool CommandDeriveChanges::setup(const std::vector<std::string>& arguments) {
("increment-version", "Increment version of deleted objects")
("keep-details", "Keep tags (and nodes of ways, members of relations) of deleted objects")
("update-timestamp", "Set timestamp of deleted objects to current time")
("ignore-timestamps", "Don't use the timestamp to compare two objects. Use this if both input files have\n" \
"version fields but one input file has not timestamps. This option prevents the diff become huge.\n")
;

po::options_description opts_common{add_common_options()};
Expand Down Expand Up @@ -93,6 +95,10 @@ bool CommandDeriveChanges::setup(const std::vector<std::string>& arguments) {
m_update_timestamp = true;
}

if (vm.count("ignore-timestamps")) {
m_ignore_metadata_changes = true;
}

return true;
}

Expand Down Expand Up @@ -148,6 +154,16 @@ void CommandDeriveChanges::write_deleted(osmium::io::Writer& writer, osmium::OSM
}

bool CommandDeriveChanges::run() {
if (m_ignore_metadata_changes) {
derive_changes<osmium::object_order_type_id_version_without_timestamp>();
} else {
derive_changes<osmium::object_order_type_id_version>();
}
return true;
}

template <typename TComp>
void CommandDeriveChanges::derive_changes() {
m_vout << "Opening input files...\n";
osmium::io::Reader reader1{m_input_files[0], osmium::osm_entity_bits::object};
osmium::io::ReaderWithProgressBar reader2{display_progress(), m_input_files[1], osmium::osm_entity_bits::object};
Expand All @@ -171,14 +187,15 @@ bool CommandDeriveChanges::run() {

reader2.progress_bar().remove();
m_vout << "Deriving changes...\n";
TComp comp;
while (it1 != end1 || it2 != end2) {
if (it2 == end2) {
write_deleted(writer, *it1);
++it1;
} else if (it1 == end1 || *it2 < *it1) {
} else if (it1 == end1 || comp(*it2, *it1)) {
writer(*it2);
++it2;
} else if (*it1 < *it2) {
} else if (comp(*it1, *it2)) {
if (it2->id() != it1->id()) {
write_deleted(writer, *it1);
}
Expand All @@ -195,7 +212,5 @@ bool CommandDeriveChanges::run() {

show_memory_used();
m_vout << "Done.\n";

return true;
}

4 changes: 4 additions & 0 deletions src/command_derive_changes.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ class CommandDeriveChanges : public Command, public with_multiple_osm_inputs, pu
bool m_keep_details = false;
bool m_update_timestamp = false;
bool m_increment_version = false;
bool m_ignore_metadata_changes = false;

template <typename TComp>
void derive_changes();

public:

Expand Down
12 changes: 12 additions & 0 deletions test/derive-changes/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,17 @@ check_derive_changes(normal "" input1.osm input2.osm ou
check_derive_changes(keep_details "--keep-details" input1.osm input2.osm output-keep-details.osc)
check_derive_changes(incr_version "--increment-version" input1.osm input2.osm output-incr-version.osc)

# Tests with input file which don't have the same amount of metadata fields:
# File 1 has all metadata fields, file 2 has only version fields.
check_derive_changes(new_file_only_versions "--ignore-timestamps" input1.osm input2-only-versions.osm output-2-only-version.osc)

# File 1 has only version and timestamp fields, file 2 has only version fields.
check_derive_changes(version_timestamp_with_version "--ignore-timestamps" input1-only-version-timestamp.osm input2-only-versions.osm output-2-only-version-timestamp.osc)

# File 1 has only version fields, file 2 has version and timestamp fields.
check_derive_changes(version_with_version_timestamp "--ignore-timestamps" input1-only-version.osm input2-only-version-timestamp.osm output-2-version-with-version-timestamp.osc)

# File 1 has only version fields, file 2 has all metadata fields.
check_derive_changes(version_with_all "--ignore-timestamps" input1-only-version.osm input2-all-with-relation.osm output-2-version-with-all.osc)

#-----------------------------------------------------------------------------
24 changes: 24 additions & 0 deletions test/derive-changes/input1-only-version-timestamp.osm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<?xml version='1.0' encoding='UTF-8'?>
<osm version="0.6" upload="false" generator="testdata">
<node id="10" version="1" timestamp="2015-01-01T01:00:00Z" lat="1" lon="1"/>
<node id="11" version="1" timestamp="2015-01-01T01:00:00Z" lat="2" lon="1"/>
<node id="12" version="1" timestamp="2015-01-01T01:00:00Z" lat="3" lon="1"/>
<node id="13" version="1" timestamp="2015-01-01T01:00:00Z" lat="4" lon="1">
<tag k="foo" v="bar"/>
</node>
<way id="20" version="1" timestamp="2015-01-01T01:00:00Z">
<nd ref="10"/>
<nd ref="11"/>
<nd ref="12"/>
<tag k="foo" v="bar"/>
</way>
<way id="21" version="1" timestamp="2015-01-01T01:00:00Z">
<nd ref="12"/>
<nd ref="13"/>
<tag k="xyz" v="abc"/>
</way>
<relation id="30" version="1" timestamp="2015-01-01T01:00:00Z">
<member type="node" ref="12" role="m1"/>
<member type="way" ref="20" role="m2"/>
</relation>
</osm>
26 changes: 26 additions & 0 deletions test/derive-changes/input1-only-version.osm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?xml version='1.0' encoding='UTF-8'?>
<osm version="0.6" generator="osmium/1.7.1">
<node id="1" version="2" lat="50" lon="10">
<tag k="highway" v="bus_stop"/>
<tag k="name" v="School"/>
</node>
<node id="2" version="2" lat="50" lon="10.01"/>
<node id="3" version="1" lat="50" lon="10.02"/>
<way id="1" version="1">
<nd ref="1"/>
<nd ref="2"/>
<nd ref="3"/>
<tag k="highway" v="residential"/>
</way>
<way id="2" version="1">
<nd ref="1"/>
<nd ref="3"/>
<tag k="highway" v="path"/>
</way>
<relation id="1" version="1">
<member type="node" ref="1" role="stop"/>
<member type="way" ref="1" role=""/>
<tag k="type" v="route"/>
<tag k="route" v="bus"/>
</relation>
</osm>
26 changes: 26 additions & 0 deletions test/derive-changes/input2-all-with-relation.osm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?xml version='1.0' encoding='UTF-8'?>
<osm version="0.6">
<node id="1" user="foo" uid="2" changeset="30" timestamp="2017-12-03T00:00:00Z" version="3" lat="50.001" lon="10.0">
<tag k="highway" v="bus_stop" />
<tag k="name" v="School" />
</node>
<node id="2" user="foo" uid="2" changeset="3" timestamp="2017-01-03T00:00:00Z" version="2" lat="50.0" lon="10.01"/>
<node id="4" user="foo" uid="2" changeset="3" timestamp="2017-12-03T00:00:00Z" version="1" lat="50.0" lon="10.02"/>
<way id="1" user="foo" uid="2" changeset="3" timestamp="2017-12-03T00:00:00Z" version="2">
<tag k="highway" v="residential" />
<nd ref="1"/>
<nd ref="2"/>
<nd ref="4"/>
</way>
<way id="2" user="foo" uid="2" changeset="3" timestamp="2017-01-03T00:00:00Z" version="1">
<tag k="highway" v="path" />
<nd ref="1"/>
<nd ref="2"/>
</way>
<relation id="1" user="foo" uid="2" changeset="3" timestamp="2017-01-03T00:00:00Z" version="1">
<tag k="type" v="route" />
<tag k="route" v="bus" />
<member type="node" ref="1" role="stop"/>
<member type="way" ref="1" role=""/>
</relation>
</osm>
26 changes: 26 additions & 0 deletions test/derive-changes/input2-only-version-timestamp.osm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?xml version='1.0' encoding='UTF-8'?>
<osm version="0.6">
<node id="1" timestamp="2017-12-03T00:00:00Z" version="3" lat="50.001" lon="10.0">
<tag k="highway" v="bus_stop" />
<tag k="name" v="School" />
</node>
<node id="2" timestamp="2017-01-03T00:00:00Z" version="2" lat="50.0" lon="10.01"/>
<node id="4" timestamp="2017-12-03T00:00:00Z" version="1" lat="50.0" lon="10.02"/>
<way id="1" timestamp="2017-12-03T00:00:00Z" version="2">
<tag k="highway" v="residential" />
<nd ref="1"/>
<nd ref="2"/>
<nd ref="4"/>
</way>
<way id="2" timestamp="2017-01-03T00:00:00Z" version="1">
<tag k="highway" v="path" />
<nd ref="1"/>
<nd ref="2"/>
</way>
<relation id="1" timestamp="2017-01-03T00:00:00Z" version="1">
<tag k="type" v="route" />
<tag k="route" v="bus" />
<member type="node" ref="1" role="stop"/>
<member type="way" ref="1" role=""/>
</relation>
</osm>
22 changes: 22 additions & 0 deletions test/derive-changes/input2-only-versions.osm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?xml version='1.0' encoding='UTF-8'?>
<osm version="0.6" generator="test">
<node id="10" version="1" lat="1" lon="1"/>
<node id="11" version="2" lat="2" lon="2"/>
<node id="12" version="1" lat="3" lon="1"/>
<node id="14" version="1" lat="5" lon="1"/>
<way id="20" version="1">
<nd ref="10"/>
<nd ref="11"/>
<nd ref="12"/>
<tag k="foo" v="bar"/>
</way>
<way id="21" version="2">
<nd ref="12"/>
<nd ref="14"/>
<tag k="xyz" v="new"/>
</way>
<relation id="30" version="1">
<member type="node" ref="12" role="m1"/>
<member type="way" ref="20" role="m2"/>
</relation>
</osm>
19 changes: 19 additions & 0 deletions test/derive-changes/output-2-only-version-timestamp.osc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<?xml version='1.0' encoding='UTF-8'?>
<osmChange version="0.6" generator="test">
<modify>
<node id="11" version="2" lat="2" lon="2"/>
</modify>
<delete>
<node id="13" version="1" timestamp="2015-01-01T01:00:00Z"/>
</delete>
<create>
<node id="14" version="1" lat="5" lon="1"/>
</create>
<modify>
<way id="21" version="2">
<nd ref="12"/>
<nd ref="14"/>
<tag k="xyz" v="new"/>
</way>
</modify>
</osmChange>
19 changes: 19 additions & 0 deletions test/derive-changes/output-2-only-version.osc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<?xml version='1.0' encoding='UTF-8'?>
<osmChange version="0.6" generator="test">
<modify>
<node id="11" version="2" lat="2" lon="2"/>
</modify>
<delete>
<node id="13" version="1" timestamp="2015-01-01T01:00:00Z"/>
</delete>
<create>
<node id="14" version="1" lat="5" lon="1"/>
</create>
<modify>
<way id="21" version="2">
<nd ref="12"/>
<nd ref="14"/>
<tag k="xyz" v="new"/>
</way>
</modify>
</osmChange>
23 changes: 23 additions & 0 deletions test/derive-changes/output-2-version-with-all.osc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?xml version='1.0' encoding='UTF-8'?>
<osmChange version="0.6" generator="test">
<modify>
<node id="1" version="3" timestamp="2017-12-03T00:00:00Z" uid="2" user="foo" changeset="30" lat="50.001" lon="10">
<tag k="highway" v="bus_stop"/>
<tag k="name" v="School"/>
</node>
</modify>
<delete>
<node id="3" version="1"/>
</delete>
<create>
<node id="4" version="1" timestamp="2017-12-03T00:00:00Z" uid="2" user="foo" changeset="3" lat="50" lon="10.02"/>
</create>
<modify>
<way id="1" version="2" timestamp="2017-12-03T00:00:00Z" uid="2" user="foo" changeset="3">
<nd ref="1"/>
<nd ref="2"/>
<nd ref="4"/>
<tag k="highway" v="residential"/>
</way>
</modify>
</osmChange>
23 changes: 23 additions & 0 deletions test/derive-changes/output-2-version-with-version-timestamp.osc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?xml version='1.0' encoding='UTF-8'?>
<osmChange version="0.6" generator="test">
<modify>
<node id="1" version="3" timestamp="2017-12-03T00:00:00Z" lat="50.001" lon="10">
<tag k="highway" v="bus_stop"/>
<tag k="name" v="School"/>
</node>
</modify>
<delete>
<node id="3" version="1"/>
</delete>
<create>
<node id="4" version="1" timestamp="2017-12-03T00:00:00Z" lat="50" lon="10.02"/>
</create>
<modify>
<way id="1" version="2" timestamp="2017-12-03T00:00:00Z">
<nd ref="1"/>
<nd ref="2"/>
<nd ref="4"/>
<tag k="highway" v="residential"/>
</way>
</modify>
</osmChange>