Latest DB.pm from snippets

nigelhorne · Dec 6, 2019 · 471d9fc · 471d9fc
1 parent 0d1b0cd
commit 471d9fc
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 33 deletions.
diff --git a/Makefile.PL b/Makefile.PL
@@ -64,15 +64,26 @@ foreach my $url(keys %urls) {
 		}) || die "$0: $cachedir: $!";
 	}
 
-	print "Downloading $url\n";
-	my $ff = File::Fetch->new(uri => $url);
-	if(my $path = $ff->fetch($url)) {
-		print "Moving $path to $urls{$url}\n";
-		rename $path, $urls{$url} or die "$urls{$url}: $!";
-	} elsif(my $err = $ff->error(1)) {
-		die $err;
+	my $download = 0;
+	my $file = $urls{$url};
+	if(-r $file) {
+		if((-M $file) > 1) {
+			$download = 1;
+		}
 	} else {
-		die "There was a problem downloading $url";
+		$download = 1;
+	}
+	if($download) {
+		print "Downloading $url\n";
+		my $ff = File::Fetch->new(uri => $url);
+		if(my $path = $ff->fetch($url)) {
+			print "Moving $path to $file\n";
+			rename $path, $file or die "$file: $!";
+		} elsif(my $err = $ff->error(1)) {
+			die $err;
+		} else {
+			die "There was a problem downloading $url";
+		}
 	}
 }
 

diff --git a/lib/Geo/Coder/Free/DB.pm b/lib/Geo/Coder/Free/DB.pm
@@ -32,7 +32,13 @@ package Geo::Coder::Free::DB;
 # my $row = $foo->fetchrow_hashref(customer_id => '12345);
 # print Data::Dumper->new([$row])->Dump();
 
-# FIXME:  there needs to be a column called 'entry' which is used for sort
+# CSV files can have empty lines of comment lines starting with '#', to make them more readable
+
+# If the table has a column called "entry", sorts are based on that
+# To turn that off, pass 'no_entry' to the constructor, for legacy
+# reasons it's enabled by default
+# TODO: Switch that to off by default, and enable by passing 'entry'
+
 # TODO: support a directory hierachy of databases
 # TODO: consider returning an object or array of objects, rather than hashes
 # TODO:	Add redis database - could be of use for Geo::Coder::Free
@@ -71,7 +77,8 @@ sub new {
 		logger => $args{'logger'} || $logger,
 		directory => $args{'directory'} || $directory,	# The directory conainting the tables in XML, SQLite or CSV format
 		cache => $args{'cache'} || $cache,
-		table => $args{'table'}	# The name of the file containing the table, defaults to the class name
+		table => $args{'table'},	# The name of the file containing the table, defaults to the class name
+		no_entry => $args{'no_entry'} || 0,
 	}, $class;
 }
 
@@ -249,7 +256,9 @@ sub _open {
 			)};
 
 			# Don't use blank lines or comments
-			@data = grep { $_->{'entry'} !~ /^#/ } grep { defined($_->{'entry'}) } @data;
+			unless($self->{no_entry}) {
+				@data = grep { $_->{'entry'} !~ /^\s*#/ } grep { defined($_->{'entry'}) } @data;
+			}
 			# $self->{'data'} = @data;
 			my $i = 0;
 			$self->{'data'} = ();
@@ -311,9 +320,10 @@ sub selectall_hash {
 	# }
 
 	my $query;
-	if($self->{'type'} eq 'CSV') {
-		# $query = "SELECT * FROM $table WHERE entry IS NOT NULL AND entry NOT LIKE '#%'";
-		$query = "SELECT * FROM $table";
+	my $done_where = 0;
+	if(($self->{'type'} eq 'CSV') && !$self->{no_entry}) {
+		$query = "SELECT * FROM $table WHERE entry IS NOT NULL AND entry NOT LIKE '#%'";
+		$done_where = 1;
 	} else {
 		$query = "SELECT * FROM $table";
 	}
@@ -329,8 +339,7 @@ sub selectall_hash {
 		if(!defined($arg)) {
 			throw Error::Simple("$query: value for $c1 is not defined");
 		}
-		# if(scalar(@query_args) || ($self->{'type'} eq 'CSV')) {
-		if(scalar(@query_args)) {
+		if($done_where) {
 			if($arg =~ /\@/) {
 				$query .= " AND $c1 LIKE ?";
 			} else {
@@ -342,10 +351,13 @@ sub selectall_hash {
 			} else {
 				$query .= " WHERE $c1 = ?";
 			}
+			$done_where = 1;
 		}
 		push @query_args, $arg;
 	}
-	# $query .= ' ORDER BY entry';
+	if(!$self->{no_entry}) {
+		$query .= ' ORDER BY entry';
+	}
 	if($self->{'logger'}) {
 		if(defined($query_args[0])) {
 			$self->{'logger'}->debug("selectall_hash $query: ", join(', ', @query_args));
@@ -407,14 +419,15 @@ sub fetchrow_hashref {
 	} else {
 		$query .= $table;
 	}
-	# if($self->{'type'} eq 'CSV') {
-		# $query .= " WHERE entry IS NOT NULL AND entry NOT LIKE '#%'";
-	# }
+	my $done_where = 0;
+	if(($self->{'type'} eq 'CSV') && !$self->{no_entry}) {
+		$query .= " WHERE entry IS NOT NULL AND entry NOT LIKE '#%'";
+		$done_where = 1;
+	}
 	my @query_args;
 	foreach my $c1(sort keys(%params)) {	# sort so that the key is always the same
 		if(my $arg = $params{$c1}) {
-			# if(scalar(@query_args) || ($self->{'type'} eq 'CSV')) {
-			if(scalar(@query_args)) {
+			if($done_where) {
 				if($arg =~ /\@/) {
 					$query .= " AND $c1 LIKE ?";
 				} else {
@@ -426,6 +439,7 @@ sub fetchrow_hashref {
 				} else {
 					$query .= " WHERE $c1 = ?";
 				}
+				$done_where = 1;
 			}
 			push @query_args, $arg;
 		}
@@ -530,13 +544,13 @@ sub AUTOLOAD {
 
 	my $query;
 	if(wantarray && !delete($params{'distinct'})) {
-		if($self->{'type'} eq 'CSV') {
+		if(($self->{'type'} eq 'CSV') && !$self->{no_entry}) {
 			$query = "SELECT $column FROM $table WHERE entry IS NOT NULL AND entry NOT LIKE '#%'";
 		} else {
 			$query = "SELECT $column FROM $table";
 		}
 	} else {
-		if($self->{'type'} eq 'CSV') {
+		if(($self->{'type'} eq 'CSV') && !$self->{no_entry}) {
 			$query = "SELECT DISTINCT $column FROM $table WHERE entry IS NOT NULL AND entry NOT LIKE '#%'";
 		} else {
 			$query = "SELECT DISTINCT $column FROM $table";

diff --git a/lib/Geo/Coder/Free/MaxMind.pm b/lib/Geo/Coder/Free/MaxMind.pm
@@ -214,7 +214,7 @@ sub geocode {
 		} elsif($admin1cache{$country} && !defined($state)) {
 			$concatenated_codes = $admin1cache{$country};
 		} else {
-			$self->{'admin1'} //= Geo::Coder::Free::DB::MaxMind::admin1->new() or die "Can't open the admin1 database";
+			$self->{'admin1'} //= Geo::Coder::Free::DB::MaxMind::admin1->new(no_entry => 1) or die "Can't open the admin1 database";
 
 			if(my $admin1 = $self->{'admin1'}->fetchrow_hashref(asciiname => $country)) {
 				$concatenated_codes = $admin1->{'concatenated_codes'};
@@ -263,7 +263,7 @@ sub geocode {
 		}
 	}
 
-	$self->{'admin2'} //= Geo::Coder::Free::DB::MaxMind::admin2->new() or die "Can't open the admin2 database";
+	$self->{'admin2'} //= Geo::Coder::Free::DB::MaxMind::admin2->new(no_entry => 1) or die "Can't open the admin2 database";
 
 	if(defined($county) && ($county =~ /^[A-Z]{2}$/) && ($country =~ /^(United States|USA|US)$/)) {
 		# US state. Not Canadian province.
@@ -343,7 +343,7 @@ sub geocode {
 		} elsif(defined($county)) {
 			# e.g. states in the US
 			if(!defined($self->{'admin1'})) {
-				$self->{'admin1'} = Geo::Coder::Free::DB::MaxMind::admin1->new() or die "Can't open the admin1 database";
+				$self->{'admin1'} = Geo::Coder::Free::DB::MaxMind::admin1->new(no_entry => 1) or die "Can't open the admin1 database";
 			}
 			my @admin1s = $self->{'admin1'}->selectall_hash(asciiname => $county);
 			foreach my $admin1(@admin1s) {
@@ -359,7 +359,9 @@ sub geocode {
 
 	if(!defined($self->{'cities'})) {
 		$self->{'cities'} = Geo::Coder::Free::DB::MaxMind::cities->new(
-			cache => $self->{cache} || CHI->new(driver => 'Memory', datastore => {}));
+			cache => $self->{cache} || CHI->new(driver => 'Memory', datastore => {}),
+			no_entry => 1,
+		);
 	}
 
 	my $options;
@@ -514,7 +516,8 @@ sub reverse_geocode {
 
 	if(!defined($self->{'cities'})) {
 		$self->{'cities'} = Geo::Coder::Free::DB::MaxMind::cities->new(
-			cache => $self->{cache} || CHI->new(driver => 'Memory', datastore => {})
+			cache => $self->{cache} || CHI->new(driver => 'Memory', datastore => {}),
+			no_entry => 1,
 		);
 	}
 
@@ -550,7 +553,7 @@ sub _prepare {
 		if($county) {
 			$loc->{'Region'} = $county;
 		} else {
-			$self->{'admin2'} //= Geo::Coder::Free::DB::MaxMind::admin2->new() or die "Can't open the admin2 database";
+			$self->{'admin2'} //= Geo::Coder::Free::DB::MaxMind::admin2->new(no_entry => 1) or die "Can't open the admin2 database";
 			my $row = $self->{'admin2'}->execute("SELECT name FROM admin2 WHERE concatenated_codes LIKE '" . uc($loc->{'Country'}) . '.%.' . uc($region) . "' LIMIT 1");
 			if(ref($row) && $row->{'name'}) {
 				$admin2cache{$row->{'name'}} = $region;

diff --git a/t/admin1.t b/t/admin1.t
@@ -14,7 +14,7 @@ BEGIN {
 
 CITIES: {
 	Geo::Coder::Free::DB::init(directory => 'lib/Geo/Coder/Free/MaxMind/databases');
-	my $admin1 = new_ok('Geo::Coder::Free::DB::MaxMind::admin1' => [logger => new_ok('MyLogger')]);
+	my $admin1 = new_ok('Geo::Coder::Free::DB::MaxMind::admin1' => [logger => new_ok('MyLogger'), no_entry => 1]);
 
 	my $england = $admin1->fetchrow_hashref({ concatenated_codes => 'GB.ENG' });
 	ok($england->{asciiname} eq 'England');

diff --git a/t/admin2.t b/t/admin2.t
@@ -14,7 +14,7 @@ BEGIN {
 
 CITIES: {
 	Geo::Coder::Free::DB::init(directory => 'lib/Geo/Coder/Free/MaxMind/databases');
-	my $admin2 = new_ok('Geo::Coder::Free::DB::MaxMind::admin2' => [logger => new_ok('MyLogger')]);
+	my $admin2 = new_ok('Geo::Coder::Free::DB::MaxMind::admin2' => [logger => new_ok('MyLogger'), no_entry => 1]);
 
 	my $kent = $admin2->fetchrow_hashref({ concatenated_codes => 'GB.ENG.G5' });
 	ok($kent->{asciiname} eq 'Kent');

diff --git a/t/cities.t b/t/cities.t
@@ -16,7 +16,7 @@ BEGIN {
 CITIES: {
 	Geo::Coder::Free::DB::init(directory => 'lib/Geo/Coder/Free/MaxMind/databases');
 
-	my $cities = new_ok('Geo::Coder::Free::DB::MaxMind::cities' => [logger => new_ok('MyLogger')]);
+	my $cities = new_ok('Geo::Coder::Free::DB::MaxMind::cities' => [logger => new_ok('MyLogger'), no_entry => 1]);
 
 	# diag($cities->population(Country => 'gb', City => 'ramsgate'));
 	ok($cities->population(Country => 'gb', City => 'ramsgate') == 38624);