Skip to content

Commit

Permalink
added automated location detection based on city database with more than
Browse files Browse the repository at this point in the history 
1000 people
  • Loading branch information
@Orbiter
Orbiter committed Jun 17, 2015
1 parent ae9aa50 commit 4501c7b
Show file tree
Hide file tree
Showing 7 changed files with 325 additions and 229 deletions.
18 changes: 18 additions & 0 deletions src/org/loklak/api/client/ClientConnection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@

package org.loklak.api.client;

import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.List;
Expand Down Expand Up @@ -126,4 +130,18 @@ public void close() {
try {this.con.disconnect();} catch (Throwable e) {}
}

public static void download(String source_url, File target_file) {
byte[] buffer = new byte[2048];
try {
ClientConnection connection = new ClientConnection(source_url);
OutputStream os = new BufferedOutputStream(new FileOutputStream(target_file));
int count;
try {
while ((count = connection.inputStream.read(buffer)) > 0) os.write(buffer, 0, count);
} catch (IOException e) {}
connection.close();
os.close();
} catch (IOException e) {
}
}
}
23 changes: 22 additions & 1 deletion src/org/loklak/data/DAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,10 @@
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.sort.SortOrder;
import org.loklak.Caretaker;
import org.loklak.api.client.ClientConnection;
import org.loklak.api.client.SearchClient;
import org.loklak.geo.GeoJsonReader;
import org.loklak.geo.GeoNames;
import org.loklak.geo.LocationSource;
import org.loklak.geo.GeoJsonReader.Feature;
import org.loklak.harvester.SourceType;
Expand Down Expand Up @@ -110,7 +112,7 @@ public class DAO {
public final static int CACHE_MAXSIZE = 10000;

public static File conf_dir;
private static File external_data, geoJson_import, geoJson_imported, assets;
private static File external_data, geoJson_import, geoJson_imported, assets, dictionaries;
private static File message_dump_dir, message_dump_dir_own, message_dump_dir_import, message_dump_dir_imported;
private static File settings_dir, customized_config;
private static RandomAccessFile messagelog, accountlog;
Expand All @@ -122,6 +124,7 @@ public class DAO {
private static QueryFactory queries;
private static BlockingQueue<Timeline> newMessageTimelines = new LinkedBlockingQueue<Timeline>();
private static Map<String, String> config = new HashMap<>();
public static GeoNames geoNames;

/**
* initialize the DAO
Expand All @@ -136,6 +139,24 @@ public static void init(File datadir) {
geoJson_imported = new File(new File(external_data, "geojson"), "imported");
geoJson_import.mkdirs();
geoJson_imported.mkdirs();
dictionaries = new File(external_data, "dictionaries");
dictionaries.mkdirs();

// load dictionaries if they are embedded here
// read the file allCountries.zip from http://download.geonames.org/export/dump/allCountries.zip
//File allCountries = new File(dictionaries, "allCountries.zip");
File cities1000 = new File(dictionaries, "cities1000.zip");
if (!cities1000.exists()) {
// download this file
ClientConnection.download("http://download.geonames.org/export/dump/cities1000.zip", cities1000);
}
if (cities1000.exists()) {
geoNames = new GeoNames(cities1000, -1);
} else {
geoNames = null;
}

// create message dump dir
message_dump_dir = new File(datadir, "dump");
message_dump_dir_own = new File(message_dump_dir, "own");
message_dump_dir_import = new File(message_dump_dir, "import");
Expand Down
24 changes: 21 additions & 3 deletions src/org/loklak/data/MessageEntry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.loklak.geo.GeoLocation;
import org.loklak.geo.LocationSource;
import org.loklak.harvester.SourceType;

Expand Down Expand Up @@ -388,6 +389,24 @@ public void enrich() {
if (link.endsWith(".mp3")) {this.audio.add(link); continue;}
if (link.indexOf("soundcloud.com") > 0) {this.audio.add(link); continue;}
}

// find location
if ((this.location_point == null || this.location_point.length == 0) && DAO.geoNames != null) {
GeoLocation loc = null;
if (this.place_name != null && this.place_name.length() > 0) {
loc = DAO.geoNames.analyse(this.place_name, 5);
}
if (loc == null) {
loc = DAO.geoNames.analyse(this.text, 5);
}
if (loc != null) {
this.place_name = loc.getName();
this.location_radius = 0;
this.location_point = new double[]{loc.lon(), loc.lat()}; //[longitude, latitude]
this.location_mark = new double[]{loc.lon(), loc.lat()}; //[longitude, latitude]
this.location_source = LocationSource.ANNOTATION;
}
}
}

private static List<String> extract(StringBuilder s, Pattern p, int g) {
Expand Down Expand Up @@ -432,9 +451,8 @@ public void toJSON(JsonGenerator json, UserEntry user, boolean calculatedData) {
json.writeObjectField("place_id", this.place_id);

// add optional location data. This is written even if calculatedData == false if the source is from REPORT to prevent that it is lost
if ((calculatedData || this.location_source == LocationSource.REPORT) &&
this.location_point != null && this.location_mark != null) {
writeArray(json, "location_point", this.location_point);
if (calculatedData || (this.location_point != null && this.location_point.length == 2 && this.location_mark != null && this.location_mark.length == 2)) {
writeArray(json, "location_point", this.location_point); // [longitude, latitude]
json.writeObjectField("location_radius", this.location_radius);
writeArray(json, "location_mark", this.location_mark);
json.writeObjectField("location_source", this.location_source.name());
Expand Down
13 changes: 5 additions & 8 deletions src/org/loklak/geo/GeoLocation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,9 @@

import java.util.Comparator;

import org.loklak.tools.UTF8;

public class GeoLocation extends IntegerGeoPoint implements Comparable<GeoLocation>, Comparator<GeoLocation> {

private byte[] name;
private String name;
private int population;

public GeoLocation(double lat, double lon) {
Expand All @@ -39,15 +37,15 @@ public GeoLocation(double lat, double lon) {

public GeoLocation(double lat, double lon, String name) {
super(lat, lon);
this.name = UTF8.getBytes(name);
this.name = name;
}

public void setName(String name) {
this.name = UTF8.getBytes(name);
this.name = name;
}

public String getName() {
return UTF8.String(this.name);
return this.name;
}

public void setPopulation(int population) {
Expand All @@ -61,8 +59,7 @@ public int getPopulation() {
@Override
public boolean equals(Object loc) {
if (!(loc instanceof GeoLocation)) return false;
if (this.name == null || ((GeoLocation) loc).name == null) return super.equals(loc);
return super.equals(loc) && this.getName().toLowerCase().equals(((GeoLocation) loc).getName().toLowerCase());
return super.equals(loc);
}

/**
Expand Down

1 comment on commit 4501c7b

@Orbiter
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this implements a suggestion from https://gist.github.com/sudheesh001/ee26ba3cf8bf07425d54

Please sign in to comment.