Skip to content
Browse files

hacking for visual output

  • Loading branch information...
1 parent 372f1f2 commit 0e13813babdb29e5e9294dc0b36be7887debbf54 Gareth Lloyd committed Feb 4, 2011
View
109 geo-vid-gen/src/main/java/com/heychinaski/historyhack/App.java
@@ -1,5 +1,8 @@
package com.heychinaski.historyhack;
+import java.awt.Component;
+import java.awt.Dimension;
+import java.awt.Graphics;
import java.awt.image.BufferedImage;
import java.awt.image.RenderedImage;
import java.io.File;
@@ -10,6 +13,7 @@
import java.util.Properties;
import javax.imageio.ImageIO;
+import javax.swing.JFrame;
import com.heychinaski.historyhack.displayobjects.StatefulBlobFactoryImpl;
import com.heychinaski.historyhack.model.GeoEventPage;
@@ -28,33 +32,34 @@
*/
public class App {
+ static StatefulBlobFactoryImpl factory;
static FrameRenderer<Object> backDropRenderer;
static FrameRenderer<List<GeoEventPage>> eventRenderer;
static FrameRenderer<Integer> yearRenderer;
static FrameRenderer<List<BufferedImage>> compositeRenderer;
static GeoEventPageProvider provider;
static String outputDirectory;
-
+ static int width;
+ static int height;
public static void main(String[] args) throws IOException {
Properties properties = new Properties();
properties.load(Thread.currentThread ().getContextClassLoader ().getResourceAsStream("config.properties"));
- int width = Integer.parseInt(properties.getProperty("width", "" + 1024));
- int height = Integer.parseInt(properties.getProperty("height", "" + 1024));
+ width = Integer.parseInt(properties.getProperty("width", "" + 1024));
+ height = Integer.parseInt(properties.getProperty("height", "" + 1024));
int frameSkip = Integer.parseInt(properties.getProperty("frameSkip", "" + 1));
String backgroundHex = properties.getProperty("backgroundColor", "0x000000");
- String inputLocation = properties.getProperty("inputLocation", "./input.json");
- String backDrop = properties.getProperty("backDrop", "./backdrop.png");
+ String inputLocation = properties.getProperty("inputLocation", "./target/data.json");
outputDirectory = properties.getProperty("outputDirectory", "./frames");
long time = System.currentTimeMillis();
backDropRenderer = new BackDropFrameRenderer(width, height, "/home/glloyd/projects/history_hackday/geo-vid-gen/target/backdrop.png");
provider = new JsonInputGEPProvider(inputLocation);
- eventRenderer = new GenerationalFrameRenderer(width, height,
- new StatefulBlobFactoryImpl());
+ factory = new StatefulBlobFactoryImpl(width, height);
+ eventRenderer = new GenerationalFrameRenderer(width, height, factory);
yearRenderer = new YearRenderer(width, height);
compositeRenderer = new CompositeFrameRenderer(width, height);
@@ -68,39 +73,109 @@ public static void main(String[] args) throws IOException {
frame++;
}
- render(pages, currentFrame);
+ renderToFile(pages, currentFrame);
currentFrame++;
}
// pad ending
for (int i = 0; i < 50; i++) {
List<GeoEventPage> empty = Collections.emptyList();
- render(empty, currentFrame++);
+ renderToFile(empty, currentFrame++);
}
long finishTime = System.currentTimeMillis();
System.out.println("Done");
System.out.println("Took " + (finishTime - time) + "ms");
}
- private static void render(List<GeoEventPage> pages, int currentFrame) throws IOException {
+ /**
+ * Process and return all the required overlays.
+ *
+ * @param pages
+ * @return
+ */
+ private static List<BufferedImage> getOverlays(List<GeoEventPage> pages) {
eventRenderer.renderNextFrame(pages);
+
int year = provider.getCurrentYear();
+ factory.renderNextFrame(year);
System.out.println("Current year " + year);
yearRenderer.renderNextFrame(year);
ArrayList<BufferedImage> overlays = new ArrayList<BufferedImage>();
- BufferedImage backDrop = backDropRenderer.getCurrentFrame();
- BufferedImage geoImage = eventRenderer.getCurrentFrame();
- BufferedImage yearImage = yearRenderer.getCurrentFrame();
- overlays.add(backDrop);
- overlays.add(geoImage);
- overlays.add(yearImage);
- compositeRenderer.renderNextFrame(overlays);
+ overlays.add(backDropRenderer.getCurrentFrame());
+ overlays.add(eventRenderer.getCurrentFrame());
+ overlays.add(factory.getCurrentFrame());
+ overlays.add(yearRenderer.getCurrentFrame());
+ return overlays;
+ }
+
+ /**
+ * Output successive frames to png files.
+ *
+ * @param pages
+ * @param currentFrame
+ * @throws IOException
+ */
+ private static void renderToFile(List<GeoEventPage> pages, int currentFrame) throws IOException {
+ compositeRenderer.renderNextFrame(getOverlays(pages));
+
RenderedImage image = compositeRenderer.getCurrentFrame();
String frameNumber = String.format("%05d", currentFrame);
ImageIO.write(image, "png", new File(outputDirectory + "/frame" + frameNumber + ".png"));
}
+
+
+ /**
+ * basically here for "how does it look" debugging,
+ * this method draws repeatedly to an ImageComponent
+ * in a JFrame.
+ */
+ private static JFrame jframe = null;
+ private static ImageComponent imageComponent = null;
+ private static void renderToScreen(List<GeoEventPage> pages, int currentFrame) {
+ if (jframe == null || imageComponent == null) {
+ imageComponent = new ImageComponent(width, height);
+ jframe = new JFrame();
+ jframe.add(imageComponent);
+ jframe.pack();
+ jframe.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+ jframe.setVisible(true);
+ }
+
+ List<BufferedImage> overlays = getOverlays(pages);
+ compositeRenderer.renderNextFrame(overlays);
+ imageComponent.setCurrentFrame(compositeRenderer.getCurrentFrame());
+ imageComponent.repaint();
+ }
+
+ /**
+ * Stupid component to support rendering to screen.
+ */
+ private static class ImageComponent extends Component {
+ private BufferedImage current;
+ int width;
+ int height;
+
+ public ImageComponent(int width, int height) {
+ this.width = width;
+ this.height = height;
+ }
+
+ @Override
+ public void paint(Graphics g) {
+ g.drawImage(current, 0, 0, null);
+ }
+
+ @Override
+ public Dimension getPreferredSize() {
+ return new Dimension(width, height);
+ }
+
+ public void setCurrentFrame(BufferedImage current) {
+ this.current = current;
+ }
+ }
}
View
106 geo-vid-gen/src/main/java/com/heychinaski/historyhack/DropletTest.java
@@ -1,106 +0,0 @@
-package com.heychinaski.historyhack;
-
-import java.awt.AlphaComposite;
-import java.awt.Color;
-import java.awt.Component;
-import java.awt.Dimension;
-import java.awt.Graphics;
-import java.awt.Graphics2D;
-import java.awt.Point;
-import java.awt.RenderingHints;
-import java.awt.image.BufferedImage;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.swing.JFrame;
-
-import com.heychinaski.historyhack.displayobjects.StatefulBlob;
-import com.heychinaski.historyhack.displayobjects.StatefulBlobFactoryImpl;
-import com.heychinaski.historyhack.displayobjects.StatefulBlobImpl;
-import com.heychinaski.historyhack.model.GeoEventPage;
-import com.heychinaski.historyhack.renderer.BackDropFrameRenderer;
-import com.heychinaski.historyhack.renderer.CompositeFrameRenderer;
-import com.heychinaski.historyhack.renderer.GenerationalFrameRenderer;
-
-/**
- * Quick class to allow me to test out various blob shapes
- * etc.
- *
- * @author glloyd
- *
- */
-public class DropletTest {
- public static final int WIDTH = 1600;
- public static final int HEIGHT = 800;
-
-
- public static void main(String[] args) {
- JFrame jframe = new JFrame();
- Compositer compositer = new Compositer();
- jframe.add(compositer);
- jframe.pack();
- jframe.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
- jframe.setVisible(true);
-
-
- StatefulBlob sb = new StatefulBlobImpl(new Point(0, 0), Color.WHITE);
- for (int i = 0; i < 1000; i++) {
- // frame delay
- try {Thread.sleep(500);} catch (InterruptedException e) {}
-
- Graphics2D g2d = null;
- if (sb.willDrawFinalState()) {
- g2d = compositer.getBackdrop().createGraphics();
- g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
- GraphicsUtils.scaleGraphics2D(g2d, WIDTH, HEIGHT);
- sb.draw(g2d);
- sb = new StatefulBlobImpl(new Point(random(), random()), Color.WHITE);
- g2d.dispose();
- }
- else {
- BufferedImage currentFrame = new BufferedImage(WIDTH, HEIGHT, BufferedImage.TYPE_4BYTE_ABGR);
- g2d = currentFrame.createGraphics();
- compositer.setCurrentFrame(currentFrame);
- g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
-
- GraphicsUtils.scaleGraphics2D(g2d, WIDTH, HEIGHT);
- sb.draw(g2d);
- g2d.dispose();
- }
- compositer.repaint();
- }
- System.exit(0);
- }
-
- private static int random() {
- Double d = Double.valueOf(Math.random() * 10000.0);
- return d.intValue();
- }
-
- static class Compositer extends Component {
- BufferedImage backdrop = new BackDropFrameRenderer(WIDTH, HEIGHT,
- "/home/glloyd/projects/history_hackday/geo-vid-gen/target/backdrop.png").getCurrentFrame();
- private BufferedImage current;
-
-
- @Override
- public void paint(Graphics g) {
- g.drawImage(backdrop, 0, 0, null);
- g.drawImage(current, 0, 0, null);
- }
-
- @Override
- public Dimension getPreferredSize() {
- return new Dimension(backdrop.getWidth(), backdrop.getHeight());
- }
-
- public void setCurrentFrame(BufferedImage current) {
- this.current = current;
- }
-
- public BufferedImage getBackdrop() {
- return backdrop;
- }
- }
-
-}
View
205 ...gen/src/main/java/com/heychinaski/historyhack/displayobjects/StatefulBlobFactoryImpl.java
@@ -1,30 +1,211 @@
package com.heychinaski.historyhack.displayobjects;
+import java.awt.AlphaComposite;
import java.awt.Color;
+import java.awt.Graphics2D;
import java.awt.Point;
+import java.awt.RadialGradientPaint;
+import java.awt.image.BufferedImage;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Random;
+import com.heychinaski.historyhack.GraphicsUtils;
+import static com.heychinaski.historyhack.GraphicsUtils.RESOLUTION_FACTOR;
import com.heychinaski.historyhack.model.GeoEventPage;
+import com.heychinaski.historyhack.renderer.FrameRenderer;
-public class StatefulBlobFactoryImpl implements StatefulBlobFactory {
-
- List<Color> colors = new ArrayList<Color>();
- static final int EXPECTED_LOWEST_YEAR = -499;
- static final int EXPECTED_HIGHEST_YEAR = 2011;
+public class StatefulBlobFactoryImpl implements StatefulBlobFactory, FrameRenderer<Integer> {
+ private static final int NUM = 25;
+ private static final int RADIAL_WIDTH = 40000;
+ private int width;
+ private int height;
+ private BufferedImage currentFrame;
+ private Random random = new Random();
- public StatefulBlobFactoryImpl() {
- colors.add(e);
+ private static List<Color> colors = new ArrayList<Color>();
+ static {
+ colors.add(Color.decode("0xffffff"));
+ colors.add(Color.decode("0xdcdb6e"));
+ colors.add(Color.decode("0xccefe2"));
+ colors.add(Color.decode("0xa0d9c3"));
+ colors.add(Color.decode("0x7ec1b1"));
+ colors.add(Color.decode("0x68a3ac"));
+ colors.add(Color.decode("0x415587"));
}
+ private List<GeoEventPage> thisEventCohort = new ArrayList<GeoEventPage>();
+
+ MovingWeightedAverage xPos = new MovingWeightedAverage(50);
+ MovingWeightedAverage yPos = new MovingWeightedAverage(50);
+ MovingWeightedAverage xSize = new MovingWeightedAverage(400);
+ /**
+ * Constructor. Set width and height for rendering purposes.
+ * @param width
+ * @param height
+ */
+ public StatefulBlobFactoryImpl(int width, int height) {
+ this.width = width;
+ this.height = height;
+ currentFrame = newFrame();
+ }
+
+ @Override
public StatefulBlob createStatefulBlob(GeoEventPage page, Point point) {
- Color color = getColorFromYear(page.getYear());
- return new StatefulBlobImpl(point, color);
+ thisEventCohort.add(page);
+ return new StatefulBlobImpl(point, colors.get(random.nextInt(colors.size())));
}
- private Color getColorFromYear(int year) {
- // TODO Auto-generated method stub
- return null;
+ @Override
+ public BufferedImage getCurrentFrame() {
+ return currentFrame;
}
+ @Override
+ public void renderNextFrame(Integer year) {
+ currentFrame = newFrame();
+ Graphics2D g2d = currentFrame.createGraphics();
+ GraphicsUtils.scaleGraphics2D(g2d, width, height);
+
+ // add new EventCohortStats at beginning of list
+ EventCohortStats eventCohortStats = new EventCohortStats(thisEventCohort);
+
+ xPos.addMeasure(eventCohortStats.averageLongitude);
+ yPos.addMeasure(eventCohortStats.averageLatitude);
+ xSize.addMeasure(eventCohortStats.horizSize);
+
+ int width = xSize.asInt() * RESOLUTION_FACTOR;
+ // hack to make it look right
+ if (width < 23000 || year < -487)
+ width = 23000;
+ // radial gradient should be outside circle
+ width += RADIAL_WIDTH * 2;
+
+ int xPosInt = ((xPos.asInt()) * RESOLUTION_FACTOR);
+ int yPosInt = -1 * (yPos.asInt() * RESOLUTION_FACTOR);
+
+ float[] fractions = getFractions(width);
+ Color[] radialColors = new Color[]{new Color(0, 0, 0, 0), new Color(0, 0, 0, 100)};
+ RadialGradientPaint rgp = new RadialGradientPaint(xPosInt, yPosInt, width, fractions, radialColors);
+
+ g2d.setPaint(rgp);
+ int ScaledWidth = currentFrame.getWidth() * RESOLUTION_FACTOR;
+ int ScaledHeight = currentFrame.getHeight() * RESOLUTION_FACTOR;
+ g2d.fillRect(-ScaledWidth / 2, -ScaledHeight / 2, ScaledWidth, ScaledHeight);
+
+ g2d.dispose();
+ }
+
+ // find the fractions to apply to the gradient taking into account
+ // the width in relation to the desired width of gradient
+ private float[] getFractions(int width) {
+ int radius = width / 2;
+ float proportion = radius / (float) (RADIAL_WIDTH + radius);
+ return new float[] {proportion, 1f};
+ }
+
+ private BufferedImage newFrame() {
+ BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_4BYTE_ABGR);
+ Graphics2D g2 = bi.createGraphics();
+
+ // Following block renders zero alpha to whole image
+ g2.setComposite(AlphaComposite.Clear);
+ g2.fillRect(0, 0, bi.getWidth(), bi.getHeight());
+ g2.dispose();
+ return bi;
+ }
+
+ /**
+ * Calculates a weighted average, with the highest
+ * weight given to the most recently added measure.
+ */
+ private class MovingWeightedAverage {
+ private LinkedList<Double> measures = new LinkedList<Double>();
+ final int maxMeasures;
+ private double avg = 0.0;
+
+ public MovingWeightedAverage(int maxMeasures) {
+ this.maxMeasures = maxMeasures;
+ }
+
+ public double addMeasure(double measure) {
+ measures.addFirst(measure);
+ int lim = Math.min(measures.size(), maxMeasures);
+ double numerator = 0.0;
+ double denominator = 0.0;
+ for (int i = 0; i < lim; i++) {
+ numerator += (lim - i) * measures.get(i);
+ denominator += (lim - i);
+ }
+ return avg = numerator / denominator;
+ }
+
+ public double get() {
+ return avg;
+ }
+
+ public int asInt() {
+ return (int) Math.round(avg);
+ }
+ }
+
+ /**
+ * calculates statistics for one cohort of
+ * {@link GeoEventPage}s,
+ */
+ private class EventCohortStats {
+ final double averageLatitude;
+ final double averageLongitude;
+ final double horizSize;
+ final double vertSize;
+
+ public EventCohortStats(List<GeoEventPage> cohort) {
+ int numEvents = cohort.size();
+ if (numEvents == 0) {
+ averageLatitude = averageLongitude = horizSize = vertSize = 0.0;
+ return;
+ }
+ double totalLat = 0.0, totalLong = 0.0;
+
+ List<Double> lats = new ArrayList<Double>();
+ List<Double> longs = new ArrayList<Double>();
+ for (GeoEventPage page : cohort) {
+ totalLat += page.getLatitude();
+ totalLong += page.getLongitude();
+ lats.add(page.getLatitude());
+ longs.add(page.getLongitude());
+ }
+ averageLatitude = totalLat / numEvents;
+ averageLongitude = totalLong / numEvents;
+
+ Collections.sort(longs);
+ Collections.sort(lats);
+ horizSize = percentile(longs, 80) - percentile(longs, 10);
+ vertSize = percentile(lats, 90) - percentile(lats, 10);
+ }
+
+ /**
+ * return the pth percentile of the inputs. Inputs
+ * must be sorted beforehand.
+ * @param longs
+ * @param p
+ * @return
+ */
+ private double percentile(List<Double> inputs, double p) {
+ int numElements = inputs.size();
+ if (p > 100.0 || p < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+ if (numElements == 0) {
+ return 0.0;
+ }
+ int i = (int) ((numElements / 100.0) * p);
+ if (i == numElements) {
+ i -= 1;
+ }
+ return inputs.get(i);
+ }
+ }
}
View
13 geo-vid-gen/src/main/java/com/heychinaski/historyhack/displayobjects/StatefulBlobImpl.java
@@ -11,8 +11,8 @@
public class StatefulBlobImpl implements StatefulBlob {
static final Color CENTRAL_CIRCLE_CLR = Color.decode("0xa0d9c3");
- static final int C_RADIUS = 800;
- static final float C_ALPHA = 0.2f;
+ static final int C_RADIUS = 1600;
+ static final float C_ALPHA = .05f;
public static final BlobState[] DEFAULT_STATES = {
new BlobState(0.8f, 800, Color.decode("0x68a3ac")),
@@ -53,14 +53,13 @@ public void draw(Graphics2D g2) {
g2.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
// draw central part
g2.setComposite(AlphaComposite.getInstance(AlphaComposite.SRC_OVER, C_ALPHA));
- g2.setColor(state.getColor());
- g2.fillOval(point.x - C_RADIUS, point.y - C_RADIUS, C_RADIUS * 2, C_RADIUS * 2);
-
+ g2.setColor(color);
+ for (int r = 200; r < C_RADIUS; r += 400) {
+ g2.fillOval(point.x - r, point.y - r, r * 2, r * 2);
+ }
// draw ring
if (!willDrawFinalState()) {
int radius = state.getRadius();
-// int radius = state.getRadius() + 400;
-
g2.setStroke(new BasicStroke(500));
g2.setComposite(AlphaComposite.getInstance(AlphaComposite.SRC_OVER, state.getAlpha()));
g2.drawOval(point.x - radius, point.y - radius, radius * 2, radius * 2);
View
12 geo-vid-gen/src/main/java/com/heychinaski/historyhack/renderer/BackDropFrameRenderer.java
@@ -1,5 +1,7 @@
package com.heychinaski.historyhack.renderer;
+import java.awt.Color;
+import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.awt.image.ColorConvertOp;
import java.io.File;
@@ -18,10 +20,12 @@ public BackDropFrameRenderer(int width, int height, String backDropName) {
try {
URL imageSrc = new File(backDropName).toURI().toURL();
backdrop = ImageIO.read(imageSrc);
- } catch (MalformedURLException e) {
- throw new RuntimeException("Bad image location: " + backDropName);
- } catch (IOException e) {
- throw new RuntimeException("Bad image location: " + backDropName);
+ } catch (Exception e) {
+ backdrop = new BufferedImage(width, height, BufferedImage.TYPE_4BYTE_ABGR);
+ Graphics2D g2d = backdrop.createGraphics();
+ g2d.setColor(Color.decode("0x3e133d"));
+ g2d.drawRect(0, 0, width, height);
+ g2d.dispose();
}
}
View
1 geo-vid-gen/src/main/resources/config.properties
@@ -17,3 +17,4 @@ outputDirectory= ./frames
drawHalos=false
drawDebug=false
+backdrop=
View
25 wikipedia_processor/output_csv.py
@@ -1,10 +1,11 @@
+import sys
from pymongo import ASCENDING, DESCENDING
from events_processor import DataSource
d = DataSource()
JSON_FMT = '{"title" : "%s", "latitude": %s, "longitude": %s, "year" : %s, "month": %s, "day": %s, "eventText": "%s"}'
CSV_FMT = '"%s","%s %s","%s","%s","%s","%s"'
-fmt = JSON_FMT
+fmt = CSV_FMT
# apply a bounding box to lats and longs
MIN_LAT = -90
@@ -13,21 +14,13 @@
MAX_LONG = 180
# drop and recreate the reference counts
-#d.db.drop_collection("ref_counts")
refCountCollection = d.db.ref_counts
+if refCountCollection.count() == 0:
+ print "ERROR: collection ref_counts has not been populated"
+ sys.exit()
-## count references to pages
-#for event in d.eventCollection.find():
-# for link in event['links']:
-# p = d.pageCollection.find_one({"_id" : link})
-# if p:
-# ref = refCountCollection.find_one({"_id" : p["_id"]})
-# if not ref:
-# refCountCollection.insert({"_id" : p["_id"], "refs": 1})
-# else:
-# refCountCollection.update({'_id': p["_id"]}, {"$inc" : { "refs": 1 }})
-
-print refCountCollection.count()
+if fmt == CSV_FMT:
+ print '"title","location","year","month","day","event"'
# output location-linked events. Prefer less-linked pages
for event in d.eventCollection.find().sort([("year", ASCENDING), ("month", ASCENDING), ("day", ASCENDING)]):
@@ -40,11 +33,11 @@
if ref["refs"] < leastRefs:
pageToLink = p
if pageToLink:
- eventText = event["event_text"].replace(',', '').encode('utf-8')
+ eventText = event["event_text"].replace('"', '').encode('utf-8')
title = pageToLink["_id"].replace(',', '').encode('utf-8')
coord = pageToLink["coords"][0]
latitude, longitude = coord["latitude"], coord["longitude"]
month = event["month"] if event["month"] else 0
day = event["day"] if event["day"] else 0
if (MIN_LAT <= latitude <= MAX_LAT) and (MIN_LONG <= longitude <= MAX_LONG):
- print fmt % (title, latitude, longitude, event["year"], month, day, eventText)
+ print fmt % (title, latitude, longitude, event["year"], month, day, eventText)
View
19 wikipedia_processor/output_invalid.py
@@ -0,0 +1,19 @@
+import sys
+import urllib
+from pymongo import ASCENDING, DESCENDING
+from events_processor import DataSource
+d = DataSource()
+
+JSON_FMT = '{"title" : "%s", "invalid_coord": %s}'
+CSV_FMT = '"%s","%s"'
+fmt = CSV_FMT
+
+
+if fmt == CSV_FMT:
+ print '"page","invalid string"'
+
+# output location-linked events. Prefer less-linked pages
+for page in d.invalidCoordCollection.find().sort("_id", ASCENDING):
+ title = "http://en.wikipedia.org/wiki/%s" % urllib.quote(page['_id'].encode('utf_8'))
+ invalid = page['coord_string'].encode('utf_8').replace('"', '')
+ print fmt % (title, invalid)
View
14 wikipedia_processor/output_pages_csv.py
@@ -0,0 +1,14 @@
+import sys
+import urllib
+from pymongo import ASCENDING, DESCENDING
+from events_processor import DataSource
+d = DataSource()
+
+JSON_FMT = '{"title" : "%s", "latitude": %s, "longitude": %s}'
+CSV_FMT = '"%s","%s %s"'
+fmt = CSV_FMT
+
+# output location-linked events. Prefer less-linked pages
+for page in d.pageCollection.find().sort("_id", ASCENDING):
+ title = "http://en.wikipedia.org/wiki/%s" % urllib.quote(page['_id'].encode('utf_8'))
+ print fmt % (title, page['coords'][0]['latitude'], page['coords'][0]['longitude'])
View
21 wikipedia_processor/ref_counts.py
@@ -0,0 +1,21 @@
+from pymongo import ASCENDING, DESCENDING
+from events_processor import DataSource
+d = DataSource()
+
+# drop and recreate the reference counts
+#d.db.drop_collection("ref_counts")
+refCountCollection = d.db.ref_counts
+
+# count references to pages
+#for event in d.eventCollection.find():
+# for link in event['links']:
+# p = d.pageCollection.find_one({"_id" : link})
+# if p:
+# ref = refCountCollection.find_one({"_id" : p["_id"]})
+# if not ref:
+# refCountCollection.insert({"_id" : p["_id"], "refs": 1})
+# else:
+# refCountCollection.update({'_id': p["_id"]}, {"$inc" : { "refs": 1 }})
+
+for ref in refCountCollection.find().sort("refs", DESCENDING):
+ print ref["refs"], ref["_id"]

0 comments on commit 0e13813

Please sign in to comment.
Something went wrong with that request. Please try again.