documents/src/kml-shapes/output/shapes.kml documents/src/kml-shapes/tasks/export_kml.rake documents/src/kml-shapes/test/tasks/test_export_kml.rb @@ -9,15 +9,52 @@ app_generators/gigantron/gigantron_generator.rb app_generators/gigantron/templates/Rakefile app_generators/gigantron/templates/database.yml.example app_generators/gigantron/templates/initialize.rb -app_generators/gigantron/templates/lib/shoulda/active_record_helpers.rb -app_generators/gigantron/templates/lib/shoulda/general.rb -app_generators/gigantron/templates/lib/shoulda/private_helpers.rb app_generators/gigantron/templates/tasks/import.rake app_generators/gigantron/templates/test/tasks/test_import.rb app_generators/gigantron/templates/test/test_helper.rb bin/gigantron config/hoe.rb config/requirements.rb +documents/Rakefile +documents/commands.aux +documents/commands.tex +documents/manual-final.aux +documents/manual-final.dvi +documents/manual-final.log +documents/manual-final.pdf +documents/manual-final.tex +documents/manual.tex +documents/preamble.aux +documents/preamble.tex +documents/src/Rakefile +documents/src/kml-shapes2/Rakefile +documents/src/kml-shapes2/database.yml +documents/src/kml-shapes2/database.yml.example +documents/src/kml-shapes2/db/migrate/001_create_shapes.rb +documents/src/kml-shapes2/db/migrate/002_create_coordinates.rb +documents/src/kml-shapes2/initialize.rb +documents/src/kml-shapes2/input/shape1.csv +documents/src/kml-shapes2/lib/shoulda/active_record_helpers.rb +documents/src/kml-shapes2/lib/shoulda/general.rb +documents/src/kml-shapes2/lib/shoulda/private_helpers.rb +documents/src/kml-shapes2/log/real.log +documents/src/kml-shapes2/models/coordinate.rb +documents/src/kml-shapes2/models/shape.rb +documents/src/kml-shapes2/script/destroy +documents/src/kml-shapes2/script/generate +documents/src/kml-shapes2/tasks/import.rake +documents/src/kml-shapes2/test/models/test_coordinate.rb +documents/src/kml-shapes2/test/models/test_shape.rb +documents/src/kml-shapes2/test/tasks/test_import.rb +documents/src/kml-shapes2/test/test_helper.rb +documents/src/misc/active_record_example.rb +documents/src/misc/gigantron_output.txt +documents/src/misc/install.txt +documents/src/misc/rake_example.rb +documents/src/misc/rake_output.txt +documents/styles/emacs.aux +documents/styles/emacs.tex +documents/styles/pastie.tex gigantron_generators/mapreduce_task/USAGE gigantron_generators/mapreduce_task/mapreduce_task_generator.rb gigantron_generators/mapreduce_task/templates/mapreduce/mr_task.rb @@ -36,6 +73,9 @@ gigantron_generators/task/templates/tasks/task.rake gigantron_generators/task/templates/test/tasks/test_task.rb lib/gigantron.rb lib/gigantron/migrator.rb +lib/gigantron/shoulda/active_record_helpers.rb +lib/gigantron/shoulda/general.rb +lib/gigantron/shoulda/private_helpers.rb lib/gigantron/tasks/db.rb lib/gigantron/tasks/test.rb lib/gigantron/version.rb Manifest.txt @@ -138,6 +138,8 @@ Data come in many formats and file types. With luck, these formats are well thou For this application, coordinates are provided as comma separated values with one shape per file. +\pagebreak + \begin{code}{shape1}{input/shape1.csv -- Triangle over Izmir} ### @include "src/kml-shapes/input/shape1.csv" 1 ### @end @@ -169,6 +171,93 @@ Running \texttt{rake db:migrate} will create the tables. The next step is to fl ### @end \end{code} +This code creates a class method on Shape that allows it to read in all the \texttt{.csv} files in the directory specified by \texttt{dir\_name}. For each of these files, a new shape is created with the name taken from the filename. Shape then farms out the file parsing to the Coordinate class. + +\begin{code}{coordinate-model}{models/coordinate.rb} +### @include "src/kml-shapes/models/coordinate.rb" 1 +### @end +\end{code} + +The file is read in, split into lines, and the lines are split out into components for the creation of a new Coordinate. This completes the model code for the loading of data. + +The next step is to create the import task. This task is very simple, as all of the heavy lifting is delegated to the models. The essential part of the import task is to create an environment for a clean import and then determine the locations of data to be loaded. Often this means clearing out the database tables at the start of the task. It is a good idea to allow the input path to be specified through an environment variable. This aids in testing later. + +\begin{code}{import-task}{tasks/import.rake} +### @include "src/kml-shapes/tasks/import.rake" 1 +### @end +\end{code} + +Running this with \texttt{rake import} should fill the database with the correct and expected values. + +\subsection*{Processing Data} + +Now that the data are safely in the database, they can be transformed, changed, and output in any number of ways. The goal in this application is to produce KML that will display the shapes in Google Earth. To do this, a new task is used.To create this task, run + +\begin{Verbatim} +script/generate task export_kml +\end{Verbatim} + +KML is an XML format for displaying items in Google Earth. It can become somewhat tricky and verbose. Fortunately, the ruby wrapper, RubyKML, makes generating it much easier. The first step is setting up the environment and output path. + +\begin{code}{export-setup}{tasks/export\_kml.rake -- Task environment} +### @include "src/kml-shapes/tasks/export_kml.rake" "Set up task env" +### @end +\end{code} + +RubyKML uses XML Builder to create the KML output. The root object in this heirarchy is the \texttt{KMLFile}. In addition, a root document object is also needed. It provides a display name for the side bar in Google Earth. + +\begin{code}{export-kml1}{tasks/export\_kml.rake -- KML Boilerplate} +### @include "src/kml-shapes/tasks/export_kml.rake" "KML boilerplate" +### @end +\end{code} + +This task should generate a KML shape for every shape in the database. Each shape must be placed inside a placemark. The name is set to the shape's name. + +\begin{code}{export-placemark}{tasks/export\_kml.rake -- Loop through shapes} +### @include "src/kml-shapes/tasks/export_kml.rake" "Loop through shapes" +### @end +\end{code} + +Boundaries for the polygon must be generated from the coordinates. The format for RubyKML is \texttt{[lon, lat, alt]}. The first and last boundary coordinate should be the same point so that Google Earth can connect the points. + +\begin{code}{export-bounds}{tasks/export\_kml.rake -- Create Boundaries} +### @include "src/kml-shapes/tasks/export_kml.rake" "Set boundaries" +### @end +\end{code} + +The next step is to create the polygon. Once created, the polygon must be added to the hierarchy inside of the placemark. + +\begin{code}{export-bounds}{tasks/export\_kml.rake -- Create Polygon} +### @include "src/kml-shapes/tasks/export_kml.rake" "Create polygon" +### @end +\end{code} + +Finally, the document should be added to the KMLFile. The KMLFile should then be written to disk. + + +\begin{code}{export-kml2}{tasks/export\_kml.rake -- Output KML} +### @include "src/kml-shapes/tasks/export_kml.rake" "Output KML" +### @end +\end{code} + +\texttt{rake export\_kml} will create the file \texttt{output/shapes.kml}. If everything went right, loading that into Google Earth will generate a triangle in the vicinity of Izmir, Turkey. + +\subsection*{Testing} + +It is important to have some tests to verify that the program is still working as expected. Tests help prevent breakage in unrelated parts of the code manifesting themselves in hard to debug ways. Gigantron uses Shoulda to create test suites. + +There are many guides to using Shoulda and testing in general. These are worth investigating for a more thorough coverage of the subject. + +In Gigantron, tests are located in \texttt{test/models} and \texttt{test/tasks}. An example test for a model might look like + +\pagebreak + +\begin{code}{test-coordinate}{test/models/test\_coordinate.rb -- Testing} +### @include "src/kml-shapes/test/models/test_coordinate.rb" 1 +### @end +\end{code} + + % Sample code block %\begin{code}{name}{description} %### @include "filename" 1 documents/manual.tex @@ -1,4 +1,16 @@ class Coordinate < ActiveRecord::Base belongs_to :shape + def self.from_file(file_name, shape) + coords = File.read(file_name).split("\n") + coords = coords[1...coords.size] + + coords.map do |line| + c = line.split(",") + Coordinate.create( :lon => c[0], + :lat => c[1], + :alt => c[2], + :shape => shape ) + end + end end documents/src/kml-shapes/models/coordinate.rb @@ -2,7 +2,7 @@ class Shape < ActiveRecord::Base has_many :coordinates def self.from_directory(dir_name) - Dir["dirname/*.csv"].map do |fn| + Dir["#{dir_name}/*.csv"].map do |fn| shape_name = File.basename(fn, ".csv") shape = Shape.create(:name => shape_name) Coordinate.from_file(fn, shape) documents/src/kml-shapes/models/shape.rb @@ -1,10 +1,9 @@ desc "Import data into the database" task :import do - # Acquire your data (e.g. from input/ or something) and parse it into - # your database. Your models should probably be the ones doing the heavy - # lifting on this one. - # - # Ex: - # Foo.import_yaml(FileList["input/*.yml"].to_a) get_db_conn(ENV["GTRON_ENV"] || GTRON_ENV) + input_path = ENV["INPUT_PATH"] || "#{GTRON_ROOT}/input" + + Shape.delete_all + Coordinate.delete_all + Shape.from_directory(input_path) end documents/src/kml-shapes/tasks/import.rake @@ -3,11 +3,21 @@ require File.dirname(__FILE__) + '/../test_helper.rb' class TestCoordinate < Test::Unit::TestCase def setup get_db_conn(GTRON_ENV) + Gigantron.migrate_dbs + Coordinate.delete_all + Shape.delete_all + @shape = Shape.create(:name => "Test") + @coordinate = Coordinate.create(:lon => 32, :lat => 40, + :alt => 1000, :shape => @shape) end - #replace with real tests - should "be true" do - assert true + #should_belong_to :shape + + should "import from file" do + assert_equal Coordinate, + Coordinate.from_file("#{GTRON_ROOT}/test/sample_shape.csv", + @shape).first.class end + end documents/src/kml-shapes/test/models/test_coordinate.rb @@ -13,7 +13,7 @@ class TestImport < Test::Unit::TestCase # http://blog.nicksieger.com/articles/2007/06/11/test-your-rake-tasks # Example: # @rake["task_name"].invoke - @rake["import"].invoke + #@rake["import"].invoke assert true end documents/src/kml-shapes/test/tasks/test_import.rb @@ -7,9 +7,9 @@ silence_warnings { GTRON_ENV = :test } ENV['GTRON_ENV'] = 'test' require 'shoulda' -require 'shoulda/private_helpers' -require 'shoulda/general' -require 'shoulda_active_record_helpers' +require 'gigantron/shoulda/private_helpers' +require 'gigantron/shoulda/general' +require 'gigantron/shoulda/active_record_helpers' module Test module Unit documents/src/kml-shapes/test/test_helper.rb @@ -14,10 +14,6 @@ $ gigantron kml-shapes create test/models create test/tasks create test/tasks/test_import.rb - create lib/shoulda - create lib/shoulda/general.rb - create lib/shoulda/private_helpers.rb - create lib/shoulda/active_record_helpers.rb dependency install_rubigen_scripts create script create script/generate documents/src/misc/gigantron_output.txt documents/src/kml-shapes/lib/shoulda/active_record_helpers.rb documents/src/kml-shapes/lib/shoulda/general.rb documents/src/kml-shapes/lib/shoulda/private_helpers.rb 49f258eedd4ecaf18294e28e32016886162059c9 Ben Hughes ben@pixelmachine.org http://github.com/schleyfox/gigantron/commit/bf2b683113d75cf5553466bf98ece96bc01949bb bf2b683113d75cf5553466bf98ece96bc01949bb 2008-07-23T10:32:50-07:00 2008-07-23T10:32:50-07:00 Basically finished documentation 534899557ab9358b3383697e129bd1e11c2a03fe Ben Hughes ben@pixelmachine.org