Permalink
Browse files

Move to deprecated branches dir

  • Loading branch information...
1 parent 24426a5 commit 037b14cef43b5b5564cdb6c8d49cd94704ac4d31 phlapjack committed May 3, 2008
Showing with 10,890 additions and 0 deletions.
  1. +118 −0 ts_manager/History.txt
  2. +20 −0 ts_manager/License.txt
  3. +489 −0 ts_manager/Manifest.txt
  4. +186 −0 ts_manager/README.txt
  5. +4 −0 ts_manager/Rakefile
  6. +5 −0 ts_manager/app_generators/skynet_install/USAGE
  7. +95 −0 ts_manager/app_generators/skynet_install/skynet_install_generator.rb
  8. +66 −0 ts_manager/app_generators/skynet_install/templates/migration.rb
  9. +41 −0 ts_manager/app_generators/skynet_install/templates/skynet
  10. +9 −0 ts_manager/app_generators/skynet_install/templates/skynet_initializer.rb
  11. +56 −0 ts_manager/app_generators/skynet_install/templates/skynet_mysql_schema.sql
  12. +44 −0 ts_manager/bin/skynet
  13. +36 −0 ts_manager/bin/skynet_install
  14. +66 −0 ts_manager/bin/skynet_tuplespace_server
  15. +75 −0 ts_manager/config/hoe.rb
  16. +17 −0 ts_manager/config/requirements.rb
  17. +7 −0 ts_manager/extras/README
  18. +87 −0 ts_manager/extras/init.d/skynet
  19. +121 −0 ts_manager/extras/nagios/check_skynet.sh
  20. +41 −0 ts_manager/extras/rails/controllers/skynet_controller.rb
  21. +137 −0 ts_manager/extras/rails/views/skynet/index.rhtml
  22. +91 −0 ts_manager/lib/skynet.rb
  23. +74 −0 ts_manager/lib/skynet/mapreduce_helper.rb
  24. +56 −0 ts_manager/lib/skynet/mapreduce_test.rb
  25. +70 −0 ts_manager/lib/skynet/message_queue_adapters/message_queue_adapter.rb
  26. +502 −0 ts_manager/lib/skynet/message_queue_adapters/mysql.rb
  27. +324 −0 ts_manager/lib/skynet/message_queue_adapters/tuple_space.rb
  28. +280 −0 ts_manager/lib/skynet/skynet_active_record_extensions.rb
  29. +233 −0 ts_manager/lib/skynet/skynet_config.rb
  30. +50 −0 ts_manager/lib/skynet/skynet_console.rb
  31. +66 −0 ts_manager/lib/skynet/skynet_console_helper.rb
  32. +122 −0 ts_manager/lib/skynet/skynet_debugger.rb
  33. +68 −0 ts_manager/lib/skynet/skynet_guid_generator.rb
  34. +876 −0 ts_manager/lib/skynet/skynet_job.rb
  35. +27 −0 ts_manager/lib/skynet/skynet_launcher.rb
  36. +62 −0 ts_manager/lib/skynet/skynet_logger.rb
  37. +644 −0 ts_manager/lib/skynet/skynet_manager.rb
  38. +359 −0 ts_manager/lib/skynet/skynet_message.rb
  39. +132 −0 ts_manager/lib/skynet/skynet_message_queue.rb
  40. +93 −0 ts_manager/lib/skynet/skynet_partitioners.rb
  41. +53 −0 ts_manager/lib/skynet/skynet_ruby_extensions.rb
  42. +118 −0 ts_manager/lib/skynet/skynet_task.rb
  43. +80 −0 ts_manager/lib/skynet/skynet_tuplespace_server.rb
  44. +406 −0 ts_manager/lib/skynet/skynet_worker.rb
  45. +19 −0 ts_manager/lib/skynet/skynet_worker_queue.rb
  46. +9 −0 ts_manager/lib/skynet/version.rb
  47. +71 −0 ts_manager/lib/skynet/worker_queue_adapters/mysql.rb
  48. +49 −0 ts_manager/lib/skynet/worker_queue_adapters/tuple_space.rb
  49. 0 ts_manager/log/debug.log
  50. +29 −0 ts_manager/log/skynet.log
  51. +7 −0 ts_manager/log/skynet_tuplespace_server.log
  52. +1 −0 ts_manager/log/skynet_worker.pid
  53. +14 −0 ts_manager/script/destroy
  54. +14 −0 ts_manager/script/generate
  55. +74 −0 ts_manager/script/txt2html
  56. +1,585 −0 ts_manager/setup.rb
  57. +34 −0 ts_manager/tasks/deployment.rake
  58. +7 −0 ts_manager/tasks/environment.rake
  59. +17 −0 ts_manager/tasks/website.rake
  60. +138 −0 ts_manager/test/test_active_record_extensions.rb
  61. +20 −0 ts_manager/test/test_generator_helper.rb
  62. +10 −0 ts_manager/test/test_helper.rb
  63. +263 −0 ts_manager/test/test_mysql_message_queue_adapter.rb
  64. +19 −0 ts_manager/test/test_skynet.rb
  65. +49 −0 ts_manager/test/test_skynet_install_generator.rb
  66. +717 −0 ts_manager/test/test_skynet_job.rb
  67. +157 −0 ts_manager/test/test_skynet_manager.rb
  68. +229 −0 ts_manager/test/test_skynet_message.rb
  69. +24 −0 ts_manager/test/test_skynet_task.rb
  70. +174 −0 ts_manager/test/test_tuplespace_message_queue.rb
  71. +124 −0 ts_manager/website/index.html
  72. +59 −0 ts_manager/website/index.txt
  73. +285 −0 ts_manager/website/javascripts/rounded_corners_lite.inc.js
  74. +138 −0 ts_manager/website/stylesheets/screen.css
  75. +48 −0 ts_manager/website/template.rhtml
View
118 ts_manager/History.txt
@@ -0,0 +1,118 @@
+== 0.9.3 2008-04-10
+ - Support starting Skynet with ./script/skynet start and stop to daemonize
+ - Close file handles on exec.
+ Skynet::Worker and Skynet::Manager now call Skynet.fork_and_exec instead of their own versions.
+ Skynet.fork_and_exec prevents file descriptor exhaustion by calling Skynet.close_file_handles.
+ Skynet::Manager detatches from console by calling Skynet.close_console
+ - Added printlog logging method which always prints to the log as [LOG]
+ - Deprecated Skynet.new to Skynet.start
+ - Mysql Message Queue Adapter - Make delete_expired_messages much safer.
+ - ActiveRecord::Base.distributed_find - Patch submitted by Lourens Naude (lourens@methodmissing.com) which checks the model for the primary_key name as opposed to assuming it is
+ 'id'
+ - We don't want to use rails constantize so I've temporarily borrowed the method from ActiveSupport inflector and added it to skynet_ruby_extensions.
+ - Fix bug in Job comment where it referenced MapreduceTest instead of Skynet::MapreduceTest
+ - Fix tests. For some reason you still can't run ALL the test at once with rake test, but if the files are run individually they all pass.
+
+== 0.9.2 2008-01-22
+Highlights:
+ - Multiple Message Queues
+ - Many more Job options including options to control how jobs are distributed.
+ - The various options for how a job is run has been made much clearer.
+ - The Mysql Message Queue Adapter has been optimized and made more reliable.
+ - You can now control how many times skynet retries failed master, map and reduce tasks.
+ - Large data sets can now be streamed to the queue.
+
+Details:
+
+Active::Record#distributed_find
+ - Active Record distributed_find now handles REALLY large sets by breaking them into seperate jobs. 1MM models per master broken into ranges of 1000.
+
+Skynet::Job
+ - code path through Skynet::Job is now clear. There are 3 ways to run Skynet::Job. Local Master (default), Remote Master, Async (implies remote master)
+ - Skynet::Job supports keep_map_tasks and keep_reduce_tasks settings.
+ If true, the master will run the tasks locally.
+ If a number is provided, the master will run the tasks locally if there are LESS THAN OR EQUAL TO the number provided
+ There are also Skynet::CONFIG settings for defaults. DEFAULT_KEEP_REDUCE_TASKS, DEFAULT_KEEP_MAP_TASKS
+ I can see there being a problem with the timeouts being a little off... Since your kinda in a master and kinda in a map or reduce timeout. The task timeouts will be correct at least. Though, you won't get the benefit of redoes yet.
+ - Skynet::Job now supports setting RETRY times per job by MASTER, MAP and REDUCE. So you can have a MASTER_RETRY=0, but have MAP_RETRY=2 and REDUCE_RETRY=3. There are now defaults for those as well :DEFAULT_MASTER_RETRY, :DEFAULT_MAP_RETRY, :DEFAULT_REDUCE_RETRY. If a message passes its RETRY it will be marked with an iteration of -1. delete_expired_messages removes those messages as well. These show up in the stats as :failed_tasks. Skynet::Task and Skynet::Message now have retry fields denoting the maximum number of retries.
+ - You can now pass queue_id or queue to Skynet::Job
+ - There is now a :MAX_RETRIES config setting that controls how many iterations Skynet will even look for tasks as well.
+ - You can now stream map_data to the queue by passing an Enumerable for your map_data
+ - Refactored Skynet::Job to be much cleaner and easier to test.
+ - Skynet::Job now has access to a local queue which it can treat almost like the real one.
+ - deprecate Skynet::Job#run_master
+ - rename reduce_partitioner method to just reduce_partition
+ - Skynet::Job has better support for running tasks locally. A job may run tasks in its own process if
+ you are running in solo mode, you've made a "single" job, or you set the keep_map_tasks or keep_reduce_tasks below.
+ When a job runs tasks locally it now honors the retry settings and timeouts.
+ - Skynet::Job and Skynet::AsyncJob are now almost identical. In fact you can just use Skynet::Job and tell it to run async.
+ - Skynet::Job Changed map_tasks and reduce_tasks to mappers and reducers respectively. This was to remove the ambiguity between the actual map/reduce tasks and the number of mappers/reducers desired.
+ - Skynet::Jobs can not be told what queue to use for that job by passing :queue or :queue_id DEFAULT 0
+ - Skynet::Job won't call the reduce_partitioner if there are no valid results from the map_step.
+
+MapreduceHelper mixin
+ - You can include MapreduceHelper into your class and then implement self.map_each and self.reduce_each methods. The included self.map and self.reduce methods will handle iterating over the map_data and reduce_data, passing each element to your map_each and reduce_each methods respectively. They will also handle error handling within that loop to make sure even if a single map or reduce fails, processing will continue. If you do not want processing to continue if a map fails, do not use the MapreduceHelper mixin.
+
+Multiple Message Queues!
+ - Add the ability to have multiple message queues in the same table message_queue_table.
+ - You can start skynet with a --queue_id or --queue option to determine which queue workers should look in.
+ - Skynet::Jobs can not be told what queue to use for that job by passing :queue or :queue_id. DEFAULT 0
+ - Queues can be configred via Skynet::CONFIG[:MESSAGE_QUEUES] = [] which comes with an array of queues id 1 through 10 named "one" through "ten"
+
+Skynet Console
+ - You now start the skynet console by running 'skynet console' at the command line. There is no longer a skynet_console app.
+ - The console now loads the configs that are in your local skynet script.
+
+Skynet::Config
+ - Added Skynet.silent {} Runs your code with no debugging output.
+ - Made sure all config options for TupleSpace? adapter begin with TS and all Mysql adapter CONFIG settings start with MYSQL.
+ - There is now a :MAX_RETRIES config setting that controls how many iterations Skynet will even look for tasks as well.
+
+Skynet::Partitioners
+ - Created a Skynet::Partitioners class where various partitioners can be found. To specify one of them merely provide that specific Skynet::Partitioners subclass as your reduce_partitioner in your Skynet::Job
+
+Skynet Install
+ - skynet_install now has a --mysql option. This installs the migration as well as a skynet_schema.sql file.
+
+Mysql Message Queue Adapter
+ - You can now configure your database options outside of rails with Skynet::CONFIG[:MYSQL_*] options.
+ - Mysql adapter now updates the updated_on time of rows in skynet_message_queues
+ - Fixed Mysql Message Adaptor to take_next_task safer and more efficiently. There seems to be far less risk of a race condition where two workers would take the same task.
+ - Eliminated 1 db update per every task taken making it MUCH more efficient.
+ - Skynet::MessageQueueAdaptor::Mysql now tries to reconnect if it gets disconnected. This was to solve the "Mysql Server has Gone Away" errors.
+ - Implemented version_active? in mysql message queue adapter. It's a way for workers to check to see if a version is still in the queue.
+
+Skynet::Task
+ - Skynet::Task#master_task takes care of creating the master job and task now. I have mixed feelings about this.
+ - ENFORCED TIMEOUTS - Even though a master might give up on a worker if it didn't respond in time, there was nothing to step any given worker from running forever. We now enforce the timeouts (master_timeout, map_timeout, reduce_timeout) given in Skynet::Job using the Timeout module. This causes a Timeout::Error to be thrown. If you are using the mysql adapter, this can cause strange results sometimes. If the Timeout error is thrown during a DB query, ActiveRecord will throw an ActiveRecord::StatementInvalid exception which includes the Timeout::Error exception in it. Not sure how to prevent that from happening.
+
+Skynet::Worker
+ - Workers now have a Skynet::CONFIG[:WORKER_MAX_PROCESSED] setting to control when to respawn based on how many that worker has processed.
+ - You can start skynet with a --queue_id or --queue option to determine which queue workers should look in.
+ - Workers do not restart until there are no more items in the queue of that version.
+
+Skynet::Message
+ - Skynet::Message now stores fields as an array. It is far more efficient now as well.
+
+Now 90% more Tests!
+
+BUGFIXES
+- Skynet Workers now restart properly when the worker_version changes.
+- starting tuplespce_server, you no longer need to provide the --port if you're already providing the drburi-
+- Fixed a bug where Skynet::MessageQueueAdapter::Mysql would sometimes pick up tasks another worker had already picked up.
+- Fix bug in Skynet::Worker where it wouldn't die right if the max processed was reached.
+- Workers were supposed to restart when the worker_version changed. They do that properly now.
+
+Thanks to Jason Rimmer for finding these bugs.
+- Fix bug in Skynet::Message where it would calculate the iteration improperly.
+- The skynet gem appears to be missing the Rubigen dependency
+- Running skynet_install even without the rails arg still generates code with rails dependencies and tailings: RAILS_ROOT, RAILS_ENV, and the various directory tailings such as 'db/migrate', etc.
+- Generated skynet script is missing "require 'rubygems'"
+- Specification of pid directory and file is incorrect as 'skynet_manager.rb' wants only a directory with it specifying the file
+- The sleep while waiting to start the queue server isn't long enough. There is now a CONFIG setting TS_SERVER_START_DELAY.
+
+
+== 0.0.1 2007-12-16
+
+* 1 major enhancement:
+ * Initial release
View
20 ts_manager/License.txt
@@ -0,0 +1,20 @@
+Copyright (c) 2007 Adam Pisoni, Geni.com
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
View
489 ts_manager/Manifest.txt
@@ -0,0 +1,489 @@
+History.txt
+License.txt
+Manifest.txt
+README.txt
+Rakefile
+app_generators/skynet_install/USAGE
+app_generators/skynet_install/skynet_install_generator.rb
+app_generators/skynet_install/templates/migration.rb
+app_generators/skynet_install/templates/skynet
+app_generators/skynet_install/templates/skynet_initializer.rb
+app_generators/skynet_install/templates/skynet_mysql_schema.sql
+bin/skynet
+bin/skynet_install
+bin/skynet_tuplespace_server
+config/hoe.rb
+config/requirements.rb
+extras/README
+extras/init.d/skynet
+extras/nagios/check_skynet.sh
+extras/rails/controllers/skynet_controller.rb
+extras/rails/views/skynet/index.rhtml
+lib/skynet.rb
+lib/skynet/mapreduce_helper.rb
+lib/skynet/mapreduce_test.rb
+lib/skynet/message_queue_adapters/message_queue_adapter.rb
+lib/skynet/message_queue_adapters/mysql.rb
+lib/skynet/message_queue_adapters/tuple_space.rb
+lib/skynet/skynet_active_record_extensions.rb
+lib/skynet/skynet_config.rb
+lib/skynet/skynet_console.rb
+lib/skynet/skynet_console_helper.rb
+lib/skynet/skynet_debugger.rb
+lib/skynet/skynet_guid_generator.rb
+lib/skynet/skynet_job.rb
+lib/skynet/skynet_launcher.rb
+lib/skynet/skynet_logger.rb
+lib/skynet/skynet_manager.rb
+lib/skynet/skynet_message.rb
+lib/skynet/skynet_message_queue.rb
+lib/skynet/skynet_partitioners.rb
+lib/skynet/skynet_ruby_extensions.rb
+lib/skynet/skynet_task.rb
+lib/skynet/skynet_tuplespace_server.rb
+lib/skynet/skynet_worker.rb
+lib/skynet/skynet_worker_queue.rb
+lib/skynet/version.rb
+lib/skynet/worker_queue_adapters/mysql.rb
+lib/skynet/worker_queue_adapters/tuple_space.rb
+log/debug.log
+log/skynet.log
+log/skynet_tuplespace_server.log
+log/skynet_worker.pid
+script/destroy
+script/generate
+script/txt2html
+setup.rb
+tasks/deployment.rake
+tasks/environment.rake
+tasks/website.rake
+test/test_active_record_extensions.rb
+test/test_generator_helper.rb
+test/test_helper.rb
+test/test_mysql_message_queue_adapter.rb
+test/test_skynet.rb
+test/test_skynet_install_generator.rb
+test/test_skynet_job.rb
+test/test_skynet_manager.rb
+test/test_skynet_message.rb
+test/test_skynet_task.rb
+test/test_tuplespace_message_queue.rb
+tmtags
+website/doc/classes/ActiveRecord.html
+website/doc/classes/ActiveRecord/Base.html
+website/doc/classes/ActiveRecord/Base.src/M000307.html
+website/doc/classes/ActiveRecord/Mapreduce.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000293.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000294.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000295.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000296.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000297.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000298.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000299.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000300.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000301.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000303.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000304.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000305.html
+website/doc/classes/ActiveRecord/Mapreduce.src/M000306.html
+website/doc/classes/Enumerable.html
+website/doc/classes/Enumerable.src/M000308.html
+website/doc/classes/ExceptionReport.html
+website/doc/classes/ExceptionReport.src/M000001.html
+website/doc/classes/ExceptionReport.src/M000002.html
+website/doc/classes/MapreduceHelper.html
+website/doc/classes/MapreduceHelper.src/M000312.html
+website/doc/classes/MapreduceHelper.src/M000313.html
+website/doc/classes/MapreduceHelper.src/M000314.html
+website/doc/classes/Rinda.html
+website/doc/classes/Rinda/Tuple.html
+website/doc/classes/Rinda/Tuple.src/M000309.html
+website/doc/classes/Rinda/TupleSpaceProxy.html
+website/doc/classes/Rinda/TupleSpaceProxy.src/M000310.html
+website/doc/classes/Rinda/TupleSpaceProxy.src/M000311.html
+website/doc/classes/Skynet/AbstractClassError.html
+website/doc/classes/Skynet/ActiveRecordAsync.html
+website/doc/classes/Skynet/ActiveRecordAsync.src/M000101.html
+website/doc/classes/Skynet/AsyncJob.html
+website/doc/classes/Skynet/AsyncJob.src/M000070.html
+website/doc/classes/Skynet/AsyncJob.src/M000071.html
+website/doc/classes/Skynet/AsyncJob.src/M000072.html
+website/doc/classes/Skynet/Config.html
+website/doc/classes/Skynet/Config.src/M000073.html
+website/doc/classes/Skynet/Config.src/M000074.html
+website/doc/classes/Skynet/Config.src/M000075.html
+website/doc/classes/Skynet/Config.src/M000076.html
+website/doc/classes/Skynet/Config.src/M000077.html
+website/doc/classes/Skynet/ConnectionError.html
+website/doc/classes/Skynet/Console.html
+website/doc/classes/Skynet/Console.src/M000100.html
+website/doc/classes/Skynet/ConsoleHelper.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000011.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000012.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000013.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000014.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000015.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000016.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000017.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000018.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000019.html
+website/doc/classes/Skynet/ConsoleHelper.src/M000020.html
+website/doc/classes/Skynet/Error.html
+website/doc/classes/Skynet/GuidGenerator.html
+website/doc/classes/Skynet/GuidGenerator.src/M000004.html
+website/doc/classes/Skynet/InvalidMessage.html
+website/doc/classes/Skynet/Job.html
+website/doc/classes/Skynet/Job.src/M000208.html
+website/doc/classes/Skynet/Job.src/M000209.html
+website/doc/classes/Skynet/Job.src/M000210.html
+website/doc/classes/Skynet/Job.src/M000211.html
+website/doc/classes/Skynet/Job.src/M000212.html
+website/doc/classes/Skynet/Job.src/M000213.html
+website/doc/classes/Skynet/Job.src/M000214.html
+website/doc/classes/Skynet/Job.src/M000215.html
+website/doc/classes/Skynet/Job.src/M000216.html
+website/doc/classes/Skynet/Job.src/M000217.html
+website/doc/classes/Skynet/Job.src/M000218.html
+website/doc/classes/Skynet/Job.src/M000219.html
+website/doc/classes/Skynet/Job.src/M000220.html
+website/doc/classes/Skynet/Job.src/M000221.html
+website/doc/classes/Skynet/Job.src/M000222.html
+website/doc/classes/Skynet/Job.src/M000223.html
+website/doc/classes/Skynet/Job.src/M000224.html
+website/doc/classes/Skynet/Job.src/M000225.html
+website/doc/classes/Skynet/Job.src/M000226.html
+website/doc/classes/Skynet/Job.src/M000227.html
+website/doc/classes/Skynet/Job.src/M000228.html
+website/doc/classes/Skynet/Job.src/M000229.html
+website/doc/classes/Skynet/Job.src/M000230.html
+website/doc/classes/Skynet/Job.src/M000231.html
+website/doc/classes/Skynet/Job.src/M000232.html
+website/doc/classes/Skynet/Job.src/M000233.html
+website/doc/classes/Skynet/Job.src/M000234.html
+website/doc/classes/Skynet/Job.src/M000235.html
+website/doc/classes/Skynet/Job.src/M000236.html
+website/doc/classes/Skynet/Job.src/M000237.html
+website/doc/classes/Skynet/Job.src/M000238.html
+website/doc/classes/Skynet/Job.src/M000239.html
+website/doc/classes/Skynet/Job.src/M000240.html
+website/doc/classes/Skynet/Job.src/M000241.html
+website/doc/classes/Skynet/Job.src/M000242.html
+website/doc/classes/Skynet/Job.src/M000243.html
+website/doc/classes/Skynet/Job.src/M000244.html
+website/doc/classes/Skynet/Job.src/M000245.html
+website/doc/classes/Skynet/Job.src/M000246.html
+website/doc/classes/Skynet/Job.src/M000247.html
+website/doc/classes/Skynet/Job.src/M000248.html
+website/doc/classes/Skynet/Job.src/M000249.html
+website/doc/classes/Skynet/Job.src/M000250.html
+website/doc/classes/Skynet/Job.src/M000251.html
+website/doc/classes/Skynet/Job.src/M000252.html
+website/doc/classes/Skynet/Job.src/M000253.html
+website/doc/classes/Skynet/Job/BadMapOrReduceError.html
+website/doc/classes/Skynet/Job/Error.html
+website/doc/classes/Skynet/Job/LocalMessageQueue.html
+website/doc/classes/Skynet/Job/LocalMessageQueue.src/M000254.html
+website/doc/classes/Skynet/Job/LocalMessageQueue.src/M000255.html
+website/doc/classes/Skynet/Job/LocalMessageQueue.src/M000256.html
+website/doc/classes/Skynet/Job/LocalMessageQueue.src/M000257.html
+website/doc/classes/Skynet/Job/LocalMessageQueue.src/M000258.html
+website/doc/classes/Skynet/Job/LocalMessageQueue.src/M000259.html
+website/doc/classes/Skynet/Job/LocalMessageQueue.src/M000260.html
+website/doc/classes/Skynet/Job/LocalMessageQueue.src/M000261.html
+website/doc/classes/Skynet/Job/WorkerError.html
+website/doc/classes/Skynet/Loggable.html
+website/doc/classes/Skynet/Loggable.src/M000005.html
+website/doc/classes/Skynet/Loggable.src/M000006.html
+website/doc/classes/Skynet/Loggable.src/M000007.html
+website/doc/classes/Skynet/Loggable.src/M000008.html
+website/doc/classes/Skynet/Loggable.src/M000009.html
+website/doc/classes/Skynet/Loggable.src/M000010.html
+website/doc/classes/Skynet/Logger.html
+website/doc/classes/Skynet/Logger.src/M000269.html
+website/doc/classes/Skynet/Logger.src/M000270.html
+website/doc/classes/Skynet/Manager.html
+website/doc/classes/Skynet/Manager.src/M000168.html
+website/doc/classes/Skynet/Manager.src/M000169.html
+website/doc/classes/Skynet/Manager.src/M000170.html
+website/doc/classes/Skynet/Manager.src/M000171.html
+website/doc/classes/Skynet/Manager.src/M000172.html
+website/doc/classes/Skynet/Manager.src/M000173.html
+website/doc/classes/Skynet/Manager.src/M000174.html
+website/doc/classes/Skynet/Manager.src/M000175.html
+website/doc/classes/Skynet/Manager.src/M000176.html
+website/doc/classes/Skynet/Manager.src/M000177.html
+website/doc/classes/Skynet/Manager.src/M000178.html
+website/doc/classes/Skynet/Manager.src/M000179.html
+website/doc/classes/Skynet/Manager.src/M000180.html
+website/doc/classes/Skynet/Manager.src/M000181.html
+website/doc/classes/Skynet/Manager.src/M000182.html
+website/doc/classes/Skynet/Manager.src/M000183.html
+website/doc/classes/Skynet/Manager.src/M000184.html
+website/doc/classes/Skynet/Manager.src/M000185.html
+website/doc/classes/Skynet/Manager.src/M000186.html
+website/doc/classes/Skynet/Manager.src/M000187.html
+website/doc/classes/Skynet/Manager.src/M000188.html
+website/doc/classes/Skynet/Manager.src/M000189.html
+website/doc/classes/Skynet/Manager.src/M000190.html
+website/doc/classes/Skynet/Manager.src/M000191.html
+website/doc/classes/Skynet/Manager.src/M000192.html
+website/doc/classes/Skynet/Manager.src/M000193.html
+website/doc/classes/Skynet/Manager.src/M000194.html
+website/doc/classes/Skynet/Manager.src/M000195.html
+website/doc/classes/Skynet/Manager.src/M000196.html
+website/doc/classes/Skynet/Manager.src/M000197.html
+website/doc/classes/Skynet/Manager.src/M000198.html
+website/doc/classes/Skynet/Manager.src/M000199.html
+website/doc/classes/Skynet/Manager/Error.html
+website/doc/classes/Skynet/MapreduceTest.html
+website/doc/classes/Skynet/MapreduceTest.src/M000271.html
+website/doc/classes/Skynet/MapreduceTest.src/M000272.html
+website/doc/classes/Skynet/Message.html
+website/doc/classes/Skynet/Message.src/M000021.html
+website/doc/classes/Skynet/Message.src/M000022.html
+website/doc/classes/Skynet/Message.src/M000023.html
+website/doc/classes/Skynet/Message.src/M000024.html
+website/doc/classes/Skynet/Message.src/M000025.html
+website/doc/classes/Skynet/Message.src/M000026.html
+website/doc/classes/Skynet/Message.src/M000027.html
+website/doc/classes/Skynet/Message.src/M000028.html
+website/doc/classes/Skynet/Message.src/M000029.html
+website/doc/classes/Skynet/Message.src/M000030.html
+website/doc/classes/Skynet/Message.src/M000031.html
+website/doc/classes/Skynet/Message.src/M000032.html
+website/doc/classes/Skynet/Message.src/M000033.html
+website/doc/classes/Skynet/Message.src/M000034.html
+website/doc/classes/Skynet/Message.src/M000035.html
+website/doc/classes/Skynet/Message.src/M000036.html
+website/doc/classes/Skynet/Message.src/M000037.html
+website/doc/classes/Skynet/Message.src/M000038.html
+website/doc/classes/Skynet/Message.src/M000039.html
+website/doc/classes/Skynet/Message.src/M000040.html
+website/doc/classes/Skynet/Message.src/M000041.html
+website/doc/classes/Skynet/Message.src/M000042.html
+website/doc/classes/Skynet/Message.src/M000043.html
+website/doc/classes/Skynet/Message.src/M000044.html
+website/doc/classes/Skynet/Message.src/M000045.html
+website/doc/classes/Skynet/Message.src/M000046.html
+website/doc/classes/Skynet/Message.src/M000047.html
+website/doc/classes/Skynet/Message.src/M000048.html
+website/doc/classes/Skynet/Message.src/M000049.html
+website/doc/classes/Skynet/Message.src/M000050.html
+website/doc/classes/Skynet/Message/BadMessage.html
+website/doc/classes/Skynet/Message/Payload.html
+website/doc/classes/Skynet/MessageQueue.html
+website/doc/classes/Skynet/MessageQueue.src/M000078.html
+website/doc/classes/Skynet/MessageQueue.src/M000079.html
+website/doc/classes/Skynet/MessageQueue.src/M000080.html
+website/doc/classes/Skynet/MessageQueue.src/M000081.html
+website/doc/classes/Skynet/MessageQueue.src/M000082.html
+website/doc/classes/Skynet/MessageQueue.src/M000083.html
+website/doc/classes/Skynet/MessageQueue.src/M000084.html
+website/doc/classes/Skynet/MessageQueue.src/M000085.html
+website/doc/classes/Skynet/MessageQueue.src/M000086.html
+website/doc/classes/Skynet/MessageQueue.src/M000087.html
+website/doc/classes/Skynet/MessageQueue.src/M000088.html
+website/doc/classes/Skynet/MessageQueueAdapter.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000102.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000103.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000104.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000105.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000106.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000107.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000108.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000109.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000110.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000111.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000112.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000113.html
+website/doc/classes/Skynet/MessageQueueAdapter.src/M000114.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000134.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000135.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000136.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000137.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000138.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000139.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000140.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000141.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000142.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000143.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000144.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000145.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000146.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000147.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000148.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000149.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000150.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000151.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000152.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000153.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000154.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000155.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000156.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000157.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000158.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000159.html
+website/doc/classes/Skynet/MessageQueueAdapter/Mysql.src/M000160.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000115.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000116.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000117.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000118.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000119.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000120.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000121.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000122.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000123.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000124.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000125.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000126.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000127.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000128.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000129.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000130.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000131.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000132.html
+website/doc/classes/Skynet/MessageQueueAdapter/TupleSpace.src/M000133.html
+website/doc/classes/Skynet/Partitioners.html
+website/doc/classes/Skynet/Partitioners/ArrayDataSplitByFirstEntry.html
+website/doc/classes/Skynet/Partitioners/ArrayDataSplitByFirstEntry.src/M000163.html
+website/doc/classes/Skynet/Partitioners/RecombineAndSplit.html
+website/doc/classes/Skynet/Partitioners/RecombineAndSplit.src/M000162.html
+website/doc/classes/Skynet/Partitioners/SimplePartitionData.html
+website/doc/classes/Skynet/Partitioners/SimplePartitionData.src/M000161.html
+website/doc/classes/Skynet/ProfileCountTest.html
+website/doc/classes/Skynet/ProfileCountTest.src/M000204.html
+website/doc/classes/Skynet/ProfileCountTest.src/M000205.html
+website/doc/classes/Skynet/ProfileCountTest.src/M000206.html
+website/doc/classes/Skynet/QueueTimeout.html
+website/doc/classes/Skynet/RequestExpiredError.html
+website/doc/classes/Skynet/Server.html
+website/doc/classes/Skynet/Server.src/M000207.html
+website/doc/classes/Skynet/Task.html
+website/doc/classes/Skynet/Task.src/M000089.html
+website/doc/classes/Skynet/Task.src/M000090.html
+website/doc/classes/Skynet/Task.src/M000091.html
+website/doc/classes/Skynet/Task.src/M000092.html
+website/doc/classes/Skynet/Task.src/M000093.html
+website/doc/classes/Skynet/Task.src/M000094.html
+website/doc/classes/Skynet/Task.src/M000095.html
+website/doc/classes/Skynet/Task.src/M000096.html
+website/doc/classes/Skynet/Task/ConstructorError.html
+website/doc/classes/Skynet/Task/TimeoutError.html
+website/doc/classes/Skynet/TaskIterator.html
+website/doc/classes/Skynet/TaskIterator.src/M000262.html
+website/doc/classes/Skynet/TaskIterator.src/M000263.html
+website/doc/classes/Skynet/TaskIterator.src/M000264.html
+website/doc/classes/Skynet/TaskIterator.src/M000265.html
+website/doc/classes/Skynet/TaskIterator.src/M000266.html
+website/doc/classes/Skynet/TaskIterator.src/M000267.html
+website/doc/classes/Skynet/TaskIterator.src/M000268.html
+website/doc/classes/Skynet/TaskIterator/Error.html
+website/doc/classes/Skynet/UniqueDBNumGenerator.html
+website/doc/classes/Skynet/UniqueDBNumGenerator.src/M000200.html
+website/doc/classes/Skynet/UniqueDBNumGenerator.src/M000201.html
+website/doc/classes/Skynet/UniqueDBNumGenerator.src/M000202.html
+website/doc/classes/Skynet/UniqueDBNumGenerator.src/M000203.html
+website/doc/classes/Skynet/UniqueDBNumGenerator/Config.html
+website/doc/classes/Skynet/Worker.html
+website/doc/classes/Skynet/Worker.src/M000051.html
+website/doc/classes/Skynet/Worker.src/M000052.html
+website/doc/classes/Skynet/Worker.src/M000053.html
+website/doc/classes/Skynet/Worker.src/M000054.html
+website/doc/classes/Skynet/Worker.src/M000055.html
+website/doc/classes/Skynet/Worker.src/M000056.html
+website/doc/classes/Skynet/Worker.src/M000057.html
+website/doc/classes/Skynet/Worker.src/M000058.html
+website/doc/classes/Skynet/Worker.src/M000059.html
+website/doc/classes/Skynet/Worker.src/M000060.html
+website/doc/classes/Skynet/Worker.src/M000061.html
+website/doc/classes/Skynet/Worker.src/M000062.html
+website/doc/classes/Skynet/Worker.src/M000063.html
+website/doc/classes/Skynet/Worker.src/M000064.html
+website/doc/classes/Skynet/Worker.src/M000065.html
+website/doc/classes/Skynet/Worker.src/M000066.html
+website/doc/classes/Skynet/Worker.src/M000067.html
+website/doc/classes/Skynet/Worker.src/M000068.html
+website/doc/classes/Skynet/Worker.src/M000069.html
+website/doc/classes/Skynet/Worker/ConnectionFailure.html
+website/doc/classes/Skynet/Worker/Error.html
+website/doc/classes/Skynet/Worker/NoManagerError.html
+website/doc/classes/Skynet/Worker/RespawnWorker.html
+website/doc/classes/Skynet/WorkerStatusMessage.html
+website/doc/classes/Skynet/WorkerStatusMessage.src/M000097.html
+website/doc/classes/Skynet/WorkerStatusMessage.src/M000098.html
+website/doc/classes/Skynet/WorkerStatusMessage.src/M000099.html
+website/doc/classes/Skynet/WorkerVersionMessage.html
+website/doc/classes/Skynet/WorkerVersionMessage.src/M000164.html
+website/doc/classes/Skynet/WorkerVersionMessage.src/M000165.html
+website/doc/classes/Skynet/WorkerVersionMessage.src/M000166.html
+website/doc/classes/Skynet/WorkerVersionMessage.src/M000167.html
+website/doc/classes/SkynetDebugger.html
+website/doc/classes/SkynetDebugger.src/M000273.html
+website/doc/classes/SkynetDebugger.src/M000274.html
+website/doc/classes/SkynetDebugger.src/M000275.html
+website/doc/classes/SkynetDebugger.src/M000276.html
+website/doc/classes/SkynetDebugger.src/M000277.html
+website/doc/classes/SkynetDebugger.src/M000278.html
+website/doc/classes/SkynetDebugger.src/M000279.html
+website/doc/classes/SkynetDebugger.src/M000280.html
+website/doc/classes/SkynetDebugger.src/M000281.html
+website/doc/classes/SkynetDebugger.src/M000282.html
+website/doc/classes/SkynetDebugger/ClassMethods.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000283.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000284.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000285.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000286.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000287.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000288.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000289.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000290.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000291.html
+website/doc/classes/SkynetDebugger/ClassMethods.src/M000292.html
+website/doc/classes/SkynetMessageQueue.html
+website/doc/classes/SkynetWorkerQueue.html
+website/doc/classes/String.html
+website/doc/classes/String.src/M000003.html
+website/doc/created.rid
+website/doc/files/History_txt.html
+website/doc/files/License_txt.html
+website/doc/files/README_txt.html
+website/doc/files/bin/skynet.html
+website/doc/files/bin/skynet_install.html
+website/doc/files/bin/skynet_tuplespace_server.html
+website/doc/files/lib/skynet/mapreduce_helper_rb.html
+website/doc/files/lib/skynet/mapreduce_test_rb.html
+website/doc/files/lib/skynet/message_queue_adapters/message_queue_adapter_rb.html
+website/doc/files/lib/skynet/message_queue_adapters/mysql_rb.html
+website/doc/files/lib/skynet/message_queue_adapters/tuple_space_rb.html
+website/doc/files/lib/skynet/skynet_active_record_extensions_rb.html
+website/doc/files/lib/skynet/skynet_config_rb.html
+website/doc/files/lib/skynet/skynet_console_helper_rb.html
+website/doc/files/lib/skynet/skynet_console_rb.html
+website/doc/files/lib/skynet/skynet_debugger_rb.html
+website/doc/files/lib/skynet/skynet_guid_generator_rb.html
+website/doc/files/lib/skynet/skynet_job_rb.html
+website/doc/files/lib/skynet/skynet_launcher_rb.html
+website/doc/files/lib/skynet/skynet_logger_rb.html
+website/doc/files/lib/skynet/skynet_manager_rb.html
+website/doc/files/lib/skynet/skynet_message_queue_rb.html
+website/doc/files/lib/skynet/skynet_message_rb.html
+website/doc/files/lib/skynet/skynet_partitioners_rb.html
+website/doc/files/lib/skynet/skynet_ruby_extensions_rb.html
+website/doc/files/lib/skynet/skynet_task_rb.html
+website/doc/files/lib/skynet/skynet_tuplespace_server_rb.html
+website/doc/files/lib/skynet/skynet_worker_rb.html
+website/doc/files/lib/skynet/version_rb.html
+website/doc/files/lib/skynet_rb.html
+website/doc/files/website/index_txt.html
+website/doc/fr_class_index.html
+website/doc/fr_file_index.html
+website/doc/fr_method_index.html
+website/doc/index.html
+website/doc/rdoc-style.css
+website/index.html
+website/index.txt
+website/javascripts/rounded_corners_lite.inc.js
+website/stylesheets/screen.css
+website/template.rhtml
View
186 ts_manager/README.txt
@@ -0,0 +1,186 @@
+Skynet
+ http://skynet.rubyforge.org/
+ by Adam Pisoni and Geni.com
+
+== DESCRIPTION:
+
+Skynet is an open source Ruby implementation of Google's Map/Reduce framework, created at Geni. With Skynet, one can easily convert a time-consuming serial task, such as a computationally expensive Rails migration, into a distributed program running on many computers.
+
+Skynet is an adaptive, self-upgrading, fault-tolerant, and fully distributed system with no single point of failure. It uses a "peer recovery" system where workers watch out for each other. If a worker dies or fails for any reason, another worker will notice and pick up that task. Skynet also has no special 'master' servers, only workers which can act as a master for any task at any time. Even these master tasks can fail and will be picked up by other workers.
+
+== DOCUMENTATION
+
+Feel free to read on if you want more of an overview of Skynet with some specific examples. More specific documentation can be found here:
+
+Skynet::Job - The main interface to Skynet. These docs include an example of how to use Skynet.
+
+Skynet::Config - Configuration Options
+
+bin/skynet[link:files/bin/skynet.html] - Starting Skynet
+
+bin/skynet_install[link:files/bin/skynet_install.html] - Installing Skynet into a local project
+
+== Map/Reduce
+
+First of all, Skynet is merely a distributed computing system that allows you to follow the map/reduce steps. You don't have to use it as a map/reduce framework. You can use it as a simple distributed system, or even a simple asynchronous processing system.
+
+If you already know what Map/Reduce is, skip this section.
+
+If you want to know where all this Map/Reduce hype started, you should read Google's paper on it. http://labs.google.com/papers/mapreduce.html
+
+When I first read that Google paper some years ago, I was a little confused about what all the hypes was. At the most basic level, it seemed too simple to be revolutionary. So you've got a job with 3 steps, you put some data in, it gets split out to a map step run no many machines, the returned data gets reshuffled and parceled out to a reduce step run on many machines. All the results are then put together again. You can see it as 5 steps actually. Data -> Partition -> Map -> Partition -> Reduce. Simple enough. Almost too simple. It was only years later when I began working on Skynet that I realize what the revolutionary part of Google's framwork was. It made distributed computing accessible. Any engineer could write a complex distributed system without needing to know about the complexities of such systems. Also, since the distributed system was generalized, you would only need one class of machines to run ALL of your distributed processing, instead of specialized machines for specialized functions. THAT was revolutionary.
+
+There are a number of key differences between Google's MR system and skynet. Firstly, currently you can not actually send raw code to the workers. You are really only telling it where the code is. At first this bothered me a lot. Then I realized that in most OO systems, the amount of code you'd need duplicate and to send over the wire to every worker could be ridiculous. For example, if you want to distribute a task you need to run in Rails, you'd have to send almost all of your app and rails to every worker with every chunk of data. So, even if you COULD send code, you'd probably only be sending code that just called some other code in your system. If you can't send ALL the code it needs, then you might as well just tell it where the code is.
+
+The second big difference is that Google's MR framework uses Master federater processes to dole out tasks, recombine them, and generally watch the system. Skynet has not such masters. Instead Skynet uses a standard message queue for all communication. That same message queue allows workers to watch each other in the same way a master would, but without the single point of failure (except the queue itself).
+
+At its simplest level, a single map reduce job defines a data set, a map method and a reduce method. It may also define a partition method. The map/reduce server evenly splits up (partitions) the data given to it and sends those chunks of data, along with a copy of the code in the map method, to workers that execute the map method against the data it was given. The output from each worker is sent back to the map/reduce server. At this point the Mapreduce server evenly partitions the RESULT data returned from the workers and sends those chunks of data along with the reduce code to the workers to be executed. The reducers return the final result which is returned to whomever requested the job be done in the first place. Not all job need a reduce step, some may just have a map step.
+
+The most common example of a mapreduce job is a distributed word counter. Say you wanted to determine how many times a single word appears in a 1GB text file. The map/reduce server would break up the 1GB file into reasonable chunks, say 100 lines per chunk (or partition) and then send each 100 line partition along with the code that looks for that word, to workers. Each worker would grab its partition of the data, count how many times the word appears in the data and return that number. It might take dozens of workers to complete the task. When the map step is done, you are left with a huge list of counts returned by the workers. In this example, the reduce step would involve sending that list of counts to another worker, with the code required to sum those counts and finally return the total. In this way a task that used to be done in a linear fashion can be parallelized easily.
+
+== INSTALLATION:
+
+Skynet can be installed via RubyGems:
+
+ $ sudo gem install skynet
+
+== GETTING STARTED
+
+Skynet works by putting "tasks" on a message queue which are picked up by skynet workers, who execute the tasks, then put their results back on the message queue. Skynet works best when it runs with your code. For example, you might have a rails app and want some code you've already written to run asynchronously or in a distributed way. Skynet can run within your code by installing a skynet launcher into your app. Running this skynet launcher within your app guarantees all skynet workers will have access to your code. This will be covered later.
+
+Skynet currently supports 2 message queue systems, TupleSpace and Mysql. By default, the TupleSpace queue is used as it is the easiest to set up, though it is less powerful and less scaleable for large installations.
+
+== RUNING SKYNET FOR THE FIRST TIME
+Since Skynet is a distributed system, it requires you have a skynet message queue as well as any number of skynet workers running. To start a skynet message queue and a small number of workers:
+
+ $ skynet
+
+This starts a skynet tuple space message queue and 4 workers. You can now run the skynet console to play with skynet a little. See Skynet::ConsoleHelper for commands.
+
+ $ skynet console
+
+For help try:
+ $ skynet --help
+or
+ $ skynet console --help
+
+Here are some commands you can run in the skynet console.
+ > stats
+ > manager.worker_pids
+ > [1,2,3,1,1,4].mapreduce(Skynet::MapreduceTest)
+
+That last command actually took whatever array you gave it and counted the number of times each element appeared in the array. It's not a very useful task, but it shows how easy it is to use.
+
+For more information on creating your own Skynet jobs read the Skynet::Job documentation.
+
+== RUNING SKYNET IN YOUR APPLICATION
+
+To be really useful, you'll want to run skynet in your own application. To do that run:
+
+ $ skynet_install [--rails] [--mysql] YOUR_APP_DIRECTORY
+
+If you pass --rails it will assume it is installing in a rails app. If you pass --mysql it will assume you are using the mysql as your message queue.
+
+Once it is installed in your application, you can run skynet with
+
+ $ ./script/skynet
+ $ ./script/skynet console
+
+Now you can create your own Skynet jobs. Make sure to require your libraries in your ./script/skynet BEFORE running skynet so that skynet can use your code.
+
+== USAGE:
+
+Skynet was designed to make doing easy things easy and hard things possible. The easiest way to use skynet is to create a new class with a self.map class method. You can optionally include self.reduce, self.reduce_partitioner, self.map_partitioner as well. Each of those methods should expect a single array (regardless of what data you pass). Then, simple create an array and call mapreduce on it passing your class name. Skynet will figure out which methods your class supports and use them accordingly.
+
+== USING SKYNET IN RAILS
+
+Skynet includes an addition to ActiveRecord that is very powerful.
+
+=== distributed_find
+
+ $ YourModel.distributed_find(:all).each(YourClass)
+or
+ $ YourModel.distributed_find(:all).each(:somemethod)
+
+In the first example, a find is 'virtually' run with your model class, and the results are distributed to the skynet workers. If you've implemented a self.map method in YourClass, the retrieved objects will be passed (as arrays) on all the workers.
+
+In the second example, once the objects of YourModel are distributed, each worker merely calls :somemethod against each object.
+
+=== send_later
+
+ $ model_object.send_later(:method,options,:save)
+
+Sometimes you have a method you want to call on a model asynchronously. Using :send_later you can call a method, pass it options, and decide whether you want Skynet to save that model or not once its done calling your method.
+
+== Creating Skynet Jobs
+
+The main interface to Skynet is through Skynet::Job
+
+ job = Skynet::Job.new(options)
+ job.run
+
+There are many options you can pass or change once you have a job object. See Skynet::Job for more info.
+
+Most of the time, you will only need to pass a map_reduce_class and map_data. All other options just give you finer grain control. map_data must be an array. The map_reduce_class must AT LEAST implement a self.map class method. It may also implement self.reduce, self.reduce_partitioner, and self.map_partitioner. Skynet will assume it can use all of those methods in the map_reduce_class you pass.
+
+Your map and reduce class methods should ALWAYS assume they are being passed an array. Your map method must always return an array as well.
+
+== Skynet Logging
+
+ You might be interested in seeing what skynet is doing. There are 2 Skynet::Config options which control logging.
+ Skynet::CONFIG[:SKYNET_LOG_LEVEL] and Skynet::CONFIG[:SKYNET_LOG_FILE]
+ Skynet::CONFIG[:SKYNET_LOG_LEVEL] is set to Logger::ERROR by default. Other possibilities are Logger::DEBUG, Logger::INFO, Logger::WARN, Logger::ERROR, Logger::FATAL
+
+ You might try Logger::INFO to see more of what's going on. To use the Skynet::Logger inside your own classes simple
+ include SkynetDebugger
+
+SkynetDebugger[link:files/lib/skynet/skynet_debugger_rb.html]
+
+== CREDITS
+
+There are a number of people who either directly or indirectly worked on Skynet.
+* John Beppu (wrote the original worker/manager code)
+* Justin Balthrop
+* Zack Parker
+* Amos Elliston
+* Zack Hobson
+* Alan Braverman
+* Mike Stangel
+* Scott Steadman
+* Andrew Arrow
+* Jason Rojas
+
+Skynet was inspired by and heavily influenced by Josh Carter and this blog post.
+http://multipart-mixed.com/software/simple_mapreduce_in_ruby.html
+
+Also by Starfish by Lucas Carlson
+http://tech.rufy.com/2006/08/mapreduce-for-ruby-ridiculously-easy.html
+http://rufy.com/starfish/doc/
+
+== CONTACT:
+ Adam Pisoni, Geni.com (apisoni at geni.com)
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2007 Adam Pisoni, Geni.com
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
View
4 ts_manager/Rakefile
@@ -0,0 +1,4 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }
View
5 ts_manager/app_generators/skynet_install/USAGE
@@ -0,0 +1,5 @@
+Description:
+
+
+Usage:
+
View
95 ts_manager/app_generators/skynet_install/skynet_install_generator.rb
@@ -0,0 +1,95 @@
+class SkynetInstallGenerator < RubiGen::Base
+
+ DEFAULT_SHEBANG = File.join(Config::CONFIG['bindir'],
+ Config::CONFIG['ruby_install_name'])
+
+ default_options :in_rails => false
+ default_options :mysql => false
+
+ attr_reader :name
+ attr_reader :in_rails
+ attr_reader :mysql
+
+ def initialize(runtime_args, runtime_options = {})
+ super
+ usage if args.empty?
+ @destination_root = File.expand_path(args.shift)
+ @name = base_name
+ extract_options
+ end
+
+ def manifest
+ record do |m|
+ # Ensure appropriate folder(s) exists
+ m.directory 'script'
+ BASEDIRS.each { |path| m.directory path }
+
+ # Create stubs
+ m.template "skynet", "script/skynet", :collision => :ask, :chmod => 0775, :shebang => options[:shebang]
+ if @in_rails
+ m.template "skynet_initializer.rb", "config/initializers/skynet.rb", :collision => :ask, :chmod => 0655
+ m.directory 'config/initializers'
+ end
+ if @mysql
+ m.template "skynet_mysql_schema.sql", "db/skynet_mysql_schema.sql", :collision => :ask, :chmod => 0655
+ m.directory 'db/migrate'
+ m.migration_template "migration.rb", "db/migrate",
+ :collision => :ask,
+ :assigns => {
+ :migration_name => "CreateSkynetTables"
+ }, :migration_file_name => "create_skynet_tables"
+ end
+ end
+ end
+
+ protected
+ def banner
+ <<-EOS
+Creates a ...
+
+USAGE: #{spec.name} [--rails] [--mysql] directory (can be '.' for current)"
+Installs:
+ ./script/skynet
+EOS
+ end
+
+ def add_options!(opts)
+ opts.separator ''
+ opts.separator 'Options:'
+ # For each option below, place the default
+ # at the top of the file next to "default_options"
+ opts.on("-v", "--version", "Show the #{File.basename($0)} version number and quit.")
+ opts.on("--mysql",
+ "Include mysql migration if you want to use mysql as your message queue.
+ Installs:
+ ./db/skynet_mysql_schema.sql
+ ./db/migrate/db/migrate/###_create_skynet_tables.rb
+ ") do |mysql|
+ options[:mysql] = true if mysql
+ end
+ opts.on("-r", "--rails",
+ "Install into rails app.
+ Installs:
+ ./config/initializers/skynet.rb
+ (If using rails 1, make sure to add require 'skynet' to your environment.rb)",
+ "Default: false") do |rails|
+ options[:rails] = true if rails
+ end
+ end
+
+ def extract_options
+ # for each option, extract it into a local variable (and create an "attr_reader :author" at the top)
+ # Templates can access these value via the attr_reader-generated methods, but not the
+ # raw instance variable value.
+ @in_rails = options[:rails]
+ @mysql = options[:mysql]
+ end
+
+ # Installation skeleton. Intermediate directories are automatically
+ # created so don't sweat their absence here.
+ BASEDIRS = %w(
+ db
+ log
+ script
+ )
+end
View
66 ts_manager/app_generators/skynet_install/templates/migration.rb
@@ -0,0 +1,66 @@
+class <%= migration_name %> < ActiveRecord::Migration
+ def self.up
+ create_table :skynet_worker_queues do |t|
+ t.column :id, "bigint unsigned primary key"
+ t.column :queue_id, :integer, :default => 0
+ t.column :created_on, :timestamp
+ t.column :updated_on, :timestamp
+ t.column :tasktype, :string
+ t.column :tasksubtype, :string
+ t.column :worker_id, 'bigint unsigned'
+ t.column :hostname, :string
+ t.column :process_id, :integer
+ t.column :job_id, 'bigint unsigned'
+ t.column :task_id, 'bigint unsigned'
+ t.column :iteration, :integer
+ t.column :name, :string
+ t.column :map_or_reduce, :string
+ t.column :started_at, "decimal(16,4)"
+ t.column :version, :integer
+ t.column :processed, :integer
+ t.column :timeout, "decimal(16,4)"
+ end
+ create_table :skynet_message_queues do |t|
+ t.column :id, "bigint unsigned primary key"
+ t.column :queue_id, :integer, :default => 0
+ t.column :tran_id, "bigint unsigned"
+ t.column :created_on, :timestamp
+ t.column :updated_on, :timestamp
+ t.column :tasktype, :string
+ t.column :task_id, 'bigint unsigned'
+ t.column :job_id, 'bigint unsigned'
+ t.column :raw_payload, "longtext"
+ t.column :payload_type, :string
+ t.column :name, :string
+ t.column :expiry, :integer
+ t.column :expire_time, "decimal(16,4)"
+ t.column :iteration, :integer
+ t.column :version, :integer
+ t.column :timeout, "decimal(16,4)"
+ t.column :retry, :integer, :default => 0
+ end
+ create_table :skynet_queue_temperature do |t|
+ t.column :id, "bigint unsigned primary key"
+ t.column :queue_id, :integer, :default => 0
+ t.column :updated_on, :timestamp
+ t.column :count, :integer, :default => 0
+ t.column :temperature, "decimal(6,4) default 1"
+ t.column :type, :string
+ end
+ add_index :skynet_message_queues, :job_id
+ add_index :skynet_message_queues, :task_id
+ add_index :skynet_message_queues, :tran_id, :unique => true
+ add_index :skynet_message_queues, [:queue_id,:tasktype,:payload_type,:expire_time], :name => "index_skynet_mqueue_for_take"
+ add_index :skynet_worker_queues, [:hostname, :process_id]
+ add_index :skynet_worker_queues, :worker_id, :unique=> true
+ execute "insert into skynet_queue_temperature (queue_id,type) values (0,'master')"
+ execute "insert into skynet_queue_temperature (queue_id,type) values (0,'any')"
+ execute "insert into skynet_queue_temperature (queue_id,type) values (0,'task')"
+ end
+
+ def self.down
+ drop_table :skynet_worker_queues
+ drop_table :skynet_queue_temperature
+ drop_table :skynet_message_queues
+ end
+end
View
41 ts_manager/app_generators/skynet_install/templates/skynet
@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+<% if in_rails -%>
+# Load your rails app
+require File.expand_path(File.dirname(__FILE__) + '/../config/boot')
+require File.expand_path(File.dirname(__FILE__)) + '/../config/environment'
+<% end -%>
+require 'rubygems'
+require 'skynet'
+
+Skynet::CONFIG[:LAUNCHER_PATH] = File.expand_path(__FILE__)
+Skynet::CONFIG[:SKYNET_LOG_LEVEL] = Logger::ERROR
+<% if not in_rails -%>
+Skynet::CONFIG[:SKYNET_LOG_DIR] = File.expand_path(File.dirname(__FILE__) + "/../log")
+Skynet::CONFIG[:SKYNET_PID_DIR] = File.expand_path(File.dirname(__FILE__) + "/../log")
+<% end -%>
+
+<% if mysql -%>
+<% if not in_rails -%>
+# Use the mysql message queue adapter
+Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::Mysql"
+Skynet::CONFIG[:MYSQL_HOST] = "localhost"
+Skynet::CONFIG[:MYSQL_USERNAME] = "root"
+Skynet::CONFIG[:MYSQL_PASSWORD] = ""
+Skynet::CONFIG[:MYSQL_DATABASE] = "skynet"
+<% end -%>
+<% end -%>
+
+begin
+ mq = Skynet::MessageQueue.new
+rescue Skynet::ConnectionError
+ if Skynet::MessageQueue.adapter == :tuplespace
+ pid = fork do
+ exec("skynet_tuplespace_server start")
+ end
+ sleep Skynet::CONFIG[:TS_SERVER_START_DELAY]
+ end
+end
+
+
+Skynet.start
View
9 ts_manager/app_generators/skynet_install/templates/skynet_initializer.rb
@@ -0,0 +1,9 @@
+require 'skynet'
+Skynet::CONFIG[:SKYNET_LOG_DIR] = File.expand_path("#{RAILS_ROOT}/log")
+Skynet::CONFIG[:SKYNET_LOG_FILE] = "skynet_#{RAILS_ENV}.log"
+Skynet::CONFIG[:SKYNET_PID_DIR] = File.expand_path("#{RAILS_ROOT}/log")
+Skynet::CONFIG[:SKYNET_LOG_LEVEL] = Logger::ERROR
+<% if mysql -%>
+# Use the mysql message queue adapter
+Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::Mysql"
+<% end %>
View
56 ts_manager/app_generators/skynet_install/templates/skynet_mysql_schema.sql
@@ -0,0 +1,56 @@
+CREATE TABLE skynet_message_queues (
+ id int(11) NOT NULL auto_increment,
+ queue_id int(11) default '0',
+ tran_id bigint(20) unsigned default NULL,
+ created_on datetime default NULL,
+ updated_on datetime default NULL,
+ tasktype varchar(255) default NULL,
+ task_id bigint(20) unsigned default NULL,
+ job_id bigint(20) unsigned default NULL,
+ raw_payload longtext,
+ payload_type varchar(255) default NULL,
+ name varchar(255) default NULL,
+ expiry int(11) default NULL,
+ expire_time decimal(16,4) default NULL,
+ iteration int(11) default NULL,
+ version int(11) default NULL,
+ timeout decimal(16,4) default NULL,
+ retry int(11) default '0',
+ PRIMARY KEY (id),
+ UNIQUE KEY index_skynet_message_queues_on_tran_id (tran_id),
+ KEY index_skynet_message_queues_on_job_id (job_id),
+ KEY index_skynet_message_queues_on_task_id (task_id),
+ KEY index_skynet_mqueue_for_take (queue_id,tasktype,payload_type,expire_time)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+CREATE TABLE skynet_worker_queues (
+ id int(11) NOT NULL auto_increment,
+ queue_id int(11) default '0',
+ created_on datetime default NULL,
+ updated_on datetime default NULL,
+ tasktype varchar(255) default NULL,
+ tasksubtype varchar(255) default NULL,
+ worker_id bigint(20) unsigned default NULL,
+ hostname varchar(255) default NULL,
+ process_id int(11) default NULL,
+ job_id bigint(20) unsigned default NULL,
+ task_id bigint(20) unsigned default NULL,
+ iteration int(11) default NULL,
+ name varchar(255) default NULL,
+ map_or_reduce varchar(255) default NULL,
+ started_at decimal(16,4) default NULL,
+ version int(11) default NULL,
+ processed int(11) default NULL,
+ timeout decimal(16,4) default NULL,
+ PRIMARY KEY (id),
+ UNIQUE KEY index_skynet_worker_queues_on_worker_id (worker_id),
+ KEY index_skynet_worker_queues_on_hostname_and_process_id (hostname,process_id)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+CREATE TABLE skynet_queue_temperature (
+ id int(11) NOT NULL auto_increment,
+ queue_id int(11) default '0',
+ updated_on datetime default NULL,
+ count int(11) default '0',
+ temperature decimal(6,4) default NULL,
+ type varchar(255) default NULL,
+ PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
View
44 ts_manager/bin/skynet
@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+# This is the main skynet starter script.
+# It can be used to start the skynet_tuplespace_server as well as all skynet workers.
+# It is important that this script has access to your code. If you want to run
+# skynet within your code you'll want to read about bin/skynet_install[link:files/bin/skynet_install.html]
+#
+# Usage: skynet [options]
+# -w, --workers WORKERS Number of workers to start. The default is 4 and is stored in Skynet::CONFIG[:NUMBER_OF_WORKERS]
+# -i, --increment-worker-version Increment Worker Version
+# -a, --add-workers WORKERS Number of workers to add.
+# -k, --remove-workers WORKERS Number of workers to remove.
+# -r, --required LIBRARY Require the specified libraries
+# --restart-all-workers Restart All Workers
+# --restart-workers Restart Workers
+#
+# If you have chosen to use the TupleSpace message queue adapter this script will see if there is an available TS first
+# and start one if there is not. You can also start the bin/skynet_tuplespace_server[link:files/bin/skynet_tuplespace_server.html] manually.
+#
+# Running skynet starts a Skynet::Manager which in turn spawns the Skynet::Worker processes with the appropriate options.
+# You only need to run skynet once per machine. If you want to add more workers, use the appropriate flags above to do so.
+# Only one manager should be running per machine. The Skynet::Manager does not dole out tasks
+#
+# You should set all of your Skynet::CONFIG (or Skynet.configure()) options here (or in one of your environment files.)
+# See Skynet::Config for more information on configuration options.
+
+require 'rubygems'
+require File.expand_path(File.dirname(__FILE__)) + '/../lib/skynet.rb'
+
+Skynet::CONFIG[:WORKER_CHECK_DELAY] ||= 4
+Skynet::CONFIG[:LAUNCHER_PATH] = File.expand_path(__FILE__)
+
+begin
+ mq = Skynet::MessageQueue.new
+rescue Skynet::ConnectionError
+ if Skynet::MessageQueue.adapter == :tuplespace
+ pid = fork do
+ exec("skynet_tuplespace_server start")
+ end
+ sleep 3
+ end
+end
+
+Skynet.start
View
36 ts_manager/bin/skynet_install
@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+# skynet_install is used to install skynet binaries into your application.
+# The value of using your own skynet binaries is that they can ensure your code
+# is available to all skynet workers.
+# This is also how you can run skynet from within rails. (using --rails)
+#
+# USAGE: skynet_install [--rails] directory (can be '.' for current)"
+#
+# Options:
+# -v, --version Show the skynet_install version number and quit.
+# --include-migration Include mysql migration if you want to use mysql as your message queue
+# -r, --rails Install into rails app
+# Default: false
+# General Options:
+# -h, --help Show this help message and quit.
+# -p, --pretend Run but do not make any changes.
+# -f, --force Overwrite files that already exist.
+# -s, --skip Skip files that already exist.
+# -q, --quiet Suppress normal output.
+# -t, --backtrace Debugging: show backtrace on errors.
+# -c, --svn Modify files with subversion. (Note: svn must be in path)
+#
+
+require 'rubygems'
+require 'rubigen'
+
+if %w(-v --version).include? ARGV.first
+ require 'skynet_install/version'
+ puts "#{File.basename($0)} #{SkynetInstall::VERSION::STRING}"
+ exit(0)
+end
+
+require 'rubigen/scripts/generate'
+RubiGen::Base.use_application_sources! :skynet_install
+RubiGen::Scripts::Generate.new.run(ARGV, :generator => 'skynet_install')
View
66 ts_manager/bin/skynet_tuplespace_server
@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+# SkynetTupleSpace server is one of the message queues you can use with Skynet. Make sure you set:
+# Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::TupleSpace"
+#
+# Usage: skynet_tuplespace_server (start|stop|run) [options]
+# -t, --ontop TRUE Dont Daemonize
+# -p, --port PORT Port to listen on. default 7647
+# -o, --log LOGFILE Logfile to log to
+# -l, --loglevel LOGLEVEL Log level defaults to DEBUG
+# -d, --piddir PIDDIR Directory to put pidfile
+# -u, --drburi Drb URI What DRbURI to use
+
+require 'rubygems'
+require 'daemons'
+require 'pp'
+
+require File.expand_path(File.dirname(__FILE__)) + '/../lib/skynet.rb'
+
+options = {
+ :port => 7647,
+ :logfile => Skynet::Config.pidfile_location,
+ :loglevel => "ERROR",
+ :piddir => Skynet::CONFIG[:SKYNET_PID_DIR]
+}
+
+OptionParser.new do |opt|
+ opt.banner = "Usage: skynet_tuplespace_server (start|stop|run) [options]"
+ opt.on('-t', '--ontop TRUE', 'Dont Daemonize') do |v|
+ options[:ontop] = true if v.upcase == "TRUE" or v == "1"
+ end
+ opt.on('-p', '--port PORT', 'Port to listen on. default 7647') do |v|
+ options[:port] = v.to_i
+ end
+ opt.on('-o', '--logfile LOGFILE', 'Logfile to log to') do |v|
+ options[:logfile] = v
+ end
+ opt.on('-l', '--loglevel LOGLEVEL', 'Log level defaults to DEBUG') do |v|
+ options[:loglevel] = v
+ end
+ opt.on('-d', '--piddir PIDDIR', 'Directory to put pidfile') do |v|
+ options[:piddir] = File.expand_path(v)
+ end
+ opt.on('-u', '--drburi Drb URI', 'What DRbURI to use') do |v|
+ if v =~ %r{druby://}
+ options[:drburi] = v
+ else
+ options[:drburi] = "druby://#{v}"
+ end
+ options[:drburi] =~ /druby:\/\/.+?:(\d*)/
+ options[:port] = $1.to_i
+ end
+
+ opt.parse!(ARGV)
+end
+
+Daemons.run_proc("skynet_tuplespace_server#{options[:port]}",
+{
+ :dir_mode => :normal,
+ :dir => options[:piddir],
+ :backtrace => true,
+ # :monitor => true,
+ :ontop => options[:ontop] || false
+}) do
+ server = Skynet::Server.new(options)
+end
View
75 ts_manager/config/hoe.rb
@@ -0,0 +1,75 @@
+require 'skynet/version'
+
+AUTHOR = 'Adam Pisoni' # can also be an array of Authors
+EMAIL = "apisoni@geni.com"
+DESCRIPTION = "Skynet - A Ruby Map/Reduce Framework"
+GEM_NAME = 'skynet' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'skynet' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "unknown"
+def rubyforge_username
+ unless @config
+ begin
+ @config = YAML.load(File.read(File.expand_path(@config_file)))
+ rescue
+ puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+ EOS
+ exit
+ end
+ end
+ RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil
+# UNCOMMENT IF REQUIRED:
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = Skynet::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'skynet documentation',
+ "--opname", "index.html",
+ "--line-numbers",
+ "--main", "README",
+ "--inline-source"]
+
+class Hoe
+ def extra_deps
+ @extra_deps.reject! { |x| Array(x).first == 'hoe' }
+ @extra_deps
+ end
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+ p.author = AUTHOR
+ p.description = DESCRIPTION
+ p.email = EMAIL
+ p.summary = DESCRIPTION
+ p.url = HOMEPATH
+ p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+ p.test_globs = ["test/**/test_*.rb"]
+ p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store'] #An array of file patterns to delete on clean.
+
+ # == Optional
+ p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+ p.extra_deps = [
+ ['daemons',">= 1"],
+ ['rubigen', ">=1.1.1"]
+ ]
+ # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+
+ #p.spec_extras = {} # A hash of extra values to set in the gemspec.
+
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+hoe.rsync_args = '-av --delete --ignore-errors'
View
17 ts_manager/config/requirements.rb
@@ -0,0 +1,17 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+ begin
+ require req_gem
+ rescue LoadError
+ puts "This Rakefile requires the '#{req_gem}' RubyGem."
+ puts "Installation: gem install #{req_gem} -y"
+ exit
+ end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
+
+require 'skynet'
View
7 ts_manager/extras/README
@@ -0,0 +1,7 @@
+I've included some extras to help you deploy and monitor skynet.
+
+init.d/skynet # sample init.d script to start/stop/restart skynet
+
+nagios/check_skynet.sh # A script we use to have nagios (a monitoring tool) check our skynet Q.
+
+rails/* # I've included the controller and view we use to monitor our skynet Q
View
87 ts_manager/extras/init.d/skynet
@@ -0,0 +1,87 @@
+#!/bin/sh
+#
+# skynet start/stop skynet
+#
+# processname: solr
+#
+
+# Source function library
+. /etc/init.d/functions
+
+# Get network config
+. /etc/sysconfig/network
+
+HOME=/skynet/current
+RAILS_ENV=production
+
+cd $HOME
+RETVAL=0
+start() {
+ echo -n $"Starting Skynet: "
+ ./script/skynet &
+ RETVAL=$?
+ echo
+ return $RETVAL
+}
+
+stop() {
+ echo -n $"Stopping Skynet: "
+ pid=`cat /skynet/logs/skynet_production.pid`
+ kill ${pid}
+ RETVAL=$?
+ return $RETVAL
+}
+
+restart() {
+ echo -n $"Restarting Skynet: "
+ ./script/skynet -r
+ RETVAL=$?
+ echo
+ return $RETVAL
+}
+
+
+reload() {
+ stop
+ start
+}
+
+case "$1" in
+ start)
+ start
+ ;;
+ stop)
+ stop
+ ;;
+ status)
+ exit
+ ;;
+ restart)
+ restart
+ ;;
+ condrestart)
+ [ -f /var/lock/subsys/skynet ] && restart || :
+ ;;
+ reload)
+ reload
+ ;;
+ *)
+ echo $"Usage: $0 {start|stop|status|restart|condrestart|reload}"
+ exit 1
+esac
+get_pid_for_cmd ()
+{
+ pid=`cat /skynet/logs/skynet_worker7647.pid`
+ return pid
+
+ #for pid in `ls -t /proc`; do
+ # if [ -d /proc/$pid ] && [ -f /proc/$pid/cmdline ]; then
+ # if [ "$1" = "$(</proc/$pid/cmdline)" ]; then
+ # return
+ # fi
+ # fi
+ #done
+ #pid=0
+}
+
+exit $?
View
121 ts_manager/extras/nagios/check_skynet.sh
@@ -0,0 +1,121 @@
+#! /bin/sh
+#
+# Usage: ./check_skynet --help
+#
+# Examples:
+# ./check_skynet -w 300 -c 2000 -u
+
+# Paths to commands used in this script. These
+# may have to be modified to match your system setup.
+
+PATH=""
+
+ECHO="/bin/echo"
+SED="/bin/sed"
+GREP="/bin/grep"
+TAIL="/bin/tail"
+CAT="/bin/cat"
+CUT="/bin/cut"
+WC="/bin/wc"
+CURL="/usr/bin/curl -f"
+
+PROGNAME=`/bin/basename $0`
+PROGPATH=`echo $0 | /bin/sed -e 's,[\\/][^\\/][^\\/]*$,,'`
+REVISION=`echo '$Revision: 0.1 $' | /bin/sed -e 's/[^0-9.]//g'`
+
+. /usr/local/nagios/libexec/utils.sh
+
+print_usage() {
+ echo "Usage: $PROGNAME -w <threshold> -c <threshold> -u <url of skynet status page>"
+ echo "Usage: $PROGNAME --help"
+ echo "Usage: $PROGNAME --version"
+}
+
+print_help() {
+ print_revision $PROGNAME $REVISION
+ echo ""
+ print_usage
+ echo ""
+ echo "Check Skynet's untaken_tasks"
+ echo ""
+ support
+}
+
+# Make sure the correct number of command line
+# arguments have been supplied
+
+if [ $# -lt 3 ]; then
+ print_usage
+ exit $STATE_UNKNOWN
+fi
+
+# Grab the command line arguments
+
+exitstatus=$STATE_UNKNOWN #default
+while test -n "$1"; do
+ case "$1" in
+ --help)
+ print_help
+ exit $STATE_OK
+ ;;
+ -h)
+ print_help
+ exit $STATE_OK
+ ;;
+ --version)
+ print_revision $PROGNAME $VERSION
+ exit $STATE_OK
+ ;;
+ -V)
+ print_revision $PROGNAME $VERSION
+ exit $STATE_OK
+ ;;
+ -w)
+ WARNING=$2;
+ shift;
+ ;;
+ -c)
+ CRITICAL=$2;
+ shift;
+ ;;
+ -u)
+ URL=$2;
+ shift;
+ ;;
+ *)
+ echo "Unknown argument: $1"
+ print_usage
+ exit $STATE_UNKNOWN
+ ;;
+ esac
+ shift
+done
+
+
+CURRENT=$($CURL $URL | $GREP "untaken_tasks\|down"| $CUT -d: -f2)
+
+if [ -z $CURRENT ] ;then
+ $ECHO "CANNOT GATHER SKYNET TASKS CALL NOPS"
+ exit $STATE_UNKNOWN
+fi
+
+if [ $CURRENT -ge $CRITICAL ]; then
+ $ECHO "Skynet untaken_tasks: $CURRENT threshold: $CRITICAL CRITICAL"
+ $ECHO " "
+ exit $STATE_CRITICAL
+fi
+
+if [ $CURRENT -ge $WARNING ]; then
+ $ECHO "Skynet untaken_tasks: $CURRENT threshold: $WARNING WARNING"
+ $ECHO " "
+ exit $STATE_WARNING
+fi
+
+if [[ $CURRENT -lt $CRITICAL && $CURRENT -lt $WARNING ]]; then
+ $ECHO "Skynet untaken_tasks: $CURRENT threshold critical: $CRITICAL threshold warning: $WARNING OK"
+ $ECHO " "
+ exit $STATE_OK
+
+fi
+$ECHO "NO SCRIPT OUTPUT CALL NOPS!"
+exit $STATE_CRITICAL
View
41 ts_manager/extras/rails/controllers/skynet_controller.rb
@@ -0,0 +1,41 @@
+class Admin::SkynetController < AdminController
+
+ def index
+ begin
+ setup
+ if params[:skynet_message_queue]
+ Skynet.configure(:MYSQL_MESSAGE_QUEUE_TABLE => params[:skynet_message_queue]) do
+ @stats = @mq.stats
+ end
+ else
+ @stats = @mq.stats
+ end
+ @stats.merge!(Skynet::Manager.stats_for_hosts)
+ @stats[:hosts] = @stats[:servers].size
+ rescue Exception => e
+ logger.error "ERROR #{e.inspect} #{e.backtrace.join("\n")}"
+ end
+ end
+
+ # plain text page that will be used by monitoring scripts
+ def status
+ begin
+ setup
+ stats = @mq.stats
+ stats[:servers] = stats[:servers].keys.join(",")
+ stats.each { |k,v| stats[k.to_s] = stats.delete(k) }
+ text = stats.keys.sort.collect{ |k| "#{k}:#{stats[k]}" }.join("\n") + "\n"
+ render :text => text, :content_type => 'text/plain'
+ rescue Exception => e
+ render :text => "skynet is down\n", :content_type => 'text/plain'
+ end
+ end
+
+ private
+
+ def setup
+ @mq ||= Skynet::MessageQueue.new(Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER])
+ @last_updated = Time.now.strftime('%r')
+ end
+
+end
View
137 ts_manager/extras/rails/views/skynet/index.rhtml
@@ -0,0 +1,137 @@
+
+ <div class="hd">
+ <h1>SKYNET STATUS</h1>
+ <span>Last updated: <%= @last_updated || 'N/A' %></span>
+ <br />
+ <br />
+ </div>
+
+ <% if @stats %>
+
+ <h2>Overall Stats</h2>
+ <table class="admin_table">
+ <thead>
+ <tr>
+ <th>Hosts</th>
+ <th>Workers</th>
+ <th>Active Workers<BR>(Masters / Tasks / Either)</th>
+ <th>Idle Workers<BR>(Masters / Tasks / Either)</th>
+ <th>Untaken Tasks<BR>(Master / Task)</th>
+ <th>Taken Tasks<BR>(Master / Task)</th>
+ <th>Results</th>
+ <th>Processed <br />(by active workers)</th>
+ <th>Process Time</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td class="numeric"><%= @stats[:hosts] %></td>
+ <td class="numeric"><%= @stats[:number_of_workers] %></td>
+ <td class="numeric"><b><%= @stats[:active_workers] %></b> (<%= @stats[:active_masters] %> / <%= @stats[:active_taskworkers] %> / <%= @stats[:active_master_or_task_workers] %>)</td>
+ <td class="numeric"><b><%= @stats[:idle_workers] %></b> (<%= @stats[:idle_masters] %> / <%= @stats[:idle_taskworkers] %> / <%= @stats[:idle_master_or_task_workers] %>)</td>
+ <td class="numeric"><%= @stats[:untaken_tasks] %> (<b><%= @stats[:untaken_master_tasks] %></b> / <%= @stats[:untaken_task_tasks] %>)</td>
+ <td class="numeric"><b><%= @stats[:taken_tasks] %></b> (<%= @stats[:taken_master_tasks] %> / <%= @stats[:taken_task_tasks] %>)</td>
+ <td class="numeric"><%= @stats[:results] %></td>
+ <td class="numeric"><%= @stats[:processed] %> (<%= @stats[:processed_by_active_workers] %>)</td>
+ <td><%= @stats[:time] %></td>
+ </tr>
+ </tbody>
+ </table>
+
+
+ <BR>
+ <h2>Servers</h2>
+ <table class="admin_table">
+ <colgroup>
+ <col span="1" width="3%" />
+ <col span="1" width="10%" />
+ <col span="1" width="5%" />
+ <col span="1" width="5%" />
+ <col span="1" width="5%" />
+ <col span="1" width="5%" />
+ </colgroup>
+ <thead>
+ <tr>
+ <th class="numeric">#</th>
+ <th>Hostname</th>
+ <th>Number of Workers</th>
+ <th>Active Workers<BR>Masters / Tasks / Either</th>
+ <th>Idle Workers<BR>Masters / Tasks / Either</th>
+ <th>Processed <br />(by active workers)</th>
+ </tr>
+ </thead>
+
+ <% if @stats[:servers] %>
+ <tbody>
+ <% @stats[:servers].keys.sort.each do |hostname| %>
+ <% server = @stats[:servers][hostname] %>
+ <% i = 1 %>
+ <tr>
+ <td class="numeric"><%= i %></td>
+ <td><%= server[:hostname] %></td>
+ <td class="numeric"><%= server[:number_of_workers] %></td>
+ <td class="numeric"><%= server[:active_workers] %> (<%= server[:active_masters] %> / <%= server[:active_taskworkers] %> / <%= server[:active_master_or_task_workers] %>) </td>
+ <td class="numeric"><%= server[:idle_workers] %> (<%= server[:idle_masters] %> / <%= server[:idle_taskworkers] %> / <%= server[:idle_master_or_task_workers] %>) </td>
+ <td class="numeric"><%= server[:processed] %> (<%= server[:processed_by_active_workers] %>)</td>
+ </tr>
+ <% i += 1 %>
+ <% end %>
+ </tbody>
+ <% end %>
+
+ </table>
+
+ <% end %>
+
+ <!-- <table class="admin_table">
+ <colgroup>
+ <col span="1" width="5%" />
+ <col span="1" width="8%" />
+ <col span="1" width="10%" />
+ <col span="1" width="10%" />
+ <col span="1" width="5%" />
+ <col span="1" width="5%" />
+ </colgroup>
+ <thead>
+ <tr>
+ <th class="numeric">#</th>
+ <th>Worker ID</th>
+ <th>Hostname</th>
+ <th>PID</th>
+ <th>Job ID</th>
+ <th>Task ID</th>
+ <th>Version</th>
+ <th>Processed</th>
+ <th>M/R</th>
+ <th>Name</th>
+ </tr>
+ </thead>
+
+ <% if false and @servers %>
+ <tbody>
+ <% @servers.each do |server| %>
+ <tr>
+ <td colspan="10"><%= server[:manager] %></td>
+ </tr>
+ <% i = 1 %>
+ <% server[:workers].each do |w| %>
+ <tr>
+ <td class="numeric"><%= i %></td>
+ <td><%= w.worker_id %></td>
+ <td><%= w.hostname %></td>
+ <td class="numeric"><%= w.process_id %></td>
+ <td class="numeric"><%= w.job_id %></td>
+ <td class="numeric"><%= w.task_id %></td>
+ <td class="numeric"><%= w.version %></td>
+ <td class="numeric"><%= w.processed %></td>
+ <td><%= w.map_or_reduce || '-' %></td>
+ <td><%= w.name %></td>
+ </tr>
+ <% i += 1 %>
+ <% end %>
+ <% end %>
+ </tbody>
+ <% end %>
+
+ </table> -->
+
View
91 ts_manager/lib/skynet.rb
@@ -0,0 +1,91 @@
+$:.unshift File.dirname(__FILE__)
+$:.unshift File.dirname(__FILE__) + '/skynet'
+
+# path = File.expand_path(File.dirname(__FILE__))
+
+
+require 'drb'
+require 'skynet_guid_generator'
+require 'skynet_logger'
+require 'skynet_config'
+require 'timeout'
+
+Skynet::CONFIG[:SKYNET_PATH] ||= File.expand_path(File.dirname(__FILE__) +"/..")
+# Skynet::CONFIG[:LAUNCHER_PATH] ||= File.expand_path(ENV['_'])
+
+require 'skynet_debugger'
+require 'skynet_message'
+require 'message_queue_adapters/message_queue_adapter'
+require 'message_queue_adapters/tuple_space'
+require 'worker_queue_adapters/tuple_space'
+require "skynet_message_queue"
+require "skynet_worker_queue"
+require 'skynet_partitioners'
+require 'skynet_job'
+require 'skynet_worker'
+require 'skynet_task'
+require 'skynet_manager'
+require 'skynet_tuplespace_server'
+require 'skynet_ruby_extensions'
+begin
+ require 'active_record'
+ require 'skynet_active_record_extensions'
+ require 'message_queue_adapters/mysql'
+ require 'worker_queue_adapters/mysql'
+rescue LoadError => e
+end
+require 'mapreduce_test'
+require 'skynet_launcher'
+require 'skynet_console'
+require 'mapreduce_helper'
+
+
+begin
+ require 'fastthread'
+rescue LoadError
+ # puts 'fastthread not installed, using thread instead'
+ require 'thread'
+end
+
+class Skynet
+
+ # kinda like system() but gives me back a pid
+ def self.fork_and_exec(command)
+ sleep 0.01 # remove contention on manager drb object
+ log = Skynet::Logger.get
+ info "executing /bin/sh -c \"#{command}\""
+ pid = safefork do
+ close_files
+ exec("/bin/sh -c \"#{command}\"")
+ exit
+ end
+ Process.detach(pid)
+ pid
+ end
+
+ def self.safefork (&block)
+ @fork_tries ||= 0
+ fork(&block)
+ rescue Errno::EWOULDBLOCK
+ raise if @fork_tries >= 20
+ @fork_tries += 1
+ sleep 5
+ retry
+ end
+
+
+ # close open file descriptors starting with STDERR+1
+ def self.close_files(from=3, to=50)
+ close_console
+ (from .. to).each do |fd|
+ IO.for_fd(fd).close rescue nil
+ end
+ end
+
+ def self.close_console
+ STDIN.reopen "/dev/null"
+ STDOUT.reopen "/dev/null", "a"
+ STDERR.reopen STDOUT
+ end
+
+end
View
74 ts_manager/lib/skynet/mapreduce_helper.rb
@@ -0,0 +1,74 @@
+module MapreduceHelper
+# You can include the MapreduceHelper into your class to give you standard self.map and self.reduce methods.
+# You need only implement self.map_each and self.reduce_each methods which accept a single item (istead of an arrad)
+#
+# Example Usage:
+# This example is a bit contrived.
+#
+# class MapReduceTest
+# include MapreduceHelper
+#
+# def self.run
+# job = Skynet::Job.new(
+# :mappers => 2,
+# :reducers => 1,
+# :map_reduce_class => self,
+# :map_data => ['http://www.geni.com'.'http://www.yahoo.com','http://www.cnet.com']
+# )
+# results = job.run
+# end
+#
+# def self.map_each(url)
+# SomeUrlSlurper.gather_results(url) # returns an array of urls of sites that link to the given url
+# end
+#
+# def self.reduce_each(linked_from_url)
+# SomeUrlSluper.find_text("mysite", linked_from_url) # finds all the times "mysite" appears in the given url, which we know links to the url given in the map_data
+# end
+# end
+#
+# MapReduceTest.run
+
+
+ def self.included(base)
+ base.extend MapreduceHelper
+ end
+
+ # Takes an array of map_data, iterates over that array calling self.map_each(item) for each
+ # item in that array. Catches exceptions in each iteration and continues processing.
+ def map(map_data_array)
+ raise Skynet::Job::BadMapOrReduceError.new("#{self.class} has no self.map_each method.") unless self.respond_to?(:map_each)
+ if map_data_array.is_a?(Array)
+ results = []
+ map_data_array.each do |data|
+ begin
+ results << map_each(data)
+ rescue Exception => e
+ error "ERROR IN #{self} [#{e.class} #{e.message}] #{e.backtrace.join("\n")}"
+ end
+ end
+ results
+ else
+ map_each(map_data_array)
+ end
+ end
+
+ # Takes an array of post reduce_partitioned data, iterates over that array calling self.reduce_each(item) for each
+ # item in that array. Catches exceptions in each iteration and continues processing.
+ def reduce(reduce_partitioned_data_array)
+ raise Skynet::Job::BadMapOrReduceError.new("#{self.class} has no self.reduce_each method.") unless self.respond_to?(:reduce_each)
+ if reduce_partitioned_data_array.is_a?(Array)
+ results = []
+ reduce_partitioned_data_array.each do |data|
+ begin
+ results << reduce_each(data)
+ rescue Exception => e
+ error "ERROR IN #{self} [#{e.class} #{e.message}] #{e.backtrace.join("\n")}"
+ end
+ end
+ results
+ else
+ reduce_each(reduce_partitioned_data_array)
+ end
+ end
+end
View
56 ts_manager/lib/skynet/mapreduce_test.rb
@@ -0,0 +1,56 @@
+class Skynet
+ class MapreduceTest
+ include SkynetDebugger
+
+ def self.map(datas)
+ results = {}
+ datas.each do |data|
+ results[data] ||= 0
+ results[data] += 1
+ end
+ [results]
+ end
+
+ def self.reduce(datas)
+ results = {}
+ datas.each do |hashes|
+ hashes.each do |key,value|
+ results[key] ||= 0
+ results[key] += value
+ end
+ end
+ results
+ end
+ end
+
+ class ProfileCountTest
+ def self.run
+ job = Skynet::Job.new(
+ :mappers => 2,
+ :reducers => 1,
+ :map_reduce_class => self,
+ :map_data => [OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 3})]
+ )
+ results = job.run
+ end
+
+ def self.map(profiles)
+ result = Array.new
+ profiles.each do |profile|
+ result << [profile.created_by, 1] if profile.created_by
+ end
+ result
+ end
+
+ def self.reduce(pairs)
+ totals = Hash.new
+ pairs.each do |pair|
+ created_by, count = pair[0], pair[1]
+ totals[created_by] ||= 0
+ totals[created_by] += count
+ end
+ totals
+ end
+ end
+
+end
View
70 ts_manager/lib/skynet/message_queue_adapters/message_queue_adapter.rb
@@ -0,0 +1,70 @@
+class Skynet
+
+ class Error < StandardError
+ end
+
+ class RequestExpiredError < Skynet::Error
+ end
+
+ class InvalidMessage < Skynet::Error
+ end
+
+ class AbstractClassError < Skynet::Error
+ end
+
+ class MessageQueueAdapter
+
+ def list_results(data,timeout=nil)
+ raise AbstractClassError.new("You must implement list_results in a subclass.")
+ end
+
+ def list_tasks(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def take_next_task(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def write_message(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def write_result(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def take_result(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def write_error(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def write_worker_status(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def take_worker_status(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def read_all_worker_statuses(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def get_worker_version(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def set_worker_version(template,timeout=nil)
+ raise AbstractClassError.new("You must implement method in a subclass.")
+ end
+
+ def clear_outstanding_tasks
+ raise AbstractClassError.new("You must implement clear_outstanding_tasks in a subclass.")
+ end
+
+ end
+end
View
502 ts_manager/lib/skynet/message_queue_adapters/mysql.rb
@@ -0,0 +1,502 @@
+class SkynetMessageQueue < ActiveRecord::Base
+end
+
+class SkynetWorkerQueue < ActiveRecord::Base
+end
+
+
+class Skynet
+
+ # require 'mysql'
+
+ class Error < StandardError
+ end
+
+ class RequestExpiredError < Skynet::Error
+ end
+
+ class InvalidMessage < Skynet::Error
+ end
+
+ class MessageQueueAdapter::Mysql < Skynet::MessageQueueAdapter
+
+ include SkynetDebugger
+ include Skynet::GuidGenerator
+
+ SEARCH_FIELDS = [:tasktype, :task_id, :job_id, :payload_type, :expire_time, :iteration, :version] unless defined?(SEARCH_FIELDS)
+
+ Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY] ||= 30
+
+ @@db_set = false
+
+ def self.adapter
+ :mysql
+ end
+
+ def self.start_or_connect(options={})
+ new
+ end
+
+ def initialize
+ if Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TABLE]
+ SkynetMessageQueue.table_name = Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TABLE]
+ end
+ if not @@db_set
+ if Skynet::CONFIG[:MYSQL_QUEUE_DATABASE]
+ begin
+ SkynetMessageQueue.establish_connection Skynet::CONFIG[:MYSQL_QUEUE_DATABASE]
+ SkynetWorkerQueue.establish_connection Skynet::CONFIG[:MYSQL_QUEUE_DATABASE]
+ rescue ActiveRecord::AdapterNotSpecified => e
+ error "#{Skynet::CONFIG[:MYSQL_QUEUE_DATABASE]} not defined as a database adaptor #{e.message}"
+ end
+ elsif (not ActiveRecord::Base.connected?) and Skynet::CONFIG[:MYSQL_DATABASE]
+ db_options = {
+ :adapter => Skynet::CONFIG[:MYSQL_ADAPTER],
+ :host => Skynet::CONFIG[:MYSQL_HOST],
+ :username => Skynet::CONFIG[:MYSQL_USERNAME],
+ :password => Skynet::CONFIG[:MYSQL_PASSWORD],
+ :database => Skynet::CONFIG[:MYSQL_DATABASE]
+ }
+ ActiveRecord::Base.establish_connection(db_options)
+ end
+ end
+ @@db_set = true
+
+ end
+
+ def message_queue_table
+ Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TABLE] || SkynetMessageQueue.table_name
+ end
+
+ def self.debug_class_desc
+ "MYSQLMQ"
+ end
+
+ def message_to_conditions(message)
+ template_to_conditions(message.to_a)
+ end
+
+ def template_to_conditions(template,fields=Skynet::Message.fields)
+ conditions = []
+ values = []
+
+ fields.each_with_index do |field,ii|
+ value = template[ii]
+ next unless value
+ if value.is_a?(Range)
+ conditions << "#{field} BETWEEN #{value.first} AND #{value.last}"
+ elsif value.is_a?(Symbol) or value.is_a?(String)
+ conditions << "#{field} = '#{value}'"
+ else
+ conditions << "#{field} = #{value}"
+ end
+ end
+ return '' if conditions.empty?
+ return conditions.join(" AND ")
+ end
+
+ def message_to_hash(message,timeout=nil,fields=Skynet::Message.fields)
+ timeout ||= message.expiry
+ hash = {}
+ fields.each do |field|
+ next if field == :drburi
+ # next unless message.send(field)
+ if message.send(field).is_a?(Symbol)
+ hash[field] = message.send(field).to_s
+ elsif field == :payload
+ hash[:raw_payload] = message.raw_payload
+ else
+ hash[field] = message.send(field)
+ end
+ end
+ if timeout
+ hash[:timeout] = timeout
+ hash[:expire_time] = (Time.now.to_f + timeout) unless hash[:expire_time]
+ end
+ hash
+ end
+
+ def write_fallback_message(message_row, message)
+ tran_id = get_unique_id(1)
+ ftm = message.fallback_task_message
+ update_sql = %{
+ update #{message_queue_table}
+ SET iteration = #{ftm.iteration },
+ expire_time = #{ftm.expire_time},
+ updated_on = '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}',
+ tran_id = #{tran_id}
+ WHERE id = #{message_row.id} AND iteration = #{message.iteration}
+ AND tran_id #{(message_row.tran_id ? " =#{message_row.tran_id}" : ' IS NULL')}
+ }
+ rows = update(update_sql) || 0
+ message_row.tran_id = tran_id if rows == 1
+ rows
+ end
+
+ def take_next_task(curver,timeout=0.5,payload_type=nil,queue_id=0)
+ timeout = Skynet::CONFIG[:MYSQL_NEXT_TASK_TIMEOUT] if timeout < 1
+ debug "TASK NEXT TASK!!!!!!! timeout: #{timeout} queue_id:#{queue_id}"
+
+ start = Time.now
+ template = Skynet::Message.next_task_template(curver, payload_type)
+ message = nil
+
+ loop do
+ rows = 0
+ message = nil
+ template = Skynet::Message.next_task_template(curver, payload_type)
+ begin
+ message_row = find_next_message(template, payload_type)
+ if message_row
+ message = Skynet::Message.new(message_row.attributes)
+ rows = write_fallback_message(message_row, message)
+
+ if rows < 1
+ old_temp = temperature(payload_type)
+ set_temperature(payload_type, template_to_conditions(template), queue_id)
+ debug "MISSCOLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
+ else
+ break
+ end
+ else # no messages on queue with this temp
+ old_temp = temperature(payload_type)
+ if old_temp > 1
+ set_temperature(payload_type, template_to_conditions(template), queue_id)
+ end
+ debug "MISS PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
+ end
+ rescue Skynet::Message::BadMessage => e
+ message_row.destroy
+ next
+ rescue ActiveRecord::StatementInvalid => e
+ if e.message =~ /Deadlock/
+ old_temp = temperature(payload_type)
+ set_temperature(payload_type, template_to_conditions(template), queue_id)
+ debug "COLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
+ else
+ raise e