Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

We’re showing branches in this repository, but you can also compare across forks.

...
  • 3 commits
  • 7 files changed
  • 0 commit comments
  • 1 contributor
Commits on Apr 17, 2012
Rhett Sutphin rsutphin Add task to preserve staff data without importing it. #1999.
This is an alternative to running the (not yet written) importer. The choice of using this
vs. actually importing the staff data will depend on each center's needs.
b9bbb7f
Rhett Sutphin rsutphin Whitespace. 38c67f8
Rhett Sutphin rsutphin Add cap task to create importer passthrough directories. Closes #1999.
Implementation was copied from Core.
2693d00
58 Gemfile.lock
View
@@ -28,7 +28,7 @@ GEM
activemodel (= 3.0.10)
activesupport (= 3.0.10)
activesupport (3.0.10)
- addressable (2.2.6)
+ addressable (2.2.7)
aker (3.0.2)
activesupport (>= 2.3.0)
castanet (~> 1.0.0)
@@ -63,7 +63,7 @@ GEM
selenium-webdriver (~> 2.0)
xpath (~> 0.1.4)
castanet (1.0.1)
- childprocess (0.2.2)
+ childprocess (0.2.9)
ffi (~> 1.0.6)
chronic (0.6.4)
ci_reporter (1.6.5)
@@ -80,17 +80,7 @@ GEM
capybara (>= 1.1.1)
cucumber (>= 1.1.0)
nokogiri (>= 1.5.0)
- data_mapper (1.2.0)
- dm-aggregates (~> 1.2.0)
- dm-constraints (~> 1.2.0)
- dm-core (~> 1.2.0)
- dm-migrations (~> 1.2.0)
- dm-serializer (~> 1.2.0)
- dm-timestamps (~> 1.2.0)
- dm-transactions (~> 1.2.0)
- dm-types (~> 1.2.0)
- dm-validations (~> 1.2.0)
- data_objects (0.10.7)
+ data_objects (0.10.8)
addressable (~> 2.1)
database_cleaner (0.6.7)
diff-lcs (1.1.3)
@@ -108,14 +98,6 @@ GEM
dm-postgres-adapter (1.2.0)
dm-do-adapter (~> 1.2.0)
do_postgres (~> 0.10.6)
- dm-serializer (1.2.1)
- dm-core (~> 1.2.0)
- fastercsv (~> 1.5.4)
- json (~> 1.6.1)
- json_pure (~> 1.6.1)
- multi_json (~> 1.0.3)
- dm-timestamps (1.2.0)
- dm-core (~> 1.2.0)
dm-transactions (1.2.0)
dm-core (~> 1.2.0)
dm-types (1.2.1)
@@ -128,8 +110,8 @@ GEM
uuidtools (~> 2.1.2)
dm-validations (1.2.0)
dm-core (~> 1.2.0)
- do_postgres (0.10.7)
- data_objects (= 0.10.7)
+ do_postgres (0.10.8)
+ data_objects (= 0.10.8)
erubis (2.6.6)
abstract (>= 1.0.0)
exception_notification (2.5.2)
@@ -155,11 +137,11 @@ GEM
activesupport (~> 3.0)
haml (~> 3.0)
railties (~> 3.0)
- highline (1.6.8)
+ highline (1.6.11)
i18n (0.5.0)
inifile (0.4.1)
- json (1.6.1)
- json_pure (1.6.1)
+ json (1.6.6)
+ json_pure (1.6.6)
json_spec (0.7.0)
multi_json (~> 1.0.0)
rspec (~> 2.0)
@@ -171,17 +153,25 @@ GEM
mime-types (~> 1.16)
treetop (~> 1.4.8)
mime-types (1.17.2)
- multi_json (1.0.3)
- ncs_mdes (0.4.2)
+ multi_json (1.0.4)
+ ncs_mdes (0.5.0)
nokogiri (~> 1.4)
- ncs_mdes_warehouse (0.1.1)
+ ncs_mdes_warehouse (0.4.1)
activesupport (~> 3.0)
bcdatabase (~> 1.1)
builder (>= 2.1.2)
- data_mapper (~> 1.2.0)
+ childprocess (~> 0.2.3)
+ dm-aggregates (~> 1.2.0)
+ dm-constraints (~> 1.2.0)
+ dm-core (~> 1.2.0)
+ dm-migrations (~> 1.2.0)
dm-postgres-adapter (~> 1.2.0)
+ dm-transactions (~> 1.2.0)
+ dm-types (~> 1.2.0)
+ dm-validations (~> 1.2.0)
i18n (~> 0.4)
- ncs_mdes (~> 0.4, >= 0.4.2)
+ json (~> 1.6)
+ ncs_mdes (~> 0.5)
ncs_navigator_configuration (~> 0.2)
nokogiri (~> 1.5.0)
rubyzip (~> 0.9.4)
@@ -198,7 +188,7 @@ GEM
net-ssh (2.2.1)
net-ssh-gateway (1.1.0)
net-ssh (>= 1.99.1)
- nokogiri (1.5.0)
+ nokogiri (1.5.2)
pg (0.11.0)
polyglot (0.3.3)
rack (1.2.4)
@@ -238,7 +228,7 @@ GEM
railties (~> 3.0)
rspec (~> 2.6.0)
rubytree (0.7.0)
- rubyzip (0.9.4)
+ rubyzip (0.9.7)
scoped_search (2.3.5)
activerecord (>= 2.1.0)
selenium-webdriver (2.9.1)
@@ -248,7 +238,7 @@ GEM
rubyzip
shoulda (2.11.3)
spork (0.9.0.rc9)
- stringex (1.3.0)
+ stringex (1.3.2)
tabs_on_rails (2.0.2)
term-ansicolor (1.0.7)
thor (0.14.6)
22 config/deploy.rb
View
@@ -75,11 +75,26 @@
desc "#{t} task is a no-op with mod_rails"
task t, :roles => :app do ; end
end
-
+
desc "Fix permissions"
task :permissions do
sudo "chmod -R g+w #{shared_path} #{current_path} #{release_path}"
end
+
+ desc 'Set up shared paths used by the importer'
+ task :setup_import_directories do
+ shared_import = File.join(shared_path, 'importer_passthrough')
+ release_import = File.join(current_path, 'importer_passthrough')
+ cmds = [
+ "mkdir -p '#{shared_import}'",
+ # Only chmod if owned; this is the only case in which chmod is
+ # allowed. Will be owned if just created, which is the important
+ # case.
+ "if [ -O '#{shared_import}' ]; then chmod g+w '#{shared_import}'; fi",
+ "if [ ! -e '#{release_import}' ]; then ln -s '#{shared_import}' '#{release_import}'; fi"
+ ]
+ run cmds.join(' && ')
+ end
end
# backup the database before migrating
@@ -89,7 +104,8 @@
after 'deploy:update_code', 'deploy:cleanup'
# after deploying symlink, aggressively set permissions, copy images to current image config location.
-after 'deploy:symlink', 'deploy:permissions', 'config:images', 'config:static_authority'
+after 'deploy:symlink', 'deploy:permissions', 'config:images', 'config:static_authority',
+ 'deploy:setup_import_directories'
# Database
namespace :db do
@@ -104,7 +120,7 @@
task :images, :roles => :app do
run "cd #{current_path} && bundle exec rake RAILS_ENV=#{rails_env} config:images"
end
-
+
desc "Create the static auth file for the application user"
task :static_authority, :roles => :app do
run "cd #{current_path} && bundle exec rake RAILS_ENV=#{rails_env} static_user:create"
5 lib/ncs_navigator/staff_portal/warehouse.rb
View
@@ -4,6 +4,9 @@
module NcsNavigator::StaffPortal
module Warehouse
- autoload :Enumerator, 'ncs_navigator/staff_portal/warehouse/enumerator'
+ autoload :Enumerator, 'ncs_navigator/staff_portal/warehouse/enumerator'
+ autoload :NotImportedPassthrough,
+ 'ncs_navigator/staff_portal/warehouse/not_imported_passthrough'
+ autoload :Passthrough, 'ncs_navigator/staff_portal/warehouse/passthrough'
end
end
26 lib/ncs_navigator/staff_portal/warehouse/not_imported_passthrough.rb
View
@@ -0,0 +1,26 @@
+module NcsNavigator::StaffPortal::Warehouse
+ ##
+ # An "importer" that skips importing. Instead, it copies all the
+ # tables that SP cares about into a VDR-formatted XML file in the
+ # `importer_passthrough` directory. This file can then be included
+ # in the warehouse ETL process alongside the SP transformer to merge
+ # old and new data in the reported file.
+ #
+ # This class is very similar to the "unused" table passthroughs in
+ # NCS Navigator Core. The difference is that this passthrough is an
+ # alternative to doing a staff portal import rather than a
+ # complement for preserving data that isn't imported in a normal
+ # import.
+ class NotImportedPassthrough
+ include XmlPassthrough
+
+ def filename
+ 'outside_staff'
+ end
+
+ def passthrough_tables
+ Enumerator.record_producers.
+ select { |rp| rp.respond_to?(:model) }.collect(&:model).collect(&:mdes_table_name)
+ end
+ end
+end
34 lib/ncs_navigator/staff_portal/warehouse/xml_passthrough.rb
View
@@ -0,0 +1,34 @@
+require 'ncs_navigator/staff_portal'
+
+module NcsNavigator::StaffPortal::Warehouse
+ module XmlPassthrough
+ def initialize(wh_config)
+ @wh_config = wh_config
+ end
+
+ def import
+ unless path.parent.exist?
+ path.parent.mkpath
+ Rails.logger.
+ warn "Had to create #{path.parent}. Passthrough files will be lost after next deploy."
+ end
+ create_emitter.emit_xml
+ end
+
+ def create_emitter
+ @emitter ||= NcsNavigator::Warehouse::XmlEmitter.new(
+ @wh_config, path,
+ :zip => false, :'include-pii' => true, :tables => passthrough_tables)
+ end
+
+ def path
+ @path ||= Rails.root + "importer_passthrough/#{filename}-#{timestamp}.xml"
+ end
+
+ protected
+
+ def timestamp
+ Time.now.getutc.iso8601.gsub(/[^\d]/, '')
+ end
+ end
+end
25 lib/tasks/import.rake
View
@@ -0,0 +1,25 @@
+namespace :import do
+ task :warehouse_setup do |t|
+ class << t; attr_accessor :config; end
+
+ source_warehouse_config_file = ENV['IMPORT_CONFIG'] || '/etc/nubic/ncs/warehouse/import.rb'
+
+ require 'ncs_navigator/warehouse'
+
+ t.config = NcsNavigator::Warehouse::Configuration.
+ from_file(source_warehouse_config_file)
+ t.config.set_up_logs
+
+ NcsNavigator::Warehouse::DatabaseInitializer.new(t.config).set_up_repository
+ end
+
+ # For import scenarios where the staff data will not go into SP, but
+ # rather will exist separately and be ETL'd in parallel.
+ desc 'Pass outside staff & outreach data through to an XML file'
+ task :passthrough => [:warehouse_setup, :environment] do
+ require 'ncs_navigator/core'
+
+ pass = NcsNavigator::StaffPortal::Warehouse::NotImportedPassthrough.new(import_wh_config)
+ pass.import
+ end
+end
54 spec/lib/ncs_navigator/staff_portal/warehouse/not_imported_passthrough_spec.rb
View
@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+module NcsNavigator::StaffPortal::Warehouse
+ describe NotImportedPassthrough, :warehouse do
+ let(:wh_config) { NcsNavigator::Warehouse::Configuration.new }
+ let(:passthrough) { NotImportedPassthrough.new(wh_config) }
+
+ describe '#create_emitter', :slow do
+ subject { passthrough.create_emitter }
+
+ let(:model_tables) { subject.models.collect(&:mdes_table_name) }
+
+ it 'includes PII' do
+ subject.include_pii?.should be_true
+ end
+
+ it 'skips the ZIP' do
+ subject.zip?.should be_false
+ end
+
+ it 'writes to a file in the importer_passthrough directory' do
+ subject.filename.to_s.should =~
+ %r(#{Rails.root}/importer_passthrough/outside_staff-\d{14}.xml)
+ end
+
+ it 'does not include core operational models' do
+ model_tables.should_not include('person')
+ end
+
+ it 'does not include instrument models' do
+ model_tables.should_not include('pre_preg')
+ end
+
+ it 'does include staff models' do
+ model_tables.should include('staff')
+ end
+
+ it 'does include outreach models' do
+ model_tables.should include('outreach')
+ end
+ end
+
+ describe '#import' do
+ let(:mock_emitter) { mock(NcsNavigator::Warehouse::XmlEmitter) }
+
+ it 'emits the XML' do
+ passthrough.should_receive(:create_emitter).and_return(mock_emitter)
+ mock_emitter.should_receive(:emit_xml)
+
+ passthrough.import
+ end
+ end
+ end
+end

No commit comments for this range

Something went wrong with that request. Please try again.