Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

first pass at regs.gov API integration

  • Loading branch information...
commit 2d10aa019a8eaf917439682162bab5959b8fd174 1 parent 2bb5ea9
@andrewcarpenter andrewcarpenter authored
View
3  Gemfile
@@ -49,8 +49,7 @@ gem "capistrano", '2.5.19', :require => false
gem "thunder_punch", '0.0.11', :require => false
gem "resque", "1.19.0"
-gem "gwt_rpc", '0.1.2'
-
+gem "httparty", "0.8.1"
gem "recaptcha", "0.3.1", :require => 'recaptcha/rails'
gem 'sendgrid', :git => "git://github.com/criticaljuncture/sendgrid.git", :branch => 'master'
gem 'modularity', '0.6.1'
View
15 Gemfile.lock
@@ -82,16 +82,12 @@ GEM
gherkin (2.2.8)
json (~> 1.4.6)
term-ansicolor (~> 1.0.5)
- gwt_rpc (0.1.2)
- activesupport
- activesupport
- json
- json
- typhoeus
- typhoeus
haml (3.0.4)
highline (1.6.1)
http_headers (0.0.2.3)
+ httparty (0.8.1)
+ multi_json
+ multi_xml
icalendar (1.1.5)
jgre-monkeyspecdoc (0.9.5)
json (1.4.6)
@@ -113,6 +109,7 @@ GEM
mongo (~> 1.2.1)
will_paginate (< 2.9)
multi_json (1.0.3)
+ multi_xml (0.4.1)
mysql (2.7)
net-scp (1.0.4)
net-ssh (>= 1.99.1)
@@ -173,8 +170,6 @@ GEM
capistrano (>= 2.5.5)
tilt (1.3.3)
timecop (0.3.5)
- typhoeus (0.3.3)
- mime-types
vegas (0.1.8)
rack (>= 1.0.0)
webrat (0.7.1)
@@ -210,9 +205,9 @@ DEPENDENCIES
formtastic (= 0.9.8)
fr2_audit_logic!
geokit (= 1.4.1)
- gwt_rpc (= 0.1.2)
haml (= 3.0.4)
http_headers (= 0.0.2.3)
+ httparty (= 0.8.1)
icalendar
jgre-monkeyspecdoc (= 0.9.5)
json
View
45 lib/content/entry_importer/regulations_dot_gov.rb
@@ -18,50 +18,7 @@ def comment_url
private
def regulationsdotgov_document
- return nil unless @entry.comments_close_on && @entry.comments_close_on >= Date.current
- possible_regulationsdotgov_search_terms.each do |term|
- doc = find_single_document(term)
- return doc if doc
- end
-
- nil
+ Content::RegulationsDotGov.new(ENV['regulations_dot_gov_api_key']).find_by_document_number(entry.document_number)
end
memoize :regulationsdotgov_document
-
- def possible_regulationsdotgov_search_terms
- terms = ["\"#{@entry.document_number}\""]
-
- @entry.docket_numbers.each do |docket_number|
- terms << "\"#{@entry.document_number}\" \"#{docket_number.number}\""
- end
-
- terms << "\"#{@entry.document_number}\" \"#{entry.title}\""
-
- @entry.agency_names.each do |agency_name|
- terms << "\"#{@entry.document_number}\" \"#{agency_name.name}\""
- end
-
- @entry.regulation_id_numbers.each do |rin|
- terms << "\"#{@entry.document_number}\" \"#{rin}\""
- end
-
- terms
- end
-
- def find_single_document(search_term)
- begin
- client = Content::RegulationsDotGov::Client.new
- documents = client.search(search_term)
-
- if documents && documents.size == 1
- documents.first
- else
- nil
- end
- rescue Exception => e
- Rails.logger.warn e
- Airbrake.notify(e)
- return nil
- end
- end
end
View
190 lib/content/regulations_dot_gov.rb
@@ -1,144 +1,82 @@
module Content
- module RegulationsDotGov
- class Client < GwtRpc::Client
- domain 'www.regulations.gov'
- js_url 'http://www.regulations.gov/Regs/'
- gwt_permutation '96ED140EA002EA7F0224967DBF229721'
- timeout 2000
- num_attempts 3
-
- map_classes 'gov.egov.erule.regs.shared.action.LoadSearchResultsResult' => 'Content::RegulationsDotGov::SearchResultPackage',
- 'gov.egov.erule.regs.shared.models.SearchResultModel' => 'Content::RegulationsDotGov::SearchResult',
- 'gov.egov.erule.regs.shared.models.Agency' => 'Content::RegulationsDotGov::Agency',
- 'gov.egov.erule.regs.shared.models.DimensionCounterFilter' => 'Content::RegulationsDotGov::DimensionCounter',
- 'gov.egov.erule.regs.shared.models.CommentPeriod' => 'Content::RegulationsDotGov::CommentPeriod',
- 'gov.egov.erule.regs.shared.models.DocumentSummaryModel' => 'Content::RegulationsDotGov::DocumentSummary',
- 'gov.egov.erule.regs.shared.resources.SharedConstants$DOCUMENT_STATUS' => 'Content::RegulationsDotGov::DocumentStatus',
- 'gov.egov.erule.regs.shared.models.DocumentType' => 'Content::RegulationsDotGov::DocumentType',
- 'gov.egov.erule.regs.shared.models.DocketType' => 'Content::RegulationsDotGov::DocketType'
-
- add_procedure(:search, :path => '/dispatch/LoadSearchResultsAction') do |term|
- "7|0|17|http://www.regulations.gov/Regs/|E4625492BC992E3FD53A93EC85103116|com.gwtplatform.dispatch.shared.DispatchService|execute|java.lang.String/2004016611|com.gwtplatform.dispatch.shared.Action|9324e41757554f59f2c8f74e588b41040fab46547bea7264778f86992a4bdcd9.e38Sc3uTa3qQe3yTbi0|gov.egov.erule.regs.shared.action.LoadSearchResultsAction/125242584|gov.egov.erule.regs.shared.models.SearchQueryModel/1556278353|java.util.ArrayList/4159755760||#{term}|gov.egov.erule.regs.shared.models.DataFetchSettings/1603506619|java.lang.Integer/3438268394|docketId|DESC|java.lang.Boolean/476441737|1|2|3|4|2|5|6|7|8|0|9|10|0|11|11|11|10|0|3|10|1|5|12|13|14|0|15|16|14|10|-8|11|11|11|17|0|1|0|"
- end
- end
-
- class SearchResultPackage
- def self.gwt_deserialize(reader)
- reader.read_object
- end
- end
-
- class SearchResult
- def self.gwt_deserialize(reader)
- reader.read_object
- reader.read_object
- reader.read_object
- end
- end
-
- class Agency
- def self.gwt_deserialize(reader)
- reader.read_string # abbr
- reader.read_object # num_results
- reader.read_string # id
- reader.read_string # name
- end
+ class RegulationsDotGov
+ class RecordNotFound < HTTParty::ResponseError; end
+ class ServerError < HTTParty::ResponseError; end
+
+ include HTTParty
+ base_uri 'http://www.regulations.gov/api/'
+
+ def initialize(api_key)
+ @api_key = api_key
end
-
- class DimensionCounter
- def self.gwt_deserialize(reader)
- reader.read_object
- reader.read_object
- end
+
+ def find_by_document_number(document_number)
+ begin
+ fetch_by_document_number(document_number)
+ rescue RecordNotFound, ServerError => e
+ revised_document_number = pad_document_number(document_number)
+ if revised_document_number != document_number
+ fetch_by_document_number(revised_document_number)
+ else
+ nil
+ end
+ end
end
-
- class CommentPeriod
- def self.gwt_deserialize(reader)
- reader.read_int
- end
+
+ private
+
+ def pad_document_number(document_number)
+ part_1, part_2 = document_number.split(/-/, 2)
+ sprintf("%s-%05d", part_1, part_2.to_i)
end
-
- class DocumentStatus
- def self.gwt_deserialize(reader)
- reader.read_int
- end
+
+ def fetch_by_document_number(document_number)
+ response = self.class.get('/getdocument/v1.json', :query => {:api_key => @api_key, :FR => document_number})
+ Document.new(response.parsed_response["document"])
end
-
- class DocumentType
- def self.gwt_deserialize(reader)
- reader.read_int
+
+ def self.get(url, options)
+ response = super
+
+ case response.code
+ when 200
+ response
+ when 404
+ raise RecordNotFound.new(response)
+ when 500
+ raise ServerError.new(response)
+ else
+ raise HTTParty::ResponseError.new(response)
end
end
-
- class DocketType
- def self.gwt_deserialize(reader)
- reader.read_int
+
+ class Document
+ def initialize(raw_attributes)
+ @raw_attributes = raw_attributes
+ @metadata = {}
+ @raw_attributes["metadata"]["entry"].each do |hsh|
+ @metadata[ hsh["@name"] ] = hsh["$"]
+ end
end
- end
-
- class DocumentSummary
- attr_accessor :document_id
-
- def initialize(document_id, allows_comments)
- @document_id = document_id
- @allows_comments = allows_comments
+
+ def document_id
+ @raw_attributes['documentId']
end
-
- def allows_comments?
- @allows_comments
+
+ def comment_due_date
+ val = @metadata["Comment Due Date"]
+ if val.present?
+ DateTime.parse(val)
+ end
end
-
+
def comment_url
- "http://www.regulations.gov/#!submitComment;D=#{document_id}" if allows_comments?
+ "http://www.regulations.gov/#!submitComment;D=#{document_id}"
end
-
+
def url
"http://www.regulations.gov/#!documentDetail;D=#{document_id}"
end
-
- def self.gwt_deserialize(reader)
- # Some stuff we don't care about
- reader.read_int
- reader.read_string
- reader.read_object
- reader.read_int
-
- allows_comments = reader.read_object
-
- # Some more stuff we don't care about
- reader.read_object
- reader.read_string
- reader.read_string
- reader.read_int
- reader.read_string
- reader.read_string
- reader.read_string
- reader.read_string
- reader.read_object
-
- document_id = reader.read_string
-
- # Some more stuff we don't care about
- reader.read_object
- reader.read_object
- reader.read_object
- reader.read_object
- reader.read_object
- reader.read_object
- reader.read_object
- reader.read_string
- reader.read_string
- reader.read_object
- reader.read_string
- reader.read_string
- reader.read_string
- reader.read_string
- reader.read_string
- reader.read_string
- reader.read_object
-
- self.new(document_id, allows_comments)
- end
end
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.