From 1d137c6cb4dd112bbab003cc51219c98487dcc03 Mon Sep 17 00:00:00 2001 From: Shane Curcuru Date: Thu, 14 Mar 2024 10:54:48 -0400 Subject: [PATCH] Refactor UK Charities lookup; add raspberrypi UK finance --- _data/taxes/uk-1129409.json | 285 +++++++++++++++++++++++++++++ assets/ruby/foundation_reporter.rb | 57 +++++- assets/ruby/uk_charities.rb | 49 ----- 3 files changed, 337 insertions(+), 54 deletions(-) create mode 100644 _data/taxes/uk-1129409.json delete mode 100644 assets/ruby/uk_charities.rb diff --git a/_data/taxes/uk-1129409.json b/_data/taxes/uk-1129409.json new file mode 100644 index 0000000..a48e552 --- /dev/null +++ b/_data/taxes/uk-1129409.json @@ -0,0 +1,285 @@ +{ + "organization": { + "organisation_number": 5002372, + "reg_charity_number": 1129409, + "group_subsid_suffix": 0, + "charity_name": "RASPBERRY PI FOUNDATION", + "charity_type": "Charitable company", + "insolvent": false, + "in_administration": false, + "prev_excepted_ind": false, + "cif_cdf_ind": null, + "cio_dissolution_ind": false, + "interim_manager_ind": null, + "date_of_interim_manager_appt": null, + "reg_status": "R", + "date_of_registration": "2009-05-05T00:00:00", + "date_of_removal": null, + "latest_acc_fin_year_start_date": "2022-01-01T00:00:00", + "latest_acc_fin_year_end_date": "2022-12-31T00:00:00", + "latest_income": 157262030.0, + "latest_expenditure": 151064960.0, + "address_line_one": "RASPBERRY PI FOUNDATION", + "address_line_two": "37 Hills Road", + "address_line_three": "CAMBRIDGE", + "address_line_four": null, + "address_line_five": null, + "address_post_code": "CB2 1NT", + "phone": "01223322633", + "email": "legal@raspberrypi.org", + "web": "www.raspberrypi.org", + "charity_co_reg_number": "6758215", + "reporting_status": "Submission Received", + "removal_reason": null, + "cio_ind": false, + "last_modified_time": "2023-10-27T16:04:56.143", + "trustee_names": [ + { + "organisation_number": 5002372, + "trustee_name": "Jonathan Ilan Drori", + "trustee_id": 2125014 + }, + { + "organisation_number": 5002372, + "trustee_name": "KIM SHILLINGLAW", + "trustee_id": 11210523 + }, + { + "organisation_number": 5002372, + "trustee_name": "Prof Richard Plumbly-Clegg", + "trustee_id": 12356985 + }, + { + "organisation_number": 5002372, + "trustee_name": "Dr John Lazar", + "trustee_id": 12463481 + }, + { + "organisation_number": 5002372, + "trustee_name": "Amali Chivanthi de Alwis", + "trustee_id": 12471600 + }, + { + "organisation_number": 5002372, + "trustee_name": "Daniel Labbad", + "trustee_id": 12471610 + }, + { + "organisation_number": 5002372, + "trustee_name": "Charles Richard Leadbeater", + "trustee_id": 3585234 + }, + { + "organisation_number": 5002372, + "trustee_name": "David Zahn", + "trustee_id": 12705147 + }, + { + "organisation_number": 5002372, + "trustee_name": "Janet Astall", + "trustee_id": 12758264 + } + ], + "who_what_where": [ + { + "classification_code": "102", + "classification_type": "What", + "classification_desc": "Education/training" + }, + { + "classification_code": "201", + "classification_type": "Who", + "classification_desc": "Children/young People" + }, + { + "classification_code": "205", + "classification_type": "Who", + "classification_desc": "Other Charities Or Voluntary Bodies" + }, + { + "classification_code": "207", + "classification_type": "Who", + "classification_desc": "The General Public/mankind" + }, + { + "classification_code": "306", + "classification_type": "How", + "classification_desc": "Provides Services" + }, + { + "classification_code": "308", + "classification_type": "How", + "classification_desc": "Sponsors Or Undertakes Research" + }, + { + "classification_code": "310", + "classification_type": "How", + "classification_desc": "Other Charitable Activities" + } + ], + "CharityAoOCountryContinent": [ + { + "country": "Scotland", + "continent": "Europe" + }, + { + "country": "India", + "continent": "Asia" + }, + { + "country": "Ireland", + "continent": "Europe" + }, + { + "country": "United States", + "continent": "North America" + } + ], + "CharityAoOLocalAuthority": [ + + ], + "CharityAoORegion": [ + { + "region": "Throughout England And Wales" + } + ], + "other_names": [ + + ], + "constituency_name": [ + { + "constituency_name": "Cambridge" + } + ] + }, + "filings": [ + { + "ar_cycle_reference": "AR18", + "financial_period_end_date": "2018-12-31T00:00:00", + "income": 31454406.0, + "expenditure": 27891494.0, + "consolidated_account": true, + "charity_only_account": null, + "income_from_govt_contracts": 421031.0, + "income_from_govt_grants": null, + "inc_donations_and_legacies": 2309738.0, + "inc_other_trading_activities": 27943854.0, + "inc_charitable_activities": 0.0, + "inc_endowments": 0.0, + "inc_legacies": 0.0, + "inc_investment": 432353.0, + "inc_other": 768461.0, + "inc_total": 31454406.0, + "exp_charitable_activities": 7700745.0, + "exp_raising_funds": 20189905.0, + "exp_governance": 46150.0, + "exp_grants_institution": 0.0, + "exp_investment_management": 844.0, + "exp_other": 844.0, + "exp_total": 27891494.0 + }, + { + "ar_cycle_reference": "AR19", + "financial_period_end_date": "2019-12-31T00:00:00", + "income": 45790187.0, + "expenditure": 44205758.0, + "consolidated_account": true, + "charity_only_account": null, + "income_from_govt_contracts": 3517858.0, + "income_from_govt_grants": null, + "inc_donations_and_legacies": 1834170.0, + "inc_other_trading_activities": 39563538.0, + "inc_charitable_activities": 0.0, + "inc_endowments": 0.0, + "inc_legacies": 0.0, + "inc_investment": 542201.0, + "inc_other": 3850278.0, + "inc_total": 45790187.0, + "exp_charitable_activities": 10001366.0, + "exp_raising_funds": 34204392.0, + "exp_governance": 73713.0, + "exp_grants_institution": 0.0, + "exp_investment_management": 8820.0, + "exp_other": 0.0, + "exp_total": 44205758.0 + }, + { + "ar_cycle_reference": "AR20", + "financial_period_end_date": "2020-12-31T00:00:00", + "income": 95818848.0, + "expenditure": 88719833.0, + "consolidated_account": true, + "charity_only_account": null, + "income_from_govt_contracts": 3412581.0, + "income_from_govt_grants": null, + "inc_donations_and_legacies": 1634873.0, + "inc_other_trading_activities": 71436388.0, + "inc_charitable_activities": 0.0, + "inc_endowments": 0.0, + "inc_legacies": 0.0, + "inc_investment": 403176.0, + "inc_other": 22344411.0, + "inc_total": 95818848.0, + "exp_charitable_activities": 9488493.0, + "exp_raising_funds": 79231340.0, + "exp_governance": 79407.0, + "exp_grants_institution": 0.0, + "exp_investment_management": 6780.0, + "exp_other": 0.0, + "exp_total": 88719833.0 + }, + { + "ar_cycle_reference": "AR21", + "financial_period_end_date": "2021-12-31T00:00:00", + "income": 112663267.0, + "expenditure": 101823872.0, + "consolidated_account": true, + "charity_only_account": null, + "income_from_govt_contracts": 3838115.0, + "income_from_govt_grants": null, + "inc_donations_and_legacies": 6595649.0, + "inc_other_trading_activities": 101307210.0, + "inc_charitable_activities": 0.0, + "inc_endowments": 0.0, + "inc_legacies": 0.0, + "inc_investment": 279837.0, + "inc_other": 4480571.0, + "inc_total": 112663267.0, + "exp_charitable_activities": 11260781.0, + "exp_raising_funds": 90563091.0, + "exp_governance": 0.0, + "exp_grants_institution": 0.0, + "exp_investment_management": 0.0, + "exp_other": 0.0, + "exp_total": 101823872.0 + }, + { + "ar_cycle_reference": "AR22", + "financial_period_end_date": "2022-12-31T00:00:00", + "income": 157262030.0, + "expenditure": 151064960.0, + "consolidated_account": true, + "charity_only_account": null, + "income_from_govt_contracts": 3104511.0, + "income_from_govt_grants": null, + "inc_donations_and_legacies": 2475529.0, + "inc_other_trading_activities": 149803245.0, + "inc_charitable_activities": 0.0, + "inc_endowments": 0.0, + "inc_legacies": 0.0, + "inc_investment": 319053.0, + "inc_other": 4664203.0, + "inc_total": 157262030.0, + "exp_charitable_activities": 11678412.0, + "exp_raising_funds": 139386548.0, + "exp_governance": 228298.0, + "exp_grants_institution": 163872.0, + "exp_investment_management": 0.0, + "exp_other": 0.0, + "exp_total": 151064960.0 + } + ], + "parseDate": "2023-10-27T16:04:56+00:00", + "data_source": "https://api.charitycommission.gov.uk", + "api_version": "2" +} \ No newline at end of file diff --git a/assets/ruby/foundation_reporter.rb b/assets/ruby/foundation_reporter.rb index 04e9f2b..d13227b 100644 --- a/assets/ruby/foundation_reporter.rb +++ b/assets/ruby/foundation_reporter.rb @@ -1,7 +1,10 @@ #!/usr/bin/env ruby module FoundationReporter DESCRIPTION = <<-HEREDOC - FoundationReporter: Various reporting utilities. + FoundationReporter: Various reporting utilities, includes: + - Simple field and data reporting + - Using Propublica990 module to download US IRS 990 financial data + - Using UK Charity commission to download UK financial data HEREDOC module_function require 'yaml' @@ -10,6 +13,7 @@ module FoundationReporter require 'pathname' require '../propublica990/propublica990' require 'optparse' + require 'faraday' DATA_DIRS = { 'foundations' => File.join(Dir.pwd, '_foundations'), @@ -91,6 +95,39 @@ def field_usage(dataset) return report end + # Fetch a full report of a single UK charity + # NOTE: various hardcoded URLs and identifiers for UK + # @param registeredNumber of charity from taxID field + # @param apiToken to access charitycommission.gov.uk + # @return UK charities data combined hash + def fetch_ukorg(registeredNumber, apiToken) + org = {} + ukCharities = 'https://api.charitycommission.gov.uk' + faraday = Faraday.new(url: ukCharities) do |config| + config.response :raise_error + config.response :json, :content_type => /\bjson$/ + config.adapter :net_http + end + faraday.headers['Ocp-Apim-Subscription-Key'] = apiToken + response = faraday.get("/register/api/allcharitydetailsV2/#{registeredNumber}/0") + org['organization'] = response.body + response = faraday.get("/register/api/charityfinancialhistory/#{registeredNumber}/0") + org['filings'] = response.body + org['parseDate'] = DateTime.parse(org['organization']['last_modified_time']) + org['data_source'] = ukCharities + org['api_version'] = '2' + return org + end + + # Load secrets/API keys from a local file + # @param filename to read secrets from + # @return relevant apikey TODO: generalize for other cases? + def get_secrets(filename) + json = JSON.parse(File.read(filename)) + apikey = json['apikey'] + return apikey + end + # ## ### #### ##### ###### # Check commandline options def parse_commandline @@ -100,9 +137,6 @@ def parse_commandline opts.on('-oOUTFILE', '--out OUTFILE', 'Output filename for operation') do |out| options[:out] = out end - opts.on('-iINFILE', '--in INFILE', 'Input filename for operation') do |infile| - options[:infile] = infile - end opts.on('-fFIELDNAME', '--field FIELDNAME', 'Single field name to report out for all foundations.') do |onefield| options[:onefield] = onefield end @@ -112,6 +146,9 @@ def parse_commandline opts.on('-rREPORT', '--report REPORT', 'Output default reports.') do |reports| options[:reports] = reports end + opts.on('-uUKCHARITY', '--uk UKCHARITY', 'Download a single UK charity report.') do |ukorg| + options[:ukorg] = ukorg + end begin opts.parse! rescue OptionParser::ParseError => e @@ -128,7 +165,16 @@ def parse_commandline # Main method for command line use if __FILE__ == $PROGRAM_NAME options = FoundationReporter.parse_commandline - options[:outfile] ||= 'foundation_reporter.json' + ukorg = options.fetch(:ukorg, nil) + if ukorg + outfile = File.join(FoundationReporter::DATA_DIRS['taxes'], "uk-#{ukorg}.json") + output = FoundationReporter.fetch_ukorg(ukorg, FoundationReporter.get_secrets('../fossfoundation-api.json')) + File.open(outfile, "w") do |f| + f.write(JSON.pretty_generate(output)) + end + puts "Done, wrote out: #{outfile}" + exit 0 + end ctype = options.fetch(:ctype, nil) if ctype @@ -148,6 +194,7 @@ def parse_commandline end reports = options.fetch(:reports, nil) + options[:outfile] ||= 'foundation_reporter.json' if reports eins = FoundationReporter.get_eins(FoundationReporter::DATA_DIRS['foundations']) orgs = Propublica990.get_orgs(eins, '_data/p990', refresh = true) diff --git a/assets/ruby/uk_charities.rb b/assets/ruby/uk_charities.rb deleted file mode 100644 index 5045608..0000000 --- a/assets/ruby/uk_charities.rb +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env ruby -module UKCharities - DESCRIPTION = <<-HEREDOC - UKCharities: Lookup charity details from UK Register of Charities. - https://register-of-charities.charitycommission.gov.uk/documentation-on-the-api - HEREDOC - module_function - require 'yaml' - require 'json' - require 'csv' - require 'faraday' - require 'optparse' - - CHARITYCOMMISSION_HOST = '''https://api.charitycommission.gov.uk''' - CHARITYCOMMISSION_ACTION = '/register/api/allcharitydetailsV2/' - - # Get full report of a single charity - # @param registeredNumber of charity from taxID field - # @param apiToken to access charitycommission.gov.uk - # @return UK charities data - def get_charity(registeredNumber, apiToken) - charity = {} - faraday = Faraday.new(url: CHARITYCOMMISSION_HOST) do |config| - config.request :authorization, :bearer, apiToken - config.response :raise_error - config.adapter :net_http - end - faraday.headers['Authorization'] = "APIKEY #{apiToken}" - response = faraday.get("#{CHARITYCOMMISSION_ACTION}#{registeredNumber}") - puts "DEBUG #{response.status}" - puts "DEBUG #{JSON.pretty_generate(response.headers)}" - charity = response.body - return charity - end -end - -# ### #### ##### ###### -# Main method for command line use -if __FILE__ == $PROGRAM_NAME - # options = UKCharities.parse_commandline - options = {} - options[:outfile] ||= 'uk_charities.json' - options[:taxID] ||= '1129409' - fixme = 'api key would go here' - output = UKCharities.get_charity(options[:taxID], fixme) - File.open(options[:outfile], "w") do |f| - f.write(JSON.pretty_generate(output)) - end -end