From 64fed363d2d64bd04271332b9f5ed87f5933f9ee Mon Sep 17 00:00:00 2001 From: Zakariyya Mughal Date: Tue, 18 May 2021 14:45:21 -0400 Subject: [PATCH 1/2] Get all CPAN distributions using scroll API for MetaCPAN This uses the ElasticSearch scroll API to get all CPAN distributions . Fixes . --- app/models/package_manager/cpan.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/models/package_manager/cpan.rb b/app/models/package_manager/cpan.rb index 371b7faf6..66ce9675c 100644 --- a/app/models/package_manager/cpan.rb +++ b/app/models/package_manager/cpan.rb @@ -13,16 +13,20 @@ def self.package_link(project, _version = nil) end def self.project_names - page = 1 projects = [] + size = 5000 + time = '1m' + scroll_start_r = get("https://fastapi.metacpan.org/v1/release/_search?scroll=#{time}&size=#{size}&q=status:latest&fields=distribution") + projects += scroll_start_r["hits"]["hits"] + scroll_id = scroll_start_r['_scroll_id'] loop do - r = get("https://fastapi.metacpan.org/v1/release/_search?q=status:latest&fields=distribution&sort=date:desc&size=5000&from=#{page * 5000}")["hits"]["hits"] - break if r == [] + r = get("https://fastapi.metacpan.org/v1/_search/scroll?scroll=#{time}&scroll_id=#{scroll_id}") + break if r["hits"]["hits"] == [] - projects += r - page += 1 + projects += r["hits"]["hits"] + scroll_id = r['_scroll_id'] end - projects.map { |project| project["fields"]["distribution"] }.uniq + projects.map { |project| project["fields"]["distribution"] }.flatten.uniq end def self.recent_names From aece71c6683ec37a487f57357b6fa9e44097c418 Mon Sep 17 00:00:00 2001 From: Zakariyya Mughal Date: Tue, 18 May 2021 14:58:32 -0400 Subject: [PATCH 2/2] Revert "Remove CPAN from missing task" This reverts commit 9c6d0f97352d03c7aca56ce4ac03cdd07a3e675f. Connects with . --- lib/tasks/projects.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/projects.rake b/lib/tasks/projects.rake index 8be1bdab3..03b73fab3 100644 --- a/lib/tasks/projects.rake +++ b/lib/tasks/projects.rake @@ -85,7 +85,7 @@ namespace :projects do desc 'Download missing packages' task download_missing: :environment do exit if ENV['READ_ONLY'].present? - ['Alcatraz', 'Bower', 'Cargo', 'Clojars', 'CocoaPods', 'CRAN', + ['Alcatraz', 'Bower', 'Cargo', 'Clojars', 'CocoaPods', 'CPAN', 'CRAN', 'Dub', 'Elm', 'Hackage', 'Haxelib', 'Hex', 'Homebrew', 'Inqlude', 'Julia', 'NPM', 'Packagist', 'Pypi', 'Rubygems'].each do |platform| "PackageManager::#{platform}".constantize.import_new_async rescue nil