fix: Force utf-8 encoding in snyk description before join

rtfpessoa · Sep 1, 2019 · e237bc9 · e237bc9
1 parent c96dc44
commit e237bc9
Show file tree

Hide file tree

Showing 10 changed files with 65 additions and 66 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    yavdb (0.5.2)
+    yavdb (0.5.3)
       execjs (~> 2.7)
       json (~> 2.2)
       kramdown (~> 2.1)
@@ -16,12 +16,14 @@ GEM
   specs:
     ansi (1.5.0)
     ast (2.4.0)
-    bibliothecary (6.6.0)
+    bibliothecary (6.8.1)
       commander
       deb_control
       librariesio-gem-parser
       ox (>= 2.8.1)
       sdl4r
+      strings
+      strings-ansi
       toml-rb (~> 1.0)
       typhoeus
     citrus (3.0.2)
@@ -31,70 +33,75 @@ GEM
     commander (4.4.7)
       highline (~> 2.0.0)
     deb_control (0.0.1)
-    dependency_spy (0.4.1)
+    dependency_spy (0.5.0)
       bibliothecary (~> 6.6)
       colorize (= 0.8.1)
       semantic_range (~> 2.2)
       thor (~> 0.20)
       yavdb (~> 0.5)
     diff-lcs (1.3)
-    docile (1.3.1)
+    docile (1.3.2)
     ethon (0.12.0)
       ffi (>= 1.3.0)
     execjs (2.7.0)
-    ffi (1.11.0)
+    ffi (1.11.1)
     highline (2.0.2)
-    jaro_winkler (1.5.2)
+    jaro_winkler (1.5.3)
     json (2.2.0)
     kramdown (2.1.0)
     librariesio-gem-parser (1.0.0)
     libv8 (3.16.14.19)
     oga (2.15)
       ast
       ruby-ll (~> 2.1)
-    ox (2.10.0)
+    ox (2.11.0)
     parallel (1.17.0)
-    parser (2.6.3.0)
+    parser (2.6.4.0)
       ast (~> 2.4.0)
     rainbow (3.0.0)
-    rake (12.3.2)
+    rake (12.3.3)
     ref (2.0.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
+    rspec-core (3.8.2)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.3)
+    rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
+    rspec-support (3.8.2)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.69.0)
+    rubocop (0.74.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.6)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-rspec (1.33.0)
+    rubocop-rspec (1.35.0)
       rubocop (>= 0.60.0)
     ruby-ll (2.1.2)
       ansi
       ast
-    ruby-progressbar (1.10.0)
+    ruby-progressbar (1.10.1)
     sdl4r (0.9.11)
     semantic_interval (0.1.0)
     semantic_range (2.2.1)
-    simplecov (0.16.1)
+    simplecov (0.17.0)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    strings (0.1.6)
+      strings-ansi (~> 0.1)
+      unicode-display_width (~> 1.5)
+      unicode_utils (~> 1.4)
+    strings-ansi (0.1.0)
     therubyracer (0.12.3)
       libv8 (~> 3.16.14.15)
       ref
@@ -104,18 +111,19 @@ GEM
     typhoeus (1.3.1)
       ethon (>= 0.9.0)
     unicode-display_width (1.6.0)
+    unicode_utils (1.4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   codacy-coverage
-  dependency_spy (~> 0.4)
+  dependency_spy
   rake (~> 12.3)
   rspec (~> 3.8)
   rspec_junit_formatter (~> 0.4)
-  rubocop (~> 0.69)
-  rubocop-rspec (~> 1.33)
+  rubocop (~> 0.74)
+  rubocop-rspec (~> 1.35)
   simplecov
   yavdb!
 

diff --git a/lib/yavdb/database.rb b/lib/yavdb/database.rb
@@ -43,7 +43,6 @@ def group_by_package_manager(vulns)
         vulns
           .group_by(&:package_manager)
           .map do |package_manager, vunerabilities_by_pm|
-
           puts "#{package_manager}: #{vunerabilities_by_pm.length}"
 
           vunerabilities_by_pm =

diff --git a/lib/yavdb/sources/npmjs.rb b/lib/yavdb/sources/npmjs.rb
@@ -40,13 +40,13 @@ class << self
           def fetch_packages_recursive(page_number)
             page = get_page_html(get_page_url(page_number), false, 'npmjs/feed')
 
-            script_tag = page.css('script').find { |script| script.text.include?('window.__context__') }.text
-            context = ExecJS.compile("var window = {};\n#{script_tag.force_encoding('utf-8')};")
+            script_tag    = page.css('script').find { |script| script.text.include?('window.__context__') }.text
+            context       = ExecJS.compile("var window = {};\n#{script_tag.force_encoding('utf-8')};")
             advisory_data = context.exec('return window.__context__.context.advisoriesData')
 
             packages = advisory_data['objects']
 
-            next_url = advisory_data['urls']['next']
+            next_url      = advisory_data['urls']['next']
             next_packages = if next_url && !next_url&.include?("page=#{page_number}")
                               fetch_packages_recursive(page_number + 1)
                             else
@@ -62,7 +62,7 @@ def parse_vulnerabilities(packages)
 
           def create(package)
             published_date = Date.strptime(package['created'], '%s')
-            updated_date = Date.strptime(package['updated'], '%s')
+            updated_date   = Date.strptime(package['updated'], '%s')
 
             cves = package['cves'] || []
 
@@ -108,13 +108,13 @@ def get_page_url(page)
 
           def parse_severity(severity)
             case severity
-              when 'low' then
+              when 'low'
                 'low'
-              when 'moderate' then
+              when 'moderate'
                 'medium'
-              when 'high' then
+              when 'high'
                 'high'
-              when 'critical' then
+              when 'critical'
                 'high'
               else
                 'high'

diff --git a/lib/yavdb/sources/ruby_advisory.rb b/lib/yavdb/sources/ruby_advisory.rb
@@ -107,9 +107,9 @@ def severity(cvss_v2_score, cvss_v3_score)
 
           def severity_level(cvss_score)
             case cvss_score
-              when 0.0..3.3 then
+              when 0.0..3.3
                 'low'
-              when 3.3..6.6 then
+              when 3.3..6.6
                 'medium'
               else
                 'high'

diff --git a/lib/yavdb/sources/rustsec.rb b/lib/yavdb/sources/rustsec.rb
@@ -25,7 +25,7 @@ module Sources
     module RustSec
       class Client
 
-        REPOSITORY_URL = 'https://github.com/RustSec/advisory-db'.freeze
+        REPOSITORY_URL  = 'https://github.com/RustSec/advisory-db'.freeze
         PACKAGE_MANAGER = 'cargo'.freeze
 
         def self.advisories
@@ -44,9 +44,9 @@ class << self
           private
 
           def create(advisory_hash)
-            date = Date.strptime(advisory_hash['date'].to_s, '%Y-%m-%d')
-            severity = 'high' # since no value is provided will use highest
-            cve = advisory_hash['aliases']&.select { |a| a.start_with?('CVE') }
+            date       = Date.strptime(advisory_hash['date'].to_s, '%Y-%m-%d')
+            severity   = 'high' # since no value is provided will use highest
+            cve        = advisory_hash['aliases']&.select { |a| a.start_with?('CVE') }
             references = advisory_hash['url'] && [advisory_hash['url']]
 
             vuln_id = "rustsec:cargo:#{advisory_hash['package']}:#{advisory_hash['id']}"

diff --git a/lib/yavdb/sources/snyk_io.rb b/lib/yavdb/sources/snyk_io.rb
@@ -168,30 +168,22 @@ def parse_body(advisory_page)
               body   = section[:body]
 
               case header.text
-                when 'Overview' then
+                when %r{^(Overview|Details)$} then
                   overview_str = body
                                    .map(&:to_xml)
+                                   .map { |e| e.force_encoding('UTF-8') }
                                    .join("\n")
-                                   .force_encoding('UTF-8')
                   begin
-                    data[:description] += '\n' if data[:description]
-                    data[:description] = '' unless data[:description]
+                    if data[:description]
+                      data[:description] += '\n'
+                    else
+                      data[:description] = ''
+                    end
+
                     data[:description] += utf8(Kramdown::Document.new(overview_str, :html_to_native => true).to_kramdown)
                   rescue StandardError
                     # ignore
                   end
-                when 'Details' then
-                  details_str = body
-                                  .map(&:to_xml)
-                                  .join("\n")
-                                  .force_encoding('UTF-8')
-                  begin
-                    data[:description] += '\n' if data[:description]
-                    data[:description] = '' unless data[:description]
-                    data[:description] += utf8(Kramdown::Document.new(details_str, :html_to_native => true).to_kramdown)
-                  rescue StandardError
-                    # ignore
-                  end
                 when 'References' then
                   references = []
                   if body.any?
@@ -211,19 +203,19 @@ def parse_side_bar(advisory_page)
 
             advisory_page.css('.l-col .card .card__content dl > *').each_slice(2).to_a.map do |key, value|
               case key.text
-                when 'Credit' then
+                when 'Credit'
                   data[:credit] = utf8(value.text.split(',').map { |str| str.strip.sub(%r{-\s*}, '') }.reject(&:empty?))
-                when 'CVE' then
+                when 'CVE'
                   data[:cve] = value.css('a').map { |a| a.text.strip.split(',') }.flatten.map(&:strip).reject(&:empty?)
-                when 'CWE' then
+                when 'CWE'
                   data[:cwe] = value.css('a').map { |a| a.text.strip.split(',') }.flatten.map(&:strip).reject(&:empty?)
-                when 'Snyk ID' then
+                when 'Snyk ID'
                   data[:id] = value.text.strip
-                when 'Disclosed' then
+                when 'Disclosed'
                   data[:disclosed_date] = value.text.strip
-                when 'Published' then
+                when 'Published'
                   data[:published_date] = value.text.strip
-                when 'Last modified' then
+                when 'Last modified'
                   data[:last_modified_date] = value.text.strip
               end
             end

diff --git a/lib/yavdb/sources/victims.rb b/lib/yavdb/sources/victims.rb
@@ -95,9 +95,9 @@ def split_versions(versions)
 
           def severity(cvss_score)
             case cvss_score
-              when 0.0..3.3 then
+              when 0.0..3.3
                 'low'
-              when 3.3..6.6 then
+              when 3.3..6.6
                 'medium'
               else
                 'high'

diff --git a/lib/yavdb/utils/http.rb b/lib/yavdb/utils/http.rb
@@ -45,9 +45,9 @@ def do_request(url)
           begin
             response = Net::HTTP.get_response(url)
             case response
-              when Net::HTTPNotFound then
+              when Net::HTTPNotFound
                 raise ArgumentError, 'page not found'
-              when Net::HTTPTooManyRequests then
+              when Net::HTTPTooManyRequests
                 raise ArgumentError, 'too many requests'
               else
                 response.body.lines

diff --git a/lib/yavdb/version.rb b/lib/yavdb/version.rb
@@ -16,6 +16,6 @@
 
 module YAVDB
 
-  VERSION = '0.5.2'
+  VERSION = '0.5.3'
 
 end
diff --git a/yavdb.gemspec b/yavdb.gemspec
@@ -31,9 +31,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'simplecov'
 
   # Linters
-  spec.add_development_dependency 'dependency_spy', ['~> 0.4']
-  spec.add_development_dependency 'rubocop', ['~> 0.69']
-  spec.add_development_dependency 'rubocop-rspec', ['~> 1.33']
+  spec.add_development_dependency 'dependency_spy'
+  spec.add_development_dependency 'rubocop', ['~> 0.74']
+  spec.add_development_dependency 'rubocop-rspec', ['~> 1.35']
 
   # Runtime
   spec.add_runtime_dependency 'execjs', ['~> 2.7']