Move JavaScript to parser backend

Author: maxjacobson

Dockerfile

@@ -13,12 +13,11 @@ RUN apt-get update && \
   RUNLEVEL=1 apt-get install --yes --no-install-recommends \
     python2.7
 
-COPY Gemfile* package.json package-lock.json ./
+COPY Gemfile* ./
 COPY vendor/php-parser/composer* ./vendor/php-parser/
 
 RUN bundle install --jobs 4 --quiet && \
-    composer install --no-interaction --quiet --working-dir ./vendor/php-parser && \
-    npm install
+    composer install --no-interaction --quiet --working-dir ./vendor/php-parser
 
 COPY . ./
 RUN chown -R app:app ./

lib/cc/engine/analyzers/analyzer_base.rb

@@ -2,6 +2,9 @@
 
 require "cc/engine/analyzers/parser_error"
 require "cc/engine/analyzers/parser_base"
+require "cc/engine/analyzers/file_list"
+require "cc/engine/processed_source"
+require "cc/engine/sexp_builder"
 
 module CC
   module Engine
@@ -120,6 +123,35 @@ def file_list
         def skip?(_path)
           nil
         end
+
+        def parse(file, request_path)
+          processed_source = ProcessedSource.new(file, request_path)
+
+          SexpBuilder.new(processed_source.ast, file).build
+        rescue CC::Parser::Client::HTTPError => ex
+          unexpected_parse_error!(ex) unless ex.response_status.to_s.start_with?("4")
+
+          CC.logger.warn("Skipping #{processed_source.path} due to #{ex.class}")
+          CC.logger.warn("Response status: #{ex.response_status}")
+          CC.logger.debug { "Contents:\n#{processed_source.raw_source}" }
+          CC.logger.debug { "Response:\n#{ex.response_body}" }
+          nil
+        rescue CC::Parser::Client::NestingDepthError => ex
+          CC.logger.warn("Skipping #{processed_source.path} due to #{ex.class}")
+          CC.logger.warn(ex.message)
+          CC.logger.debug { "Contents:\n#{processed_source.raw_source}" }
+          nil
+        rescue => ex
+          unexpected_parse_error!(ex)
+        end
+
+        def unexpected_parse_error!(ex)
+          CC.logger.error("Error processing file: #{processed_source.path}")
+          CC.logger.error(ex.message)
+          CC.logger.debug { "Contents:\n#{processed_source.raw_source}" }
+
+          raise
+        end
       end
     end
   end

lib/cc/engine/analyzers/java/main.rb

@@ -4,8 +4,6 @@
 require "json"
 require "cc/engine/analyzers/reporter"
 require "cc/engine/analyzers/analyzer_base"
-require "cc/engine/processed_source"
-require "cc/engine/sexp_builder"
 
 module CC
   module Engine
@@ -21,42 +19,16 @@ class Main < CC::Engine::Analyzers::Base
           ].freeze
           POINTS_PER_OVERAGE = 10_000
           REQUEST_PATH = "/java".freeze
-          TIMEOUT = 300
 
           private
 
           def process_file(file)
-            processed_source = ProcessedSource.new(file, REQUEST_PATH)
-
-            SexpBuilder.new(processed_source.ast, file).build
-          rescue => ex
-            if unparsable_file_error?(ex)
-              CC.logger.warn("Skipping #{processed_source.path} due to #{ex.class}")
-              CC.logger.warn("Response status: #{ex.response_status}")
-              CC.logger.debug { "Contents:\n#{processed_source.raw_source}" }
-              CC.logger.debug { "Response:\n#{ex.response_body}" }
-              nil
-            elsif ex.is_a?(CC::Parser::Client::NestingDepthError)
-              CC.logger.warn("Skipping #{processed_source.path} due to #{ex.class}")
-              CC.logger.warn(ex.message)
-              CC.logger.debug { "Contents:\n#{processed_source.raw_source}" }
-              nil
-            else
-              CC.logger.error("Error processing file: #{processed_source.path}")
-              CC.logger.error(ex.message)
-              CC.logger.debug { "Contents:\n#{processed_source.raw_source}" }
-              raise
-            end
+            parse(file, REQUEST_PATH)
           end
 
           def default_filters
             DEFAULT_FILTERS.map { |filter| Sexp::Matcher.parse filter }
           end
-
-          def unparsable_file_error?(ex)
-            ex.is_a?(CC::Parser::Client::HTTPError) &&
-              ex.response_status.to_s.start_with?("4")
-          end
         end
       end
     end

lib/cc/engine/analyzers/javascript/main.rb

@@ -1,12 +1,6 @@
 # frozen_string_literal: true
 
 require "cc/engine/analyzers/analyzer_base"
-require "cc/engine/analyzers/javascript/parser"
-require "cc/engine/analyzers/javascript/minification_checker"
-require "cc/engine/analyzers/javascript/node"
-require "cc/engine/analyzers/file_list"
-require "flay"
-require "json"
 
 module CC
   module Engine
@@ -20,26 +14,12 @@ class Main < CC::Engine::Analyzers::Base
           LANGUAGE = "javascript"
           DEFAULT_MASS_THRESHOLD = 40
           POINTS_PER_OVERAGE = 30_000
-
-          def transform_sexp(sexp)
-            sexp.flatter
-          end
+          REQUEST_PATH = "/javascript".freeze
 
           private
 
-          def process_file(path)
-            ast = js_parser.new(File.read(path), path).parse
-            Node.new(ast.syntax_tree, path).format if ast
-          end
-
-          def js_parser
-            ::CC::Engine::Analyzers::Javascript::Parser
-          end
-
-          def skip?(path)
-            if MinificationChecker.new(path).minified?
-              "the file is minified"
-            end
+          def process_file(file)
+            parse(file, REQUEST_PATH)
           end
         end
       end

lib/cc/engine/analyzers/javascript/minification_checker.rb

@@ -3,7 +3,6 @@
 require "cc/engine/analyzers/python/parser"
 require "cc/engine/analyzers/python/node"
 require "cc/engine/analyzers/reporter"
-require "cc/engine/analyzers/file_list"
 require "cc/engine/analyzers/analyzer_base"
 require "flay"