Python duplication support, fix reporting

* Adds ability to transform Python -> AST -> JSON -> S-Expression for flay.
* Runs `Flay#report` only once now to fix exponential issue bug.
* Only grabs the first reported location from flay and stores the rest as
  `other_locations`.
* Now returns `end` line instead of the current line for both beginning
  and end.
* Uses `_type` in the Python script to return JSON-ified AST since
  `type` is used by the AST.

A large portion of this code was copied from the JavaScript and Ruby
analyzers as well as a private Python repo.

Author: BlakeWilliams

Dockerfile

@@ -8,8 +8,8 @@ COPY Gemfile.lock /usr/src/app/
 COPY vendor/php-parser/composer.json /usr/src/app/vendor/php-parser/
 COPY vendor/php-parser/composer.lock /usr/src/app/vendor/php-parser/
 
-RUN apk --update add nodejs ruby ruby-io-console ruby-dev ruby-bundler build-base \
-    php-cli php-json php-phar php-openssl php-xml curl && \
+RUN apk --update add python nodejs php-cli php-json php-phar php-openssl php-xml curl\
+    ruby ruby-io-console ruby-dev ruby-bundler build-base && \
     bundle install -j 4 && \
     apk del build-base && rm -fr /usr/share/ri && \
     curl -sS https://getcomposer.org/installer | php

Gemfile.lock

@@ -39,6 +39,3 @@ DEPENDENCIES
   pry
   rake
   sexp_processor
-
-BUNDLED WITH
-   1.10.6

bin/duplication

@@ -9,6 +9,8 @@ else
   engine_config = {}
 end
 
+directory = ARGV[0] || "/code"
+
 CC::Engine::Duplication.new(
-  directory: "/code", engine_config: engine_config, io: STDOUT
+  directory: directory, engine_config: engine_config, io: STDOUT
 ).run

lib/cc/engine/analyzers/helpers/main.rb

@@ -6,15 +6,30 @@ module Analyzers
       module Helpers
         BASE_POINTS = 10_000
 
-        def new_violation(issue, location, other)
+        def parsed_hashes
+          @parsed_hashes ||= []
+        end
+
+        def flay
+          @flay ||= ::Flay.new(flay_options)
+        end
+
+        def report
+          flay.report(StringIO.new).each do |issue|
+            location = issue.locations.first
+            io.puts "#{new_violation(issue, location).to_json}\0"
+          end
+        end
+
+        def new_violation(issue, location)
           {
             "type": "issue",
             "check_name": name(issue),
             "description": "Duplication found in #{issue.name}",
             "categories": ["Duplication"],
-            "location": format_location(location),
+            "location": format_location(issue, location),
             "remediation_points": calculate_points(issue),
-            "other_locations": format_locations(other),
+            "other_locations": format_locations(issue, location),
             "content": content_body
           }
         end
@@ -31,18 +46,42 @@ def find_other_locations(all_locations, current)
           all_locations.reject { |location| location == current }
         end
 
-        def format_location(location)
+        def format_location(issue, location)
+          current_sexp = flay.hashes[issue.structural_hash].detect do |sexp|
+            sexp.line == location.line
+          end
+
+          format_sexp(current_sexp)
+        end
+
+        def format_locations(issue, location)
+          sexps = flay.hashes[issue.structural_hash].reject do |sexp|
+            sexp.line == location.line
+          end
+
+          sexps.map do |sexp|
+            format_sexp(sexp)
+          end
+        end
+
+        def format_sexp(sexp)
           {
-            "path": local_path(location.file),
+            "path": local_path(sexp.file),
             "lines": {
-              "begin": location.line,
-              "end": location.line
+              "begin": sexp.line,
+              "end": sexp_max_line(sexp, sexp.line)
             }
           }
         end
 
-        def format_locations(other)
-          other.map { |location| format_location(location) }
+        def sexp_max_line(sexp_tree, default)
+          max = default
+
+          sexp_tree.deep_each do |sexp|
+            max = sexp.line if sexp.line > max
+          end
+
+          max
         end
 
         def flay_options
@@ -64,7 +103,7 @@ def local_path(file)
         end
 
         def directory_path
-          @directory_path ||= Pathname.new(@directory).realpath.to_s
+          @directory_path ||= Pathname.new(@directory).to_s
         end
 
         def excluded_files

lib/cc/engine/analyzers/javascript/main.rb

@@ -42,15 +42,6 @@ def mass_threshold
           def start_flay(s_expressions)
             flay = ::Flay.new(flay_options)
             flay.process_sexp(s_expressions)
-
-            flay.report(StringIO.new).each do |issue|
-              all_locations = issue.locations
-
-              all_locations.each do |location|
-                other_locations = find_other_locations(all_locations, location)
-                io.puts "#{new_violation(issue, location, other_locations).to_json}\0"
-              end
-            end
           end
 
           def analyzed_files

lib/cc/engine/analyzers/javascript/parser.rb

@@ -23,6 +23,8 @@ def parse
           def on_success(output)
             parsed_json = JSON.parse(output, max_nesting: false)
             parsed_json.delete('sourceType')
+
+
             @syntax_tree = CC::Engine::Analyzers::Javascript::AST.json_to_ast(parsed_json, filename)
           end
 

lib/cc/engine/analyzers/python/main.rb

@@ -0,0 +1,51 @@
+require 'cc/engine/analyzers/helpers/main'
+require 'cc/engine/analyzers/python/parser'
+require 'cc/engine/analyzers/python/node'
+require 'flay'
+
+module CC
+  module Engine
+    module Analyzers
+      module Python
+        class Main
+          include ::CC::Engine::Analyzers::Helpers
+
+          attr_reader :directory, :engine_config, :io
+
+          def initialize(directory:, engine_config:, io:)
+            @directory = directory
+            @engine_config = engine_config || {}
+            @io = io
+          end
+
+          def run
+            files_to_analyze.each do |file|
+              start_flay(process_file(file))
+            end
+          end
+
+          def process_file(path)
+            Node.new(::CC::Engine::Analyzers::Python::Parser.new(File.binread(path), path).parse.syntax_tree, path).format
+          end
+
+          def mass_threshold
+            engine_config.fetch('config', {}).fetch('python', {}).fetch('mass_threshold', 50)
+          end
+
+          def start_flay(s_expressions)
+            return if s_expressions.nil?
+            flay.process_sexp(s_expressions)
+          end
+
+          def files_to_analyze
+            files = Dir.glob("#{directory}/**/*.py").reject do |f|
+              File.directory?(f)
+            end
+
+            files - excluded_files
+          end
+        end
+      end
+    end
+  end
+end

lib/cc/engine/analyzers/python/node.rb

@@ -0,0 +1,72 @@
+module CC
+  module Engine
+    module Analyzers
+      module Python
+        class Node
+          SCRUB_PROPERTIES = ["_type", "attributes"].freeze
+
+          def initialize(node, file, default_line = 0)
+            @node = node
+            @file = file
+
+            set_default_line(default_line)
+          end
+
+          def format
+            if @node.is_a?(Hash)
+              type = @node["_type"].to_sym
+
+              if valid_properties
+                create_sexp(type, *properties_to_sexps)
+              else
+                type
+              end
+            elsif @node.is_a?(Array)
+              @node.map do |n|
+                Node.new(n, @file, @line).format
+              end
+            end
+          end
+
+          private
+
+          def properties_to_sexps
+            valid_properties.map do |key, value|
+              if value.is_a?(Array) || value.is_a?(Hash)
+                create_sexp(key.to_sym, *Node.new(value, @file, @line).format)
+              else
+                create_sexp(key.to_sym, value)
+              end
+            end
+          end
+
+          def create_sexp(*args)
+            Sexp.new(*args).tap do |sexp|
+              sexp.file = @file
+              sexp.line = @line
+            end
+          end
+
+          def valid_properties
+            @node.reject do |key, value|
+              value_empty = value == nil || value == {} || value == []
+              SCRUB_PROPERTIES.include?(key) || value_empty
+            end
+          end
+
+          def set_default_line(default)
+            if has_line_number?
+              @line = @node["attributes"]["lineno"]
+            else
+              @line = default
+            end
+          end
+
+          def has_line_number?
+            @node.is_a?(Hash) && @node["attributes"] && @node["attributes"]["lineno"]
+          end
+        end
+      end
+    end
+  end
+end

lib/cc/engine/analyzers/python/parser.py

@@ -0,0 +1,34 @@
+import json, sys, ast
+
+def to_json(node):
+    json_ast = {'attributes': {}}
+    json_ast['_type'] = node.__class__.__name__
+    for key, value in ast.iter_fields(node):
+        json_ast[key] = cast_value(value)
+    for attr in node._attributes:
+        json_ast['attributes'][attr] = cast_value(getattr(node, attr))
+    return json_ast
+
+def cast_infinity(value):
+    if value > 0:
+        return "Infinity"
+    else:
+        return "-Infinity"
+
+def cast_value(value):
+    if value is None or isinstance(value, (bool, basestring)):
+        return value
+    elif isinstance(value, (int, float, long, complex)):
+        if abs(value) == 1e3000:
+            return cast_infinity(value)
+        return value
+    elif isinstance(value, list):
+        return [cast_value(v) for v in value]
+    else:
+        return to_json(value)
+
+if __name__ == '__main__':
+    source = ""
+    for line in sys.stdin.readlines():
+        source += line
+    print json.dumps(to_json(ast.parse(source)))

lib/cc/engine/analyzers/python/parser.rb

@@ -0,0 +1,56 @@
+require 'posix/spawn'
+require 'timeout'
+require 'json'
+
+module CC
+  module Engine
+    module Analyzers
+      module Python
+        class Parser
+          attr_reader :code, :filename, :syntax_tree
+
+          def initialize(code, filename)
+            @code = code
+            @filename = filename
+          end
+
+          def parse
+            runner = CommandLineRunner.new(python_env, self)
+            runner.run(code) do |ast|
+              json_ast = JSON.parse(ast)
+              @syntax_tree = json_ast
+            end
+
+            self
+          end
+
+          def python_env
+            file = File.expand_path(File.dirname(__FILE__)) + '/parser.py'
+            "python #{file}"
+          end
+        end
+
+        class CommandLineRunner
+          DEFAULT_TIMEOUT = 20
+
+          attr_reader :command, :delegate
+
+          def initialize(command, delegate)
+            @command = command
+            @delegate = delegate
+          end
+
+          def run(input, timeout = DEFAULT_TIMEOUT)
+            Timeout.timeout(timeout) do
+              child = ::POSIX::Spawn::Child.new(command, input: input, timeout: timeout)
+
+              if child.status.success?
+                yield child.out if block_given?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/cc/engine/analyzers/ruby/main.rb

@@ -18,15 +18,6 @@ def initialize(directory:, engine_config:, io:)
           def run
             flay = ::Flay.new(flay_options)
             flay.process(*analyzed_files)
-
-            flay.report(StringIO.new).each do |issue|
-              all_locations = issue.locations
-
-              all_locations.each do |location|
-                other_locations = find_other_locations(all_locations, location)
-                io.puts "#{new_violation(issue, location, other_locations).to_json}\0"
-              end
-            end
           end
 
           private