From 451726719241eaabfb92cee59f96989c87c95558 Mon Sep 17 00:00:00 2001 From: Will Fleming Date: Thu, 6 Oct 2016 15:17:11 -0400 Subject: [PATCH 1/4] install python3 in dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c6cd6d1b..3f8e3d7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ COPY vendor/php-parser/composer.lock /usr/src/app/vendor/php-parser/ COPY package.json /usr/src/app/ RUN curl --silent --location https://deb.nodesource.com/setup_5.x | bash - && \ - apt-get update && apt-get install -y nodejs python openssh-client php5-cli php5-json + apt-get update && apt-get install -y nodejs python python3 openssh-client php5-cli php5-json RUN gem install bundler --no-ri --no-rdoc && \ bundle install -j 4 && \ curl -sS https://getcomposer.org/installer | php From 8be471029ed2aa1d583fa42b8e736a7e0dd0d15f Mon Sep 17 00:00:00 2001 From: Will Fleming Date: Thu, 6 Oct 2016 15:26:05 -0400 Subject: [PATCH 2/4] Refactor: make analyzer available to parser, break up some methods --- lib/cc/engine/analyzers/python/main.rb | 6 +++++- lib/cc/engine/analyzers/python/parser.rb | 13 +++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/cc/engine/analyzers/python/main.rb b/lib/cc/engine/analyzers/python/main.rb index 320ee686..87d008e6 100644 --- a/lib/cc/engine/analyzers/python/main.rb +++ b/lib/cc/engine/analyzers/python/main.rb @@ -18,7 +18,11 @@ class Main < CC::Engine::Analyzers::Base private def process_file(path) - Node.new(::CC::Engine::Analyzers::Python::Parser.new(File.binread(path), path).parse.syntax_tree, path).format + Node.new(parser(path).parse.syntax_tree, path).format + end + + def parser(path) + ::CC::Engine::Analyzers::Python::Parser.new(self, File.binread(path), path) end end end diff --git a/lib/cc/engine/analyzers/python/parser.rb b/lib/cc/engine/analyzers/python/parser.rb index ae7c93a1..acbf6fe9 100644 --- a/lib/cc/engine/analyzers/python/parser.rb +++ b/lib/cc/engine/analyzers/python/parser.rb @@ -9,7 +9,8 @@ module Python class Parser < ParserBase attr_reader :code, :filename, :syntax_tree - def initialize(code, filename) + def initialize(analyzer, code, filename) + @analyzer = analyzer @code = code @filename = filename end @@ -23,9 +24,17 @@ def parse self end + private + + attr_reader :analyzer + def python_command file = File.expand_path(File.dirname(__FILE__)) + '/parser.py' - "python #{file}" + "#{python_binary} #{file}" + end + + def python_binary + "python2" end end end From e2c489f7a47d462e8a4889e907fc0ce8de96af78 Mon Sep 17 00:00:00 2001 From: Will Fleming Date: Thu, 6 Oct 2016 16:14:57 -0400 Subject: [PATCH 3/4] Support python3 from config * Support a `python_version` value under `python` key in config * Use `python3` when this is set appropriately * Update the `parser.py` script to work in both versions --- lib/cc/engine/analyzers/python/main.rb | 6 ++- lib/cc/engine/analyzers/python/parser.py | 17 ++++++-- lib/cc/engine/analyzers/python/parser.rb | 17 +++++--- spec/cc/engine/analyzers/python/main_spec.rb | 42 ++++++++++++++++++++ 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/lib/cc/engine/analyzers/python/main.rb b/lib/cc/engine/analyzers/python/main.rb index 87d008e6..5c313011 100644 --- a/lib/cc/engine/analyzers/python/main.rb +++ b/lib/cc/engine/analyzers/python/main.rb @@ -22,7 +22,11 @@ def process_file(path) end def parser(path) - ::CC::Engine::Analyzers::Python::Parser.new(self, File.binread(path), path) + ::CC::Engine::Analyzers::Python::Parser.new(python_version, File.binread(path), path) + end + + def python_version + engine_config.languages.fetch("python", {}).fetch("python_version", 2) end end end diff --git a/lib/cc/engine/analyzers/python/parser.py b/lib/cc/engine/analyzers/python/parser.py index 8444a309..88e14a8b 100644 --- a/lib/cc/engine/analyzers/python/parser.py +++ b/lib/cc/engine/analyzers/python/parser.py @@ -1,5 +1,16 @@ import json, sys, ast +PY3 = sys.version_info[0] == 3 + +def string_type(): + return str if PY3 else basestring + +def num_types(): + if PY3: + return (int, float, complex) + else: + return (int, float, long, complex) + def to_json(node): json_ast = {'attributes': {}} json_ast['_type'] = node.__class__.__name__ @@ -16,9 +27,9 @@ def cast_infinity(value): return "-Infinity" def cast_value(value): - if value is None or isinstance(value, (bool, basestring)): + if value is None or isinstance(value, (bool, string_type())): return value - elif isinstance(value, (int, float, long, complex)): + elif isinstance(value, num_types()): if abs(value) == 1e3000: return cast_infinity(value) return value @@ -31,4 +42,4 @@ def cast_value(value): source = "" for line in sys.stdin.readlines(): source += line - print json.dumps(to_json(ast.parse(source))) + print(json.dumps(to_json(ast.parse(source)))) diff --git a/lib/cc/engine/analyzers/python/parser.rb b/lib/cc/engine/analyzers/python/parser.rb index acbf6fe9..4a61184c 100644 --- a/lib/cc/engine/analyzers/python/parser.rb +++ b/lib/cc/engine/analyzers/python/parser.rb @@ -9,8 +9,8 @@ module Python class Parser < ParserBase attr_reader :code, :filename, :syntax_tree - def initialize(analyzer, code, filename) - @analyzer = analyzer + def initialize(python_version, code, filename) + @python_version = python_version @code = code @filename = filename end @@ -26,15 +26,22 @@ def parse private - attr_reader :analyzer + attr_reader :python_version def python_command - file = File.expand_path(File.dirname(__FILE__)) + '/parser.py' + file = File.expand_path(File.dirname(__FILE__)) + "/parser.py" "#{python_binary} #{file}" end def python_binary - "python2" + case python_version + when 2, "2" + "python2" + when 3, "3" + "python3" + else + raise ArgumentError, "Supported python versions are 2 and 3. You configured: #{python_version.inspect}" + end end end end diff --git a/spec/cc/engine/analyzers/python/main_spec.rb b/spec/cc/engine/analyzers/python/main_spec.rb index 2580cff6..d62e2c01 100644 --- a/spec/cc/engine/analyzers/python/main_spec.rb +++ b/spec/cc/engine/analyzers/python/main_spec.rb @@ -63,6 +63,48 @@ expect(json["fingerprint"]).to eq("019118ceed60bf40b35aad581aae1b02") end + it "finds duplication in python3 code" do + create_source_file("foo.py", <<-EOJS) +def a(thing: str): + print("Hello", str) + +def b(thing: str): + print("Hello", str) + +def c(thing: str): + print("Hello", str) + EOJS + + conf = CC::Engine::Analyzers::EngineConfig.new({ + "config" => { + "languages" => { + "python" => { + "mass_threshold" => 4, + "python_version" => 3 + } + } + } + }) + issues = run_engine(conf).strip.split("\0") + result = issues.first.strip + json = JSON.parse(result) + + expect(json["type"]).to eq("issue") + expect(json["check_name"]).to eq("Similar code") + expect(json["description"]).to eq("Similar code found in 2 other locations (mass = 16)") + expect(json["categories"]).to eq(["Duplication"]) + expect(json["location"]).to eq({ + "path" => "foo.py", + "lines" => { "begin" => 1, "end" => 2 }, + }) + expect(json["remediation_points"]).to eq(2_100_000) + expect(json["other_locations"]).to eq([ + {"path" => "foo.py", "lines" => { "begin" => 4, "end" => 5 } }, + {"path" => "foo.py", "lines" => { "begin" => 7, "end" => 8 } } + ]) + expect(json["content"]["body"]).to match /This issue has a mass of 16/ + expect(json["fingerprint"]).to eq("607cf2d16d829e667c5f34534197d14c") + end it "skips unparsable files" do create_source_file("foo.py", <<-EOPY) From 0d0140bb94fa237e80ccb82c8ba40f1231c42ac7 Mon Sep 17 00:00:00 2001 From: Will Fleming Date: Fri, 7 Oct 2016 10:47:58 -0400 Subject: [PATCH 4/4] extract default version to constant --- lib/cc/engine/analyzers/python/main.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cc/engine/analyzers/python/main.rb b/lib/cc/engine/analyzers/python/main.rb index 5c313011..e1d408fa 100644 --- a/lib/cc/engine/analyzers/python/main.rb +++ b/lib/cc/engine/analyzers/python/main.rb @@ -13,6 +13,7 @@ class Main < CC::Engine::Analyzers::Base LANGUAGE = "python" PATTERNS = ["**/*.py"] DEFAULT_MASS_THRESHOLD = 32 + DEFAULT_PYTHON_VERSION = 2 POINTS_PER_OVERAGE = 50_000 private @@ -26,7 +27,7 @@ def parser(path) end def python_version - engine_config.languages.fetch("python", {}).fetch("python_version", 2) + engine_config.languages.fetch("python", {}).fetch("python_version", DEFAULT_PYTHON_VERSION) end end end