From dfbafe8775035b07d19c02a40625773b676de4c6 Mon Sep 17 00:00:00 2001 From: Andy Blyler Date: Sat, 8 Aug 2015 21:17:34 -0400 Subject: [PATCH 1/6] add parallel phpmd processing and polish - use forking to process files in parallel (need for large code bases) - add support for specifing rulesets in the config --- .codeclimate.yml | 7 ++++- Dockerfile | 19 ++++++++---- JSONRenderer.php | 46 ++++++++++++++--------------- README.md | 36 ++++++++++++++++++++-- Runner.php | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ composer.json | 3 +- composer.lock | 40 +++++++++++++++++++++++-- engine.php | 68 +++++++++++++----------------------------- 8 files changed, 213 insertions(+), 83 deletions(-) create mode 100644 Runner.php diff --git a/.codeclimate.yml b/.codeclimate.yml index 7979c3c..e6d56e8 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,8 +1,13 @@ engines: fixme: - enabled: true + enabled: false phpmd: enabled: true + phpcodesniffer: + enabled: true + config: + - file_extensions: "php" + - standard: ["PSR1", "PSR2"] exclude_paths: - .git/**/* diff --git a/Dockerfile b/Dockerfile index e3c5b27..31b99e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,24 @@ -FROM alpine:edge +FROM phusion/baseimage WORKDIR /usr/src/app COPY composer.json /usr/src/app/ COPY composer.lock /usr/src/app/ -RUN apk --update add git php-common php-xml php-dom php-ctype php-iconv php-json php-phar php-openssl curl && \ - curl -sS https://getcomposer.org/installer | php && \ - /usr/src/app/composer.phar install && \ - apk del build-base && rm -fr /usr/share/ri +RUN apt-get update +RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y curl git +RUN add-apt-repository -y ppa:ondrej/php5-5.6 +RUN apt-get update +RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y --force-yes php5-cli -RUN adduser -u 9000 -D app +RUN curl -sS https://getcomposer.org/installer | php +RUN /usr/src/app/composer.phar install + +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN adduser --uid 9000 --disabled-password app USER app COPY . /usr/src/app CMD ["/usr/src/app/bin/codeclimate-phpmd"] + diff --git a/JSONRenderer.php b/JSONRenderer.php index 3a955b8..2bf9706 100644 --- a/JSONRenderer.php +++ b/JSONRenderer.php @@ -49,33 +49,33 @@ public function renderReport(Report $report) $writer = $this->getWriter(); foreach ($report->getRuleViolations() as $violation) { - $rule = $violation->getRule(); - $checkName = preg_replace("/^PHPMD\/Rule\//", "", str_replace("\\", "/", get_class($rule))); + $rule = $violation->getRule(); + $checkName = preg_replace("/^PHPMD\/Rule\//", "", str_replace("\\", "/", get_class($rule))); - $path = preg_replace("/^\/code\//", "", $violation->getFileName()); - $category = $this->ruleCategories[$checkName]; + $path = preg_replace("/^\/code\//", "", $violation->getFileName()); + $category = $this->ruleCategories[$checkName]; - if ($category == null) { - $category = "Style"; - } + if ($category == null) { + $category = "Style"; + } - $issue = array( - "type" => "issue", - "check_name" => $checkName, - "description" => $violation->getDescription(), - "categories" => array($category), - "location" => array( - "path" => $path, - "lines" => array( - "begin" => $violation->getBeginLine(), - "end" => $violation->getEndLine() - ) - ) - ); + $issue = array( + "type" => "issue", + "check_name" => $checkName, + "description" => $violation->getDescription(), + "categories" => array($category), + "location" => array( + "path" => $path, + "lines" => array( + "begin" => $violation->getBeginLine(), + "end" => $violation->getEndLine() + ) + ) + ); - $json = json_encode($issue, JSON_UNESCAPED_SLASHES, JSON_UNESCAPED_UNICODE); - $writer->write($json); - $writer->write(chr(0)); + $json = json_encode($issue, JSON_UNESCAPED_SLASHES, JSON_UNESCAPED_UNICODE); + $writer->write($json); + $writer->write(chr(0)); } } } diff --git a/README.md b/README.md index 3701877..254e06b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,36 @@ # Code Climate PHP Mess Detector (PHPMD) Engine -`codeclimate-phpmd` is a Code Climate Engine that wraps the PHP Mess -Detector (PHPMD) static analysis tool. +`codeclimate-phpmd` is a Code Climate Engine that wraps the PHP Mess Detector (PHPMD) static analysis tool. + +### Installation + +1. If you haven't already, [install the Code Climate CLI](https://github.com/codeclimate/codeclimate). +2. Run `codeclimate engines:enable phpmd`. This command both installs the engine and enables it in your `.codeclimate.yml` file. +3. You're ready to analyze! Browse into your project's folder and run `codeclimate analyze`. + +###Config Options + +Format the values for these config options per the [PHP_CodeSniffer documentation](https://github.com/squizlabs/PHP_CodeSniffer). + +* file_extensions - This is where you can configure the file extensions for the files that you want PHP_CodeSniffer to analyze. +* rulesets - This is the list of rulesets that you want PHPMD to use while analyzing your files. + +###Sample Config + + exclude_paths: + - "/examples/**/*" + engines: + phpmd: + enabled: true + config: + file_extensions: "php" + rulesets: "unusedcode" + ratings: + paths: + - "**.php" + +### Need help? + +For help with PHPMD, [check out their documentation](http://phpmd.org/documentation/index.html). + +If you're running into a Code Climate issue, first look over this project's [GitHub Issues](https://github.com/phpmd/phpmd/issues), as your question may have already been covered. If not, [go ahead and open a support ticket with us](https://codeclimate.com/help). diff --git a/Runner.php b/Runner.php new file mode 100644 index 0000000..578a068 --- /dev/null +++ b/Runner.php @@ -0,0 +1,77 @@ +config = $config; + } + + public function setServer($server) + { + $this->server = $server; + } + + public function queueDirectory($dir, $prefix = '') + { + $dir = rtrim($dir, '\\/'); + + foreach (scandir($dir) as $f) { + if (in_array("$prefix$f", $this->config["exclude_paths"])) { + continue; + } + + if ($f !== '.' and $f !== '..') { + if (is_dir("$dir/$f")) { + $this->queueDirectory("$dir/$f", "$prefix$f/"); + continue; + } + + $this->server->addwork(array("/code/$prefix$f")); + } + } + + $this->server->process_work(false); + } + + public function run($files) + { + $resultFile = tempnam(sys_get_temp_dir(), 'phpmd'); + + $renderer = new JSONRenderer(); + $renderer->setWriter(new StreamWriter($resultFile)); + + $ruleSetFactory = new RuleSetFactory(); + + $phpmd = new PHPMD(); + + if (isset($this->config['config']['file_extensions'])) { + $phpmd->setFileExtensions(explode(',', $this->config['config']['file_extensions'])); + } + + $rulesets = "cleancode,codesize,controversial,design,naming,unusedcode"; + + if (isset($this->config['config']['rulesets'])) { + $rulesets = $this->config['config']['rulesets']; + } + + $phpmd->processFiles( + implode(",", $files), + $rulesets, + array($renderer), + $ruleSetFactory + ); + + return $resultFile; + } +} diff --git a/composer.json b/composer.json index f9f5386..44ca9c3 100644 --- a/composer.json +++ b/composer.json @@ -1,5 +1,6 @@ { "require": { - "phpmd/phpmd": "2.2.3" + "phpmd/phpmd": "2.2.3", + "barracudanetworks/forkdaemon-php": "1.0.*" } } diff --git a/composer.lock b/composer.lock index 8816d27..b2d5225 100644 --- a/composer.lock +++ b/composer.lock @@ -1,11 +1,47 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "846cbcdaecb636c879610cc9d3da9523", + "hash": "769f093b0b48da0d80a8a36f288c52c6", "packages": [ + { + "name": "barracudanetworks/forkdaemon-php", + "version": "v1.0.2", + "source": { + "type": "git", + "url": "https://github.com/barracudanetworks/forkdaemon-php.git", + "reference": "9ca340a3c5f873dd731e0f1df2d79c16b1efd0d8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/barracudanetworks/forkdaemon-php/zipball/9ca340a3c5f873dd731e0f1df2d79c16b1efd0d8", + "reference": "9ca340a3c5f873dd731e0f1df2d79c16b1efd0d8", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "autoload": { + "files": [ + "fork_daemon.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A library to make setup and management of forking daemons in PHP easy.", + "homepage": "https://github.com/barracudanetworks/forkdaemon-php", + "keywords": [ + "daemons", + "forking", + "php" + ], + "time": "2015-05-01 13:48:28" + }, { "name": "pdepend/pdepend", "version": "2.1.0", diff --git a/engine.php b/engine.php index 44181ea..7db5fcf 100644 --- a/engine.php +++ b/engine.php @@ -1,70 +1,42 @@ setWriter(new StreamWriter(STDOUT)); - -$ruleSetFactory = new RuleSetFactory(); - -$all_files = scandir_recursive("/code"); +$runner = new Runner(); -if ($config["exclude_paths"]) { - $files = array(); +// setup forking daemon +$server = new \fork_daemon(); +$server->max_children_set(20); +$server->max_work_per_child_set(50); +$server->store_result_set(true); +$response = $server->register_child_run(array($runner, "run")); - foreach ($all_files as $file) { - if (!in_array($file, $config["exclude_paths"])) { - $files[] = "/code/".$file; - } - } -} else { - foreach ($all_files as $file) { - if (!in_array($file, $ignorePatterns)) { - $files[] = "/code/".$file; - } - } -} - -$phpmd = new PHPMD(); - -// if ($extensions !== null) { -// $phpmd->setFileExtensions(explode(',', $extensions)); -// } - -$phpmd->processFiles( - implode(",", $files), - "cleancode,codesize,controversial,design,naming,unusedcode", - array($renderer), - $ruleSetFactory -); +$config = json_decode(file_get_contents('/config.json'), true); -function scandir_recursive($dir, $prefix = '') { - $dir = rtrim($dir, '\\/'); - $result = array(); +$runner->setConfig($config); +$runner->setServer($server); +$runner->queueDirectory("/code"); - foreach (scandir($dir) as $f) { - if ($f !== '.' and $f !== '..') { - if (is_dir("$dir/$f")) { - $result = array_merge($result, scandir_recursive("$dir/$f", "$prefix$f/")); - } else { - $result[] = $prefix.$f; - } - } - } +$server->process_work(true); - return $result; +foreach ($server->get_all_results() as $result_file) { + echo file_get_contents($result_file); + unlink($result_file); } From b7c97e605e0a05fdc05c7b3f4011f44e34534080 Mon Sep 17 00:00:00 2001 From: Andy Blyler Date: Mon, 10 Aug 2015 12:05:31 -0400 Subject: [PATCH 2/6] Pin Docker to using 0.9.17 of phusion/baseimage --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 31b99e2..d011b97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM phusion/baseimage +FROM phusion/baseimage:0.9.17 WORKDIR /usr/src/app COPY composer.json /usr/src/app/ From ba7a4b3a898919a1f2ee00680ac9265345178a1d Mon Sep 17 00:00:00 2001 From: Andy Blyler Date: Mon, 10 Aug 2015 12:05:45 -0400 Subject: [PATCH 3/6] Fix references to PHPCS in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 254e06b..4a706e1 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ ###Config Options -Format the values for these config options per the [PHP_CodeSniffer documentation](https://github.com/squizlabs/PHP_CodeSniffer). +Format the values for these config options per the [PHPMD documentation](http://phpmd.org/documentation/index.html). -* file_extensions - This is where you can configure the file extensions for the files that you want PHP_CodeSniffer to analyze. +* file_extensions - This is where you can configure the file extensions for the files that you want PHPMD to analyze. * rulesets - This is the list of rulesets that you want PHPMD to use while analyzing your files. ###Sample Config From d82a32d490c741d05f62fe1d5861a004dabbda96 Mon Sep 17 00:00:00 2001 From: Andy Blyler Date: Mon, 10 Aug 2015 12:06:00 -0400 Subject: [PATCH 4/6] Add config and server to constructor of Runner --- Runner.php | 6 +----- engine.php | 10 ++++------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Runner.php b/Runner.php index 578a068..e41b091 100644 --- a/Runner.php +++ b/Runner.php @@ -12,13 +12,9 @@ class Runner private $config; private $server; - public function setConfig($config) + public function __construct($config, $server) { $this->config = $config; - } - - public function setServer($server) - { $this->server = $server; } diff --git a/engine.php b/engine.php index 7db5fcf..4e3cbf8 100644 --- a/engine.php +++ b/engine.php @@ -19,19 +19,17 @@ use PHPMD\Writer\StreamWriter; use PHPMD\Renderer\JSONRenderer; -$runner = new Runner(); +// obtain the config +$config = json_decode(file_get_contents('/config.json'), true); // setup forking daemon $server = new \fork_daemon(); $server->max_children_set(20); $server->max_work_per_child_set(50); $server->store_result_set(true); -$response = $server->register_child_run(array($runner, "run")); - -$config = json_decode(file_get_contents('/config.json'), true); +$runner = new Runner($config, $server); +$server->register_child_run(array($runner, "run")); -$runner->setConfig($config); -$runner->setServer($server); $runner->queueDirectory("/code"); $server->process_work(true); From 3bef1a4b0952f564f2a09681a860525373b71e0d Mon Sep 17 00:00:00 2001 From: Andy Blyler Date: Fri, 14 Aug 2015 11:19:59 -0400 Subject: [PATCH 5/6] Revert to using alpine linux as the base - we can use it now that it has php-pcntl support --- Dockerfile | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index d011b97..60b6164 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,17 @@ -FROM phusion/baseimage:0.9.17 +FROM alpine:edge WORKDIR /usr/src/app COPY composer.json /usr/src/app/ COPY composer.lock /usr/src/app/ -RUN apt-get update -RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y curl git -RUN add-apt-repository -y ppa:ondrej/php5-5.6 -RUN apt-get update -RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y --force-yes php5-cli +RUN apk --update add git php-common php-xml php-dom php-ctype php-iconv php-json php-pcntl php-phar php-openssl php-opcache php-sockets curl && \ + curl -sS https://getcomposer.org/installer | php && \ + /usr/src/app/composer.phar install && \ + apk del build-base && rm -fr /usr/share/ri -RUN curl -sS https://getcomposer.org/installer | php -RUN /usr/src/app/composer.phar install - -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -RUN adduser --uid 9000 --disabled-password app +RUN adduser -u 9000 -D app USER app COPY . /usr/src/app CMD ["/usr/src/app/bin/codeclimate-phpmd"] - From 66a672f8044aae41829cee6598a894bad64ece24 Mon Sep 17 00:00:00 2001 From: Andy Blyler Date: Sat, 15 Aug 2015 20:44:52 -0400 Subject: [PATCH 6/6] Change max number of children from 20 to 3 - did some performance testing after @brynary comment in GitHub and found that 3 performs best on my large code bases --- engine.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine.php b/engine.php index 4e3cbf8..bbb422e 100644 --- a/engine.php +++ b/engine.php @@ -24,7 +24,7 @@ // setup forking daemon $server = new \fork_daemon(); -$server->max_children_set(20); +$server->max_children_set(3); $server->max_work_per_child_set(50); $server->store_result_set(true); $runner = new Runner($config, $server);