Skip to content

Commit

Permalink
[BUGFIX] Fix check for binary files and do not exclude JSON or YAML f…
Browse files Browse the repository at this point in the history
…iles

Also, show list of skipped binary files, in verbose mode (-v)

Resolves: #15
  • Loading branch information
a-r-m-i-n committed Oct 23, 2023
1 parent a1c7aed commit b2fb5cb
Show file tree
Hide file tree
Showing 11 changed files with 253 additions and 18 deletions.
2 changes: 1 addition & 1 deletion README.md
Expand Up @@ -144,7 +144,7 @@ The ``ec`` binary supports the following options:
| ``--strict`` | | When set, given indention size is forced during scan and fixing. This might conflict with more detailed indention rules, checked by other linters and style-fixers in your project. |
| ``--compact`` | ``-c`` | Only shows only files with issues, not the issues itself. |
| ``--uncovered`` | ``-u`` | Lists all files which are not covered by .editorconfig. |
| ``--verbose`` | ``-v`` | Shows additional informations, like detailed info about internal time tracking. |
| ``--verbose`` | ``-v`` | Shows additional informations, like detailed info about internal time tracking and which binary files have been skipped. |
| ``--no-interaction`` | ``-n`` | Do not ask for confirmation, if more than 500 files found. |
| ``--no-error-on-exit`` | | By default ``ec`` returns code 2 when issues occurred. With this option set return code is always 0. |

Expand Down
28 changes: 22 additions & 6 deletions src/Application.php
Expand Up @@ -36,6 +36,11 @@ class Application extends SingleCommandApplication
*/
private $version;

/**
* @var bool
*/
private $isVerbose = false;

public function __construct(string $name = 'ec', ?Scanner $scanner = null)
{
TimeTrackingUtility::reset();
Expand Down Expand Up @@ -96,6 +101,8 @@ protected function executing(Input $input, Output $output): int
$io->getFormatter()->setStyle('debug', new OutputFormatterStyle('blue'));
$io->getFormatter()->setStyle('warning', new OutputFormatterStyle('black', 'yellow'));

$this->isVerbose = $output->isVerbose();

/** @var string $workingDirectory */
$workingDirectory = $input->getOption('dir');
if (empty($workingDirectory)) {
Expand Down Expand Up @@ -149,7 +156,7 @@ protected function executing(Input $input, Output $output): int
$io->writeln('Get files from git binary (command: <comment>' . $gitOnlyCommand . '</comment>):');
}
}
if (!$finderConfigPath && $output->isVerbose()) {
if (!$finderConfigPath && $this->isVerbose) {
if ($gitOnlyEnabled && $gitOnlyCommand) {
$io->writeln('<debug>Names and (auto-) excludes disabled, because of set git-only mode.</debug>');
} else {
Expand All @@ -158,7 +165,7 @@ protected function executing(Input $input, Output $output): int
$io->writeln('<debug>Auto exclude: ' . ($input->getOption('disable-auto-exclude') ? 'disabled' : 'enabled') . '</debug>');
}
}
if ($output->isVerbose()) {
if ($this->isVerbose) {
$io->writeln('<debug>Strict mode: ' . ($input->getOption('strict') ? 'enabled' : 'disabled') . '</debug>');
$io->writeln('<debug>Output mode: ' . ($input->getOption('compact') ? 'compact' : 'full') . '</debug>');
}
Expand Down Expand Up @@ -188,7 +195,16 @@ protected function executing(Input $input, Output $output): int
? $this->scan($finder, $count, $io, (bool)$input->getOption('strict'), (bool)$input->getOption('no-progress'), (bool)$input->getOption('compact'), (bool)$input->getOption('uncovered'))
: $this->fix($finder, $io, (bool)$input->getOption('strict'));

if ($output->isVerbose()) {
if ($this->isVerbose) {
if (!empty($this->scanner->getSkippedBinaryFiles())) {
$amountBinaryFiles = count($this->scanner->getSkippedBinaryFiles());
$io->newLine();
$io->writeln('<debug>' . $amountBinaryFiles . ' binary ' . StringFormatUtility::pluralizeFiles($amountBinaryFiles) . ' skipped:</debug>');
foreach ($this->scanner->getSkippedBinaryFiles() as $binaryFile => $mimeType) {
$io->writeln('<info>' . $binaryFile . '</info> <comment>[' . $mimeType . ']</comment>');
}
}

$io->newLine();
$io->writeln('<debug>Time tracking</debug>');
$io->writeln('<debug>----------------------------------------</debug>');
Expand All @@ -200,7 +216,7 @@ protected function executing(Input $input, Output $output): int
}

if ($input->getOption('no-error-on-exit')) {
if ($returnValue > 0 && $output->isVerbose()) {
if ($returnValue > 0 && $this->isVerbose) {
$io->writeln(sprintf('<debug>Bypassing error code %d</debug>', $returnValue));
}
$returnValue = 0;
Expand Down Expand Up @@ -232,7 +248,7 @@ protected function scan(Finder $finder, int $fileCount, SymfonyStyle $io, bool $
}

// Start the scan
$fileResults = $this->scanner->scan($finder, $strict, $callback);
$fileResults = $this->scanner->scan($finder, $strict, $callback, $this->isVerbose);

if (!$noProgress && $progressBar) {
// Progress bar
Expand Down Expand Up @@ -291,7 +307,7 @@ protected function fix(Finder $finder, SymfonyStyle $io, bool $strict = false):
{
$io->writeln('<comment>Starting to fix issues...</comment>');

$fileResults = $this->scanner->scan($finder, $strict);
$fileResults = $this->scanner->scan($finder, $strict, null, $this->isVerbose);
$invalidFilesCount = 0;
$errorCountTotal = 0;
$hasUnfixableExceptions = false;
Expand Down
6 changes: 2 additions & 4 deletions src/EditorConfig/Rules/Validator.php
Expand Up @@ -10,11 +10,11 @@
use Armin\EditorconfigCli\EditorConfig\Rules\File\TrimTrailingWhitespaceRule;
use Armin\EditorconfigCli\EditorConfig\Rules\Line\IndentionRule;
use Armin\EditorconfigCli\EditorConfig\Rules\Line\MaxLineLengthRule;
use Armin\EditorconfigCli\EditorConfig\Utility\MimeTypeUtility;
use Idiosyncratic\EditorConfig\Declaration\Charset;
use Idiosyncratic\EditorConfig\Declaration\Declaration;
use Idiosyncratic\EditorConfig\Declaration\TrimTrailingWhitespace;
use Symfony\Component\Finder\SplFileInfo;
use Symfony\Component\Mime\MimeTypes;

class Validator
{
Expand All @@ -36,9 +36,7 @@ public function createValidatedFileResult(SplFileInfo $file, array $editorConfig
$filePath = (string)$file->getRealPath();
$rules = [];

$mime = new MimeTypes();
$mimeType = (string)$mime->guessMimeType($filePath);
if (0 !== strpos($mimeType, 'text/')) {
if (!MimeTypeUtility::isCommonTextType($filePath) && (MimeTypeUtility::isCommonBinaryType($filePath) || MimeTypeUtility::isBinaryFileType($filePath))) {
return new FileResult($filePath, [], true); // Skip non-ascii files
}

Expand Down
26 changes: 21 additions & 5 deletions src/EditorConfig/Scanner.php
Expand Up @@ -6,6 +6,7 @@

use Armin\EditorconfigCli\EditorConfig\Rules\FileResult;
use Armin\EditorconfigCli\EditorConfig\Rules\Validator;
use Armin\EditorconfigCli\EditorConfig\Utility\MimeTypeUtility;
use Armin\EditorconfigCli\EditorConfig\Utility\TimeTrackingUtility;
use Idiosyncratic\EditorConfig\EditorConfig;
use Symfony\Component\Finder\Finder;
Expand All @@ -32,6 +33,11 @@ class Scanner
*/
private $validator;

/**
* @var array|string[]
*/
private $skippedBinaryFiles = [];

public function __construct(?EditorConfig $editorConfig = null, ?Validator $validator = null, string $rootPath = null, array $skippingRules = [])
{
$this->editorConfig = $editorConfig ?? new EditorConfig();
Expand All @@ -57,24 +63,34 @@ public function setSkippingRules(array $skippingRules): void
$this->skippingRules = $skippingRules;
}

/**
* @return array<string, string> Key is file path, value is guessed mime-type
*/
public function getSkippedBinaryFiles(): array
{
return $this->skippedBinaryFiles;
}

/**
* @param bool $strict when true, any difference of indention size is spotted
*
* @return array|FileResult[]
*/
public function scan(Finder $finderInstance, bool $strict = false, callable $tickCallback = null): array
public function scan(Finder $finderInstance, bool $strict = false, callable $tickCallback = null, bool $collectBinaryFiles = false): array
{
$results = [];
foreach ($finderInstance as $file) {
$config = $this->editorConfig->getConfigForPath((string)$file->getRealPath());

$fileResult = $this->validator->createValidatedFileResult($file, $config, $strict, $this->skippingRules);
$filePath = $fileResult->getFilePath();
if (!empty($this->rootPath)) {
$filePath = substr($filePath, strlen($this->rootPath));
}
if (!$fileResult->isBinary()) {
$filePath = $fileResult->getFilePath();
if (!empty($this->rootPath)) {
$filePath = substr($filePath, strlen($this->rootPath));
}
$results[$filePath] = $fileResult;
} elseif ($collectBinaryFiles) {
$this->skippedBinaryFiles[$filePath] = MimeTypeUtility::guessMimeType($fileResult->getFilePath());
}
if ($tickCallback) {
$tickCallback($fileResult);
Expand Down
121 changes: 121 additions & 0 deletions src/EditorConfig/Utility/MimeTypeUtility.php
@@ -0,0 +1,121 @@
<?php

namespace Armin\EditorconfigCli\EditorConfig\Utility;

use Symfony\Component\Mime\MimeTypes;

class MimeTypeUtility
{
public static function guessMimeType(string $filePath): string
{
$mime = new MimeTypes();

return (string)$mime->guessMimeType($filePath);
}

public static function isCommonTextType(string $filePath): bool
{
$mimeType = self::guessMimeType($filePath);
if (0 === strpos($mimeType, 'text/')) {
return true;
}

if (0 === strpos($mimeType, 'application/')) {
if ('script' === substr($mimeType, -strlen('script'))) {
return true;
}
if ('json' === substr($mimeType, -strlen('json'))) {
return true;
}
if ('yaml' === substr($mimeType, -strlen('yaml'))) {
return true;
}
if ('xml' === substr($mimeType, -strlen('xml'))) {
return true;
}
if ('sql' === substr($mimeType, -strlen('sql'))) {
return true;
}
}

return false;
}

public static function isCommonBinaryType(string $filePath): bool
{
$mimeType = self::guessMimeType($filePath);
if (0 === strpos($mimeType, 'font/')) {
return true;
}
if (0 === strpos($mimeType, 'image/')) {
return true;
}
if (0 === strpos($mimeType, 'audio/')) {
return true;
}
if (0 === strpos($mimeType, 'video/')) {
return true;
}
if (0 === strpos($mimeType, 'application/vnd.')) {
return true;
}
if ('application/pdf' === $mimeType) {
return true;
}
if ('application/msword' === $mimeType) {
return true;
}
if ('application/rtf' === $mimeType) {
return true;
}
if ('application/zip' === $mimeType) {
return true;
}
if ('application/tar' === $mimeType) {
return true;
}
if ('application/bzip2' === $mimeType) {
return true;
}
if ('application/octet-stream' === $mimeType) {
return true;
}
if ('application/wasm' === $mimeType) {
return true;
}
if (0 === strpos($mimeType, 'application/')) {
if ('-compressed' === substr($mimeType, -strlen('-compressed'))) {
return true;
}
if ('-ttf' === substr($mimeType, -strlen('-ttf'))) {
return true;
}
if ('-archive' === substr($mimeType, -strlen('-archive'))) {
return true;
}
}

return false;
}

public static function isBinaryFileType(string $filePath, float $threshold = .9): bool
{
$content = file_get_contents($filePath);
if (!$content) {
throw new \RuntimeException('Unable to check file "' . $filePath . '" for being binary!');
}
$length = strlen($content);
$printableCount = 0;

for ($i = 0; $i < $length; ++$i) {
$ord = ord($content[$i]);

// Printable ASCII chars (32-126), Tabulator (9), CR (10) and LF (13)
if (($ord >= 32 && $ord <= 126) || 9 === $ord || 10 === $ord || 13 === $ord) {
++$printableCount;
}
}

return ($printableCount / $length) < $threshold;
}
}
11 changes: 10 additions & 1 deletion src/EditorConfig/Utility/StringFormatUtility.php
Expand Up @@ -13,8 +13,17 @@ public static function buildScanResultMessage(int $amountIssues, int $amountFile
}

$issueText = 1 === $amountIssues ? 'issue' : 'issues';
$filesText = 1 === $amountFilesWithIssues ? 'file' : 'files';
$filesText = self::pluralizeFiles($amountFilesWithIssues);

return sprintf('%s %d %s in %d %s', $text, $amountIssues, $issueText, $amountFilesWithIssues, $filesText);
}

public static function pluralizeFiles(int $amountFiles): string
{
if (1 === $amountFiles) {
return 'file';
}

return 'files';
}
}
Binary file added tests/Fixtures/document.pdf
Binary file not shown.
Binary file added tests/Fixtures/image.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
24 changes: 23 additions & 1 deletion tests/Functional/EditorConfig/CommandInvalidCaseTest.php
Expand Up @@ -43,6 +43,28 @@ class CommandInvalidCaseTest extends AbstractTestCase
TXT,
'invalid3.txt' => " This line has spaces, but it should be tabs." . self::CRLF .
" This line has one space too much. After fixing, this line should have 2 tab chars." . self::CRLF,
'invalid4.json' => <<<JSON
{
"type": "project",
"repositories": [
{
"type": "composer",
"url": "https://example.com"
},
{
"type": "path",
"url": "packages/*"
}
]
}
JSON,
'invalid5.yaml' => <<<YAML
test:
invalid: true
invalid2: true
valid: true
valid2: true
YAML,
];


Expand All @@ -68,7 +90,7 @@ public function testInvalidCase()
self::assertStringContainsString(DIRECTORY_SEPARATOR . 'invalid3.txt [3]', $commandTester->getDisplay());
self::assertStringContainsString('Line 1: Expected indention style "tab" but found "spaces"', $commandTester->getDisplay());
self::assertStringContainsString('Line 2: Expected indention style "tab" but found "spaces"', $commandTester->getDisplay());
self::assertStringContainsString('Found 12 issues in 3 files', $commandTester->getDisplay());
self::assertStringContainsString('Found 27 issues in 5 files', $commandTester->getDisplay());
}

public function testInvalidWorkingDirectoryGiven()
Expand Down

0 comments on commit b2fb5cb

Please sign in to comment.