clang-tidy-diff.py

#!/usr/bin/env python
#
#===- clang-tidy-diff.py - ClangTidy Diff Checker ------------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#

r"""
ClangTidy Diff Checker
======================

This script reads input from a unified diff, runs clang-tidy on all changed
files and outputs clang-tidy warnings in changed lines only. This is useful to
detect clang-tidy regressions in the lines touched by a specific patch.
Example usage for git/svn users:

  git diff -U0 HEAD^ | clang-tidy-diff.py -p1
  svn diff --diff-cmd=diff -x-U0 | \
      clang-tidy-diff.py -fix -checks=-*,modernize-use-override

"""

import argparse
import glob
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback

try:
  import yaml
except ImportError:
  yaml = None

is_py2 = sys.version[0] == '2'

if is_py2:
    import Queue as queue
else:
    import queue as queue


def run_tidy(task_queue, lock, timeout):
  watchdog = None
  while True:
    command = task_queue.get()
    try:
      proc = subprocess.Popen(command,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE)

      if timeout is not None:
        watchdog = threading.Timer(timeout, proc.kill)
        watchdog.start()

      stdout, stderr = proc.communicate()

      with lock:
        sys.stdout.write(stdout.decode('utf-8') + '\n')
        sys.stdout.flush()
        if stderr:
          sys.stderr.write(stderr.decode('utf-8') + '\n')
          sys.stderr.flush()
    except Exception as e:
      with lock:
        sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
    finally:
      with lock:
        if (not timeout is None) and (not watchdog is None):
          if not watchdog.is_alive():
              sys.stderr.write('Terminated by timeout: ' +
                               ' '.join(command) + '\n')
          watchdog.cancel()
      task_queue.task_done()


def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
  for _ in range(max_tasks):
    t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout))
    t.daemon = True
    t.start()

def merge_replacement_files(tmpdir, mergefile):
  """Merge all replacement files in a directory into a single file"""
  # The fixes suggested by clang-tidy >= 4.0.0 are given under
  # the top level key 'Diagnostics' in the output yaml files
  mergekey = "Diagnostics"
  merged = []
  for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
    content = yaml.safe_load(open(replacefile, 'r'))
    if not content:
      continue # Skip empty files.
    merged.extend(content.get(mergekey, []))

  if merged:
    # MainSourceFile: The key is required by the definition inside
    # include/clang/Tooling/ReplacementsYaml.h, but the value
    # is actually never used inside clang-apply-replacements,
    # so we set it to '' here.
    output = { 'MainSourceFile': '', mergekey: merged }
    with open(mergefile, 'w') as out:
      yaml.safe_dump(output, out)
  else:
    # Empty the file:
    open(mergefile, 'w').close()


def main():
  parser = argparse.ArgumentParser(description=
                                   'Run clang-tidy against changed files, and '
                                   'output diagnostics only for modified '
                                   'lines.')
  parser.add_argument('-clang-tidy-binary', metavar='PATH',
                      default='clang-tidy',
                      help='path to clang-tidy binary')
  parser.add_argument('-p', metavar='NUM', default=0,
                      help='strip the smallest prefix containing P slashes')
  parser.add_argument('-regex', metavar='PATTERN', default=None,
                      help='custom pattern selecting file paths to check '
                      '(case sensitive, overrides -iregex)')
  parser.add_argument('-iregex', metavar='PATTERN', default=
                      r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
                      help='custom pattern selecting file paths to check '
                      '(case insensitive, overridden by -regex)')
  parser.add_argument('-j', type=int, default=1,
                      help='number of tidy instances to be run in parallel.')
  parser.add_argument('-timeout', type=int, default=None,
                      help='timeout per each file in seconds.')
  parser.add_argument('-fix', action='store_true', default=False,
                      help='apply suggested fixes')
  parser.add_argument('-checks',
                      help='checks filter, when not specified, use clang-tidy '
                      'default',
                      default='')
  parser.add_argument('-path', dest='build_path',
                      help='Path used to read a compile command database.')
  if yaml:
    parser.add_argument('-export-fixes', metavar='FILE', dest='export_fixes',
                        help='Create a yaml file to store suggested fixes in, '
                        'which can be applied with clang-apply-replacements.')
  parser.add_argument('-extra-arg', dest='extra_arg',
                      action='append', default=[],
                      help='Additional argument to append to the compiler '
                      'command line.')
  parser.add_argument('-extra-arg-before', dest='extra_arg_before',
                      action='append', default=[],
                      help='Additional argument to prepend to the compiler '
                      'command line.')
  parser.add_argument('-quiet', action='store_true', default=False,
                      help='Run clang-tidy in quiet mode')
  clang_tidy_args = []
  argv = sys.argv[1:]
  if '--' in argv:
    clang_tidy_args.extend(argv[argv.index('--'):])
    argv = argv[:argv.index('--')]

  args = parser.parse_args(argv)

  # Extract changed lines for each file.
  filename = None
  lines_by_file = {}
  for line in sys.stdin:
    match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line)
    if match:
      filename = match.group(2)
    if filename is None:
      continue

    if args.regex is not None:
      if not re.match('^%s$' % args.regex, filename):
        continue
    else:
      if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
        continue

    match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
    if match:
      start_line = int(match.group(1))
      line_count = 1
      if match.group(3):
        line_count = int(match.group(3))
      if line_count == 0:
        continue
      end_line = start_line + line_count - 1
      lines_by_file.setdefault(filename, []).append([start_line, end_line])

  if not any(lines_by_file):
    print("No relevant changes found.")
    sys.exit(0)

  max_task_count = args.j
  if max_task_count == 0:
      max_task_count = multiprocessing.cpu_count()
  max_task_count = min(len(lines_by_file), max_task_count)

  tmpdir = None
  if yaml and args.export_fixes:
    tmpdir = tempfile.mkdtemp()

  # Tasks for clang-tidy.
  task_queue = queue.Queue(max_task_count)
  # A lock for console output.
  lock = threading.Lock()

  # Run a pool of clang-tidy workers.
  start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)

  # Form the common args list.
  common_clang_tidy_args = []
  if args.fix:
    common_clang_tidy_args.append('-fix')
  if args.checks != '':
    common_clang_tidy_args.append('-checks=' + args.checks)
  if args.quiet:
    common_clang_tidy_args.append('-quiet')
  if args.build_path is not None:
    common_clang_tidy_args.append('-p=%s' % args.build_path)
  for arg in args.extra_arg:
    common_clang_tidy_args.append('-extra-arg=%s' % arg)
  for arg in args.extra_arg_before:
    common_clang_tidy_args.append('-extra-arg-before=%s' % arg)

  for name in lines_by_file:
    line_filter_json = json.dumps(
      [{"name": name, "lines": lines_by_file[name]}],
      separators=(',', ':'))

    # Run clang-tidy on files containing changes.
    command = [args.clang_tidy_binary]
    command.append('-line-filter=' + line_filter_json)
    if yaml and args.export_fixes:
      # Get a temporary file. We immediately close the handle so clang-tidy can
      # overwrite it.
      (handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
      os.close(handle)
      command.append('-export-fixes=' + tmp_name)
    command.extend(common_clang_tidy_args)
    command.append(name)
    command.extend(clang_tidy_args)

    task_queue.put(command)

  # Wait for all threads to be done.
  task_queue.join()

  if yaml and args.export_fixes:
    print('Writing fixes to ' + args.export_fixes + ' ...')
    try:
      merge_replacement_files(tmpdir, args.export_fixes)
    except:
      sys.stderr.write('Error exporting fixes.\n')
      traceback.print_exc()

  if tmpdir:
    shutil.rmtree(tmpdir)


if __name__ == '__main__':
  main()