Allow saving lint errors to database

Summary: This saves lint errors to the path change of current commit. It requires pushed revision. It doesn't save difference from previous commit mentioned in T2038#comment-4 - I don't plan doing it after all, everything would be much more complicated and the amount of data saved with this approach isn't that bad.

Test Plan: Applied patch, ran script, verified DB.

Reviewers: epriestley

Reviewed By: epriestley

CC: aran, Korvin

Maniphest Tasks: T2038

Differential Revision: https://secure.phabricator.com/D3899

Author: vrana

resources/sql/patches/repository-lint.sql

@@ -0,0 +1,24 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_branch (
+  id int unsigned NOT NULL AUTO_INCREMENT,
+  repositoryID int unsigned NOT NULL,
+  name varchar(255) NOT NULL,
+  lintCommit varchar(40),
+  dateCreated int unsigned NOT NULL,
+  dateModified int unsigned NOT NULL,
+  UNIQUE (repositoryID, name),
+  PRIMARY KEY (id)
+);
+
+CREATE TABLE {$NAMESPACE}_repository.repository_lintmessage (
+  id int unsigned NOT NULL AUTO_INCREMENT,
+  branchID int unsigned NOT NULL,
+  path varchar(512) NOT NULL,
+  line int unsigned NOT NULL,
+  code varchar(32) NOT NULL,
+  severity varchar(16) NOT NULL,
+  name varchar(255) NOT NULL,
+  description text NOT NULL,
+  INDEX (branchID, path(64)),
+  INDEX (branchID, code, path(64)),
+  PRIMARY KEY (id)
+);

scripts/repository/save_lint.php

@@ -0,0 +1,103 @@
+#!/usr/bin/env php
+<?php
+
+require_once dirname(__FILE__).'/../__init_script__.php';
+
+if (function_exists('posix_isatty') && posix_isatty(STDIN)) {
+  $command = 'xargs -0 arc lint --output json | '.__FILE__;
+  echo "Usage: git ls-files -z | {$command}\n";
+  echo "Usage: git diff --name-only -z | {$command}\n"; // TODO: Handle deletes.
+  echo "Purpose: Save all lint errors to database.\n";
+  exit(1);
+}
+
+$working_copy = ArcanistWorkingCopyIdentity::newFromPath('.');
+$api = ArcanistRepositoryAPI::newAPIFromWorkingCopyIdentity($working_copy);
+$svn_root = id(new PhutilURI($api->getSourceControlPath()))->getPath();
+
+$project_id = $working_copy->getProjectID();
+$project = id(new PhabricatorRepositoryArcanistProject())
+  ->loadOneWhere('name = %s', $project_id);
+if (!$project || !$project->getRepositoryID()) {
+  throw new Exception("Couldn't find repository for {$project_id}.");
+}
+
+$branch_name = $api->getBranchName();
+$branch = id(new PhabricatorRepositoryBranch())->loadOneWhere(
+  'repositoryID = %d AND name = %s',
+  $project->getRepositoryID(),
+  $branch_name);
+if (!$branch) {
+  $branch = id(new PhabricatorRepositoryBranch())
+    ->setRepositoryID($project->getRepositoryID())
+    ->setName($branch_name);
+}
+$branch->setLintCommit($api->getWorkingCopyRevision());
+$branch->save();
+$conn = $branch->establishConnection('w');
+
+$inserts = array();
+
+while ($json = fgets(STDIN)) {
+  $paths = json_decode(rtrim($json, "\n"), true);
+  if (!is_array($paths)) {
+    throw new Exception("Invalid JSON: {$json}");
+  }
+
+  if (!$paths) {
+    continue;
+  }
+
+  $conn->openTransaction();
+
+  foreach (array_chunk(array_keys($paths), 1024) as $some_paths) {
+    $full_paths = array();
+    foreach ($some_paths as $path) {
+      $full_paths[] = $svn_root.'/'.$path;
+    }
+    queryfx(
+      $conn,
+      'DELETE FROM %T WHERE branchID = %d AND path IN (%Ls)',
+      PhabricatorRepository::TABLE_LINTMESSAGE,
+      $branch->getID(),
+      $full_paths);
+  }
+
+  foreach ($paths as $path => $messages) {
+    // TODO: Handle multiple $json for a single path. Don't save duplicates.
+
+    foreach ($messages as $message) {
+      $inserts[] = qsprintf(
+        $conn,
+        '(%d, %s, %d, %s, %s, %s, %s)',
+        $branch->getID(),
+        $svn_root.'/'.$path,
+        idx($message, 'line', 0),
+        idx($message, 'code', ''),
+        idx($message, 'severity', ''),
+        idx($message, 'name', ''),
+        idx($message, 'description', ''));
+
+      if (count($inserts) >= 256) {
+        save_lint_messages($conn, $inserts);
+        $inserts = array();
+      }
+    }
+  }
+
+  $conn->saveTransaction();
+}
+
+save_lint_messages($conn, $inserts);
+
+function save_lint_messages($conn, array $inserts) {
+  if ($inserts) {
+    queryfx(
+      $conn,
+      'INSERT INTO %T
+        (branchID, path, line, code, severity, name, description)
+        VALUES %Q',
+      PhabricatorRepository::TABLE_LINTMESSAGE,
+      implode(', ', $inserts));
+  }
+}

src/__phutil_library_map__.php

@@ -992,6 +992,7 @@
     'PhabricatorRepositoryArcanistProjectDeleteController' => 'applications/repository/controller/PhabricatorRepositoryArcanistProjectDeleteController.php',
     'PhabricatorRepositoryArcanistProjectEditController' => 'applications/repository/controller/PhabricatorRepositoryArcanistProjectEditController.php',
     'PhabricatorRepositoryAuditRequest' => 'applications/repository/storage/PhabricatorRepositoryAuditRequest.php',
+    'PhabricatorRepositoryBranch' => 'applications/repository/storage/PhabricatorRepositoryBranch.php',
     'PhabricatorRepositoryCommit' => 'applications/repository/storage/PhabricatorRepositoryCommit.php',
     'PhabricatorRepositoryCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php',
     'PhabricatorRepositoryCommitData' => 'applications/repository/storage/PhabricatorRepositoryCommitData.php',
@@ -2175,6 +2176,7 @@
     'PhabricatorRepositoryArcanistProjectDeleteController' => 'PhabricatorRepositoryController',
     'PhabricatorRepositoryArcanistProjectEditController' => 'PhabricatorRepositoryController',
     'PhabricatorRepositoryAuditRequest' => 'PhabricatorRepositoryDAO',
+    'PhabricatorRepositoryBranch' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositoryCommit' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositoryCommitChangeParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
     'PhabricatorRepositoryCommitData' => 'PhabricatorRepositoryDAO',

src/applications/repository/storage/PhabricatorRepository.php

@@ -10,6 +10,7 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO {
   const TABLE_FILESYSTEM = 'repository_filesystem';
   const TABLE_SUMMARY = 'repository_summary';
   const TABLE_BADCOMMIT = 'repository_badcommit';
+  const TABLE_LINTMESSAGE = 'repository_lintmessage';
 
   protected $phid;
   protected $name;

src/applications/repository/storage/PhabricatorRepositoryBranch.php

@@ -0,0 +1,9 @@
+<?php
+
+final class PhabricatorRepositoryBranch extends PhabricatorRepositoryDAO {
+
+  protected $repositoryID;
+  protected $name;
+  protected $lintCommit;
+
+}

src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php

@@ -1024,6 +1024,10 @@ public function getPatches() {
         'type'    => 'sql',
         'name'    => $this->getPatchPath('dropfileproxyimage.sql'),
       ),
+      'repository-lint.sql' => array(
+        'type'    => 'sql',
+        'name'    => $this->getPatchPath('repository-lint.sql'),
+      ),
     );
   }