Merge aa40616 into f3599b2

.github/workflows/test.yml

@@ -28,3 +28,7 @@ jobs:
       - name: Unit tests
         run: |
           coverage run -m unittest
+
+      - name: primer run
+        run: |
+          black-primer

src/black_primer/cli.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 
+# coding=utf8
+
 import asyncio
 import logging
 import sys
 from datetime import datetime
-from os import cpu_count
 from pathlib import Path
 from shutil import rmtree, which
 from tempfile import gettempdir
@@ -61,7 +62,7 @@ async def async_main(
     finally:
         if not keep and work_path.exists():
             LOG.debug(f"Removing {work_path}")
-            rmtree(work_path)
+            rmtree(work_path, onerror=lib.handle_PermissionError)
 
     return -2
 
@@ -114,7 +115,7 @@ async def async_main(
 @click.option(
     "-W",
     "--workers",
-    default=int((cpu_count() or 4) / 2) or 1,
+    default=2,
     type=int,
     show_default=True,
     help="Number of parallel worker coroutines",

src/black_primer/lib.py

@@ -1,15 +1,19 @@
 #!/usr/bin/env python3
 
 import asyncio
+import errno
 import json
 import logging
+import os
+import stat
 import sys
+from functools import partial
 from pathlib import Path
 from platform import system
 from shutil import rmtree, which
 from subprocess import CalledProcessError
 from sys import version_info
-from typing import Any, Dict, NamedTuple, Optional, Sequence, Tuple
+from typing import Any, Callable, Dict, NamedTuple, Optional, Sequence, Tuple
 from urllib.parse import urlparse
 
 import click
@@ -36,7 +40,7 @@ class Results(NamedTuple):
 
 async def _gen_check_output(
     cmd: Sequence[str],
-    timeout: float = 30,
+    timeout: float = 300,
     env: Optional[Dict[str, str]] = None,
     cwd: Optional[Path] = None,
 ) -> Tuple[bytes, bytes]:
@@ -176,6 +180,30 @@ async def git_checkout_or_rebase(
     return repo_path
 
 
+def handle_PermissionError(
+    func: Callable, path: Path, exc: Tuple[Any, Any, Any]
+) -> None:
+    """
+    Handle PermissionError during shutil.rmtree.
+
+    This checks if the erroring function is either 'os.rmdir' or 'os.unlink', and that
+    the error was EACCES (i.e. Permission denied). If true, the path is set writable,
+    readable, and executable by everyone. Finally, it tries the error causing delete
+    operation again.
+
+    If the check is false, then the original error will be reraised as this function
+    can't handle it.
+    """
+    excvalue = exc[1]
+    LOG.debug(f"Handling {excvalue} from {func.__name__} ... ")
+    if func in (os.rmdir, os.unlink) and excvalue.errno == errno.EACCES:
+        LOG.debug(f"Setting {path} writable, readable, and executable by everyone... ")
+        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)  # chmod 0777
+        func(path)  # Try the error causing delete operation again
+    else:
+        raise
+
+
 async def load_projects_queue(
     config_path: Path,
 ) -> Tuple[Dict[str, Any], asyncio.Queue]:
@@ -212,6 +240,7 @@ async def project_runner(
         except asyncio.QueueEmpty:
             LOG.debug(f"project_runner {idx} exiting")
             return
+        LOG.debug(f"worker {idx} working on {project_name}")
 
         project_config = config["projects"][project_name]
 
@@ -243,7 +272,12 @@ async def project_runner(
 
         if not keep:
             LOG.debug(f"Removing {repo_path}")
-            await loop.run_in_executor(None, rmtree, repo_path)
+            rmtree_partial = partial(
+                rmtree, path=repo_path, onerror=handle_PermissionError
+            )
+            await loop.run_in_executor(None, rmtree_partial)
+
+        LOG.info(f"Finished {project_name}")
 
 
 async def process_queue(

src/black_primer/primer.json

@@ -3,7 +3,7 @@
   "projects": {
     "aioexabgp": {
       "cli_arguments": [],
-      "expect_formatting_changes": true,
+      "expect_formatting_changes": false,
       "git_clone_url": "https://github.com/cooperlees/aioexabgp.git",
       "long_checkout": false,
       "py_versions": ["all"]
@@ -17,17 +17,107 @@
     },
     "bandersnatch": {
       "cli_arguments": [],
-      "expect_formatting_changes": true,
+      "expect_formatting_changes": false,
       "git_clone_url": "https://github.com/pypa/bandersnatch.git",
       "long_checkout": false,
       "py_versions": ["all"]
     },
-    "flake8-bugbear": {
+    "channels": {
+      "cli_arguments": [],
+      "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/django/channels.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "django": {
+      "disabled_reason": "black --check --diff returned 123",
+      "disabled": true,
       "cli_arguments": [],
       "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/django/django.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "flake8-bugbear": {
+      "cli_arguments": [],
+      "expect_formatting_changes": false,
       "git_clone_url": "https://github.com/PyCQA/flake8-bugbear.git",
       "long_checkout": false,
       "py_versions": ["all"]
+    },
+    "hypothesis": {
+      "cli_arguments": [],
+      "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/HypothesisWorks/hypothesis.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "pandas": {
+      "disabled_reason": "black --check --diff returned 123",
+      "disabled": true,
+      "cli_arguments": [],
+      "expect_formatting_changes": false,
+      "git_clone_url": "https://github.com/pandas-dev/pandas.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "poetry": {
+      "cli_arguments": [],
+      "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/python-poetry/poetry.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "pyramid": {
+      "cli_arguments": [],
+      "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/Pylons/pyramid.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "ptr": {
+      "cli_arguments": [],
+      "expect_formatting_changes": false,
+      "git_clone_url": "https://github.com/facebookincubator/ptr.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "pytest": {
+      "disabled_reason": "black --check --diff returned 123",
+      "disabled": true,
+      "cli_arguments": [],
+      "expect_formatting_changes": false,
+      "git_clone_url": "https://github.com/pytest-dev/pytest.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "sqlalchemy": {
+      "cli_arguments": [],
+      "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/sqlalchemy/sqlalchemy.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "tox": {
+      "cli_arguments": [],
+      "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/tox-dev/tox.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "virtualenv": {
+      "cli_arguments": [],
+      "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/pypa/virtualenv.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
+    },
+    "warehouse": {
+      "cli_arguments": [],
+      "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/pypa/warehouse.git",
+      "long_checkout": false,
+      "py_versions": ["all"]
     }
   }
 }