[v7r2] Add pre-commit configuration and run black

.github/CODEOWNERS

@@ -9,7 +9,7 @@
 
 
 # Order is important; the last matching pattern takes the most
-# precedence. 
+# precedence.
 # e.g. when someone opens a pull request that only
 # modifies files in /DIRAC/docs/tools/ files, only @andresailer and not the global
 # owner(s) will be requested for a review.

.github/workflows/basic-python3.yml

@@ -3,6 +3,19 @@ name: Python 3 tests
 on: [push, pull_request]
 
 jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'push' || github.repository == 'DIRACGrid-test/DIRAC'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    - name: Install pre-commit
+      run: pip install pre-commit
+    - name: Run pre-commit
+      run: pre-commit run --all-files --show-diff-on-failure
+
   check:
     runs-on: ubuntu-latest
     if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'

.github/workflows/semantic.yml

@@ -30,4 +30,4 @@ jobs:
           excludeDescription: 'true' # optional: this excludes the description body of a pull request
           excludeTitle: 'true' # optional: this excludes the title of a pull request
           checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
-          accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true
+          accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true

.gitignore

@@ -39,12 +39,12 @@ pip-log.txt
 .settings
 .metadata
 
-# Vim 
+# Vim
 .*.sw[a-z]
 *.un~
 Session.vim
 
-# Emacs 
+# Emacs
 *~
 
 # Intellij
@@ -86,4 +86,3 @@ docs/source/AdministratorGuide/CommandReference
 docs/source/UserGuide/CommandReference
 docs/_build
 docs/source/_build
-

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+default_language_version:
+  python: python3.9
+
+exclude: |
+    (?x)^(
+        src/DIRAC/Interfaces/API/test/test.*\.jdl
+    )$
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+
+  - repo: https://github.com/psf/black
+    rev: 21.9b0
+    hooks:
+      - id: black
+        additional_dependencies: [".[python2]"]

.pylintrc

@@ -136,7 +136,7 @@ ignore-docstrings=yes
 ignore-mixin-members=yes
 
 # List of classes names for which member attributes should not be checked
-# Skip some sqlalchemy objects which cause trouble 
+# Skip some sqlalchemy objects which cause trouble
 ignored-classes=SQLObject,Session
 
 # List of members which are set dynamically and missed by pylint inference

AUTHORS.rst

@@ -19,4 +19,4 @@ DIRAC consortium members
 - UM (Montpellier, France)
 - PNNL (USA)
 - KEK (Japan)
-- Imperial College (UK)
+- Imperial College (UK)

Core/scripts/dirac-install.py

@@ -3,12 +3,13 @@
 import stat
 import sys
 import tempfile
+
 try:
-  # For Python 3.0 and later
-  from urllib.request import urlopen
+    # For Python 3.0 and later
+    from urllib.request import urlopen
 except ImportError:
-  # Fall back to Python 2's urllib2
-  from urllib2 import urlopen
+    # Fall back to Python 2's urllib2
+    from urllib2 import urlopen
 
 sys.stderr.write("#" * 100 + "\n")
 sys.stderr.write("#" * 100 + "\n")
@@ -28,15 +29,13 @@
 sys.stderr.write("#" * 100 + "\n")
 
 if os.environ.get("DIRAC_DEPRECATED_FAIL", "No").lower() in ("y", "yes", "t", "true", "on", "1"):
-  raise RuntimeError("Failing as DIRAC_DEPRECATED_FAIL is set")
+    raise RuntimeError("Failing as DIRAC_DEPRECATED_FAIL is set")
 
 # Download dirac-install.py
-response = urlopen(
-    "https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py"
-)
+response = urlopen("https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py")
 code = response.getcode()
 if code > 200 or code >= 300:
-  raise RuntimeError("Failed to download dirac-install.py with code %s" % code)
+    raise RuntimeError("Failed to download dirac-install.py with code %s" % code)
 
 # Write dirac-install.py to a temporay file
 tmpHandle, tmp = tempfile.mkstemp()

container/README.md

@@ -26,4 +26,4 @@ docker cp <container_ID>:/opt/dirac/user/client.key ~/.globus
 docker cp <container_ID>:/opt/dirac/user/client.pem ~/.globus
 ```
 
-You can then use these `client.{key, pem}` files for generating a proxy with dirac-proxy-init explicitly using the -K and -C options.
+You can then use these `client.{key, pem}` files for generating a proxy with dirac-proxy-init explicitly using the -K and -C options.

container/docker_compose_setup/README.md

@@ -1,7 +1,7 @@
 # DIRAC in Docker Compose Setup for Development
 ## Requirements
 We assume that your system has `Docker` and `Docker Compose` running if not then follow [here](https://www.docker.com/).
-## Initial Setup 
+## Initial Setup
 Run the below commands for the setup of DIRAC Devbox, MySQL, and ElasticSearch:
 ```
 source .env
@@ -37,13 +37,13 @@ Status of installed components:
 /opt/dirac/startup/Web_WebApp: run (pid 653) 59 seconds
 ```
 
-## DIRAC Client Setup 
+## DIRAC Client Setup
 Run the below commands:
 ```
 ~/DiracInstallation/setupDIRACClient.sh
 ```
 
-After this you can access the WebAppDIRAC from your local machine at the following URLs: 
+After this you can access the WebAppDIRAC from your local machine at the following URLs:
 ```
 http://localhost:8089
 https://localhost:8443
@@ -71,4 +71,4 @@ In order to change the versions you need to point the correct `dockerfiles` insi
 In order to stop, remove, and bring all volume mounts down use this command:
 ```
 docker-compose down --volume
-```
+```

container/docker_compose_setup/dockerfiles/ES7_Dockerfile

@@ -1 +1 @@
-FROM docker.elastic.co/elasticsearch/elasticsearch:7.0.0
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.0.0

container/docker_compose_setup/dockerfiles/MySQL57_Dockerfile

@@ -1,3 +1,3 @@
 FROM mysql/mysql-server:5.7
 
-COPY ./resources/setupMySQL.sql /resources/setupMySQL.sql
+COPY ./resources/setupMySQL.sql /resources/setupMySQL.sql

container/docker_compose_setup/dockerfiles/MySQL80_Dockerfile

@@ -1,3 +1,3 @@
 FROM mysql/mysql-server:8.0
 
-COPY ./resources/setupMySQL.sql /resources/setupMySQL.sql
+COPY ./resources/setupMySQL.sql /resources/setupMySQL.sql

container/docker_compose_setup/env_files/env_install.cfg

@@ -49,7 +49,7 @@ LocalInstallation
   Services += Framework/SystemAdministrator
   Services += Framework/Monitoring
   Services += Framework/UserProfileManager
-  
+
   #  Flag determining whether the Web Portal will be installed
   WebPortal = yes
   WebApp = yes

container/docker_compose_setup/env_files/env_setupMySQL.sql

@@ -1,3 +1,3 @@
 CREATE USER 'root'@'%' IDENTIFIED BY '$root_pass';
 GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION;
-FLUSH PRIVILEGES;
+FLUSH PRIVILEGES;

container/docker_compose_setup/resources/install.cfg

@@ -49,7 +49,7 @@ LocalInstallation
   Services += Framework/SystemAdministrator
   Services += Framework/Monitoring
   Services += Framework/UserProfileManager
-  
+
   #  Flag determining whether the Web Portal will be installed
   WebPortal = yes
   WebApp = yes

container/docker_compose_setup/resources/runsvdir-start

@@ -3,4 +3,4 @@ RUNSVCTRL='/sbin/runsvctrl'
 chpst -u dirac $RUNSVCTRL d /opt/dirac/startup/*
 killall runsv svlogd
 RUNSVDIR='/sbin/runsvdir'
-exec chpst -u dirac $RUNSVDIR -P /opt/dirac/startup 'log:  DIRAC runsv'
+exec chpst -u dirac $RUNSVDIR -P /opt/dirac/startup 'log:  DIRAC runsv'

container/docker_compose_setup/resources/runsvdir.conf

@@ -3,4 +3,4 @@ start on runlevel [2345]
 stop on runlevel [^2345]
 normal exit 0 111
 respawn
-exec /opt/dirac/sbin/runsvdir-start 
+exec /opt/dirac/sbin/runsvdir-start

container/docker_compose_setup/resources/setupCA.sh

@@ -8,4 +8,4 @@ export CI_CONFIG=~/caUtilities/
 source utilities.sh
 generateCA
 generateCertificates 365
-generateUserCredentials 365
+generateUserCredentials 365

container/docker_compose_setup/resources/setupDIRAC.sh

@@ -27,5 +27,3 @@ curl -O -L https://github.com/DIRACGrid/DIRAC/raw/integration/Core/scripts/insta
 chmod +x install_site.sh
 cp ../install.cfg .
 chmod +x /resources/setupDIRACClient.sh && cp /resources/setupDIRACClient.sh ~/DiracInstallation/
-
-

container/docker_compose_setup/resources/setupDIRACClient.sh

@@ -6,4 +6,4 @@ source /opt/dirac/bashrc
 dirac-proxy-init -g dirac_admin
 cd ~/.globus/
 openssl pkcs12 -export -out certificate.p12 -inkey userkey.pem -in usercert.pem
-cd
+cd

container/docker_compose_setup/resources/setupMySQL.sql

@@ -1,3 +1,3 @@
 CREATE USER 'root'@'%' IDENTIFIED BY 'demopass';
 GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION;
-FLUSH PRIVILEGES;
+FLUSH PRIVILEGES;

container/docker_compose_setup/setup.sh

@@ -1,4 +1,4 @@
-envsubst < env_files/env_docker-compose.yml > docker-compose.yml  
+envsubst < env_files/env_docker-compose.yml > docker-compose.yml
 envsubst < env_files/env_setupMySQL.sql > resources/setupMySQL.sql
 envsubst < env_files/env_install.cfg > resources/install.cfg
 

dashboards/diracLogsDashboard.json

@@ -188,4 +188,4 @@
       "savedObjectVersion": 2
     }
   }
-]
+]

dirac.cfg

@@ -7,16 +7,16 @@ Registry
   ## Registry options:
   # Default user group to be used:
   DefaultGroup = lhcb_user
-  
+
   # Querantine user group is usually to be used in case you want to set
   # users in groups by hand as a "punishment" for a certain period of time:
   QuarantineGroup = lowPriority_user
-  
+
   # Default proxy time expressed in seconds:
   DefaultProxyTime = 4000
   ##
-  
-  # Trusted hosts section, subsections represents host name of the DIRAC secondary servers 
+
+  # Trusted hosts section, subsections represents host name of the DIRAC secondary servers
   Hosts
   {
 
@@ -64,7 +64,7 @@ Registry
         # The host name of the VOMS server
         cclcgvomsli01.in2p3.fr
         {
-          
+
           # DN of the VOMS server certificate
           DN = /O=GRID-FR/C=FR/O=CNRS/OU=CC-IN2P3/CN=cclcgvomsli01.in2p3.fr
 
@@ -78,7 +78,7 @@ Registry
     }
   }
   ##
-  
+
   ## Groups:
   # DIRAC groups section, subsections represents the name of the group
   Groups
@@ -93,25 +93,25 @@ Registry
 
       # Group properties(set permissions of the group users)
       Properties = NormalUser # Normal user operations
-      
+
       # Permission to download proxy with this group, by default: True
       DownloadableProxy = False
-      
+
       # Role of the users in the VO
       VOMSRole = /lhcb
-      
+
       # Virtual organization associated with the group
       VOMSVO = lhcb
-      
+
       # Just for normal users:
       JobShare = 200
-      
+
       # Controls automatic Proxy upload by dirac-proxy-init:
       AutoUploadProxy = True
-      
+
       # Controls automatic Proxy upload by dirac-proxy-init for Pilot groups:
       AutoUploadPilotProxy = True
-      
+
       # Controls automatic addition of VOMS extension by dirac-proxy-init:
       AutoAddVOMS = True
     }
@@ -147,16 +147,16 @@ Registry
     {
       # Distinguish name obtained from user certificate (Mandatory)
       DN = /O=My organisation/C=FR/OU=Unit/CN=My Name
-      
+
       # User e-mail (Mandatory)
       Email = my@email.com
-      
+
       # Cellular phone number
       mobile = +030621555555
-      
+
       # Quota assigned to the user. Expressed in MBs.
       Quota = 300
-      
+
       # This subsection describes the properties associated with each DN attribute (optional)
       DNProperties
       {
@@ -167,7 +167,7 @@ Registry
 
           # Distinguish name obtained from user certificate (Mandatory)
           DN = /O=My organisation/C=FR/OU=Unit/CN=My Name
-          
+
           # Proxy provider that can generate the proxy certificate with DN in DN attribute.
           ProxyProviders = MY_DIRACCA
         }
@@ -479,7 +479,7 @@ Resources
     ## OAuth2 type:
     MY_OAuth2
     {
-      
+
       ProviderType = OAuth2
 
       # Authorization server's issuer identifier URL
@@ -625,7 +625,7 @@ Operations
         }
       }
     }
-    # Options for the pilot3 
+    # Options for the pilot3
     # See https://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/WorkloadManagement/Pilots/Pilots3.html
     Pilot
     {
@@ -671,4 +671,4 @@ Operations
   <MySetup>
   {
   }
-}
+}

docs/README

@@ -24,4 +24,3 @@ How to build DIRAC documentation
     diracdoctools/scripts/dirac-docs-build-commands.py
     diracdoctools/scripts/dirac-docs-build-code.py
     diracdoctools/scripts/dirac-docs-concatenate-diraccfg.py
-