Disable pantsd in inner runs. (#7884)

**Problem**
When a pants run under pantsd calls pants, it will also enable pantsd in the inner run. Since we don't allow parallel runs under pantsd, the inner run will block waiting for the outer run to complete, eventually timing out and crashing the whole run.
See #7881 for context.
The only real instance we've seen of this was entirely accidental.

**Solution**
Explicitly disable pantsd through an environment variable whenever we are serving a pailgun request.

**Result**
Inner runs under pantsd have the daemon turned off by default.
Fixes #7881 , Depends on #7890

Commits should be independently reviewable.

src/python/pants/pantsd/pailgun_server.py

@@ -80,6 +80,11 @@ def handle(self):
     self.logger.info('handling pailgun request: `{}`'.format(' '.join(arguments)))
     self.logger.debug('pailgun request environment: %s', environment)
 
+    # We don't support parallel runs in pantsd, and therefore if this pailgun request
+    # triggers any pants command, we want it to not use pantsd at all.
+    # See https://github.com/pantsbuild/pants/issues/7881 for context.
+    environment["PANTS_CONCURRENT"] = "True"
+
     # Execute the requested command with optional daemon-side profiling.
     with maybe_profiled(environment.get('PANTSD_PROFILE')):
       self._run_pants(self.request, arguments, environment)

testprojects/src/python/nested_runs/BUILD

@@ -0,0 +1,6 @@
+python_binary(
+  name="nested_runs",
+  source="run_pants_with_pantsd.py",
+  compatibility=['CPython>=3.6,<4']
+)
+

testprojects/src/python/nested_runs/run_pants_with_pantsd.py

@@ -0,0 +1,23 @@
+import pathlib
+import subprocess
+import sys
+
+
+def main():
+  workdir = sys.argv[1]
+  config = pathlib.Path(workdir) / 'pants.ini'
+
+  cmd = [
+    './pants',
+    '--no-pantsrc',
+    f'--pants-config-files={config}',
+    '--print-exception-stacktrace=True',
+    f'--pants-workdir={workdir}',
+    'goals'
+  ]
+  print(f'Running pants with command {cmd}')
+  subprocess.run(cmd, check=True)
+
+
+if __name__ == '__main__':
+  main()

tests/python/pants_test/pantsd/test_pantsd_integration.py

@@ -771,3 +771,25 @@ def test_unhandled_exceptions_only_log_exceptions_once(self):
         'pants.bin.daemon_pants_runner._PantsRunFinishedWithFailureException: Terminated with 1',
         result.stderr_data,
       )
+
+  def test_inner_runs_dont_deadlock(self):
+    """
+    Create a pantsd run that calls testprojects/src/python/nested_runs with the appropriate
+    bootstrap options to avoid restarting pantsd.
+
+    Regression test for issue https://github.com/pantsbuild/pants/issues/7881
+    When a run under pantsd calls pants with pantsd inside it, the inner run will time out
+    waiting for the outer run to end.
+
+    NB: testprojects/src/python/nested_runs assumes that the pants.ini file is in ${workdir}/pants.ini
+    """
+    config = {
+      'GLOBAL': {
+        'pantsd_timeout_when_multiple_invocations': 1,
+      }
+    }
+    with self.pantsd_successful_run_context(extra_config=config) as (pantsd_run, checker, workdir, _):
+      result = pantsd_run(['run', 'testprojects/src/python/nested_runs', '--', workdir], expected_runs=2)
+      checker.assert_started()
+      self.assert_success(result)
+      self.assertNotIn("Another pants invocation is running", result.stderr_data)