[FEATURE] Add TypoScript provider for sites and sets

TYPO3 sites are enhanced to be able to operate as TypoScript
templates. They act similar to sys_template records with
"clear" and "root" flags set. By design a site TypoScript provider
always defines a new scope (root-flag) and does not inherit from parent
sites (in the rootline). That means it behaves as if the clear-flag is
set in a sys_template record.
This behavior is not configurable by design, as TypoScript code sharing
is intended to be implemented via sharable sets (introduced in #103437).

TypoScript dependencies can be included via sets dependencies.
This mechanism supersedes the previous static_file_include's
or manual `@import` statements (they are still fine for local includes,
but should be avoided for cross-set/extensions dependencies),
as sets are automatically ordered and deduplicated.

Resolves: #103439
Releases: main
Change-Id: I971743fc551e51d945f45335dc6ad76404c6edba
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/83119
Tested-by: Benjamin Kott <benjamin.kott@outlook.com>
Reviewed-by: Benjamin Franzke <ben@bnf.dev>
Reviewed-by: Benni Mack <benni@typo3.org>
Tested-by: core-ci <typo3@b13.com>
Tested-by: Benjamin Franzke <ben@bnf.dev>
Tested-by: Benni Mack <benni@typo3.org>
Reviewed-by: Benjamin Kott <benjamin.kott@outlook.com>

typo3/sysext/core/Classes/Configuration/SiteConfiguration.php

@@ -31,6 +31,7 @@
 use TYPO3\CMS\Core\SingletonInterface;
 use TYPO3\CMS\Core\Site\Entity\Site;
 use TYPO3\CMS\Core\Site\Entity\SiteSettings;
+use TYPO3\CMS\Core\Site\Entity\SiteTypoScript;
 use TYPO3\CMS\Core\Site\SiteSettingsFactory;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
@@ -50,6 +51,20 @@ class SiteConfiguration implements SingletonInterface
      */
     protected string $configFileName = 'config.yaml';
 
+    /**
+     * File naming containing TypoScript Setup.
+     *
+     * @internal
+     */
+    protected string $typoScriptSetupFileName = 'setup.typoscript';
+
+    /**
+     * File naming containing TypoScript Constants.
+     *
+     * @internal
+     */
+    protected string $typoScriptConstantsFileName = 'constants.typoscript';
+
     /**
      * YAML file name with all settings related to Content-Security-Policies.
      *
@@ -107,11 +122,12 @@ public function resolveAllExistingSites(bool $useCache = true): array
             // cast $identifier to string, as the identifier can potentially only consist of (int) digit numbers
             $identifier = (string)$identifier;
             $siteSettings = $this->siteSettingsFactory->getSettings($identifier, $configuration);
+            $siteTypoScript = $this->getSiteTypoScript($identifier);
             $configuration['contentSecurityPolicies'] = $this->getContentSecurityPolicies($identifier);
 
             $rootPageId = (int)($configuration['rootPageId'] ?? 0);
             if ($rootPageId > 0) {
-                $sites[$identifier] = new Site($identifier, $rootPageId, $configuration, $siteSettings);
+                $sites[$identifier] = new Site($identifier, $rootPageId, $configuration, $siteSettings, $siteTypoScript);
             }
         }
         $this->firstLevelCache = $sites;
@@ -133,10 +149,11 @@ public function resolveAllExistingSitesRaw(): array
             // cast $identifier to string, as the identifier can potentially only consist of (int) digit numbers
             $identifier = (string)$identifier;
             $siteSettings = new SiteSettings($configuration['settings'] ?? []);
+            $siteTypoScript = $this->getSiteTypoScript($identifier);
 
             $rootPageId = (int)($configuration['rootPageId'] ?? 0);
             if ($rootPageId > 0) {
-                $sites[$identifier] = new Site($identifier, $rootPageId, $configuration, $siteSettings);
+                $sites[$identifier] = new Site($identifier, $rootPageId, $configuration, $siteSettings, $siteTypoScript);
             }
         }
         return $sites;
@@ -213,6 +230,28 @@ public function load(string $siteIdentifier): array
         return $loader->load(GeneralUtility::fixWindowsFilePath($fileName), YamlFileLoader::PROCESS_IMPORTS);
     }
 
+    protected function getSiteTypoScript(string $siteIdentifier): ?SiteTypoScript
+    {
+        $data = [
+            'setup' => $this->typoScriptSetupFileName,
+            'constants' => $this->typoScriptConstantsFileName,
+        ];
+        $definitions = [];
+        foreach ($data as $type => $fileName) {
+            $path = $this->configPath . '/' . $siteIdentifier . '/' . $fileName;
+            if (file_exists($path)) {
+                $contents = @file_get_contents(GeneralUtility::fixWindowsFilePath($path));
+                if ($contents !== false) {
+                    $definitions[$type] = $contents;
+                }
+            }
+        }
+        if ($definitions === []) {
+            return null;
+        }
+        return new SiteTypoScript(...$definitions);
+    }
+
     protected function getContentSecurityPolicies(string $siteIdentifier): array
     {
         $fileName = $this->configPath . '/' . $siteIdentifier . '/' . $this->contentSecurityFileName;

typo3/sysext/core/Classes/Site/Entity/Site.php

@@ -82,17 +82,20 @@ class Site implements SiteInterface
 
     protected SiteSettings $settings;
 
+    protected ?SiteTypoScript $typoscript;
+
     /**
      * Sets up a site object, and its languages, error handlers and the settings
      */
-    public function __construct(string $identifier, int $rootPageId, array $configuration, SiteSettings $settings = null)
+    public function __construct(string $identifier, int $rootPageId, array $configuration, SiteSettings $settings = null, ?SiteTypoScript $typoscript = null)
     {
         $this->identifier = $identifier;
         $this->rootPageId = $rootPageId;
         if ($settings === null) {
             $settings = new SiteSettings($configuration['settings'] ?? []);
         }
         $this->settings = $settings;
+        $this->typoscript = $typoscript;
         // Merge settings back in configuration for backwards-compatibility
         $configuration['settings'] = $this->settings->getAll();
         $this->configuration = $configuration;
@@ -328,8 +331,12 @@ public function getSettings(): SiteSettings
      */
     public function isTypoScriptRoot(): bool
     {
-        return $this->sets !== [];
+        return $this->sets !== [] || $this->typoscript !== null;
+    }
 
+    public function getTypoScript(): ?SiteTypoScript
+    {
+        return $this->typoscript;
     }
 
     /**

typo3/sysext/core/Classes/Site/Entity/SiteTypoScript.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\Site\Entity;
+
+final readonly class SiteTypoScript
+{
+    public function __construct(
+        public ?string $setup = null,
+        public ?string $constants = null,
+    ) {}
+
+    public static function __set_state(array $state): self
+    {
+        return new self(...$state);
+    }
+}

typo3/sysext/core/Classes/Site/Set/SetDefinition.php

@@ -31,6 +31,7 @@ public function __construct(
         public array $dependencies = [],
         public array $optionalDependencies = [],
         public array $settingsDefinitions = [],
+        public ?string $typoscript = null,
         public array $settings = [],
     ) {}
 

typo3/sysext/core/Classes/Site/Set/YamlSetDefinitionProvider.php

@@ -102,6 +102,7 @@ protected function createDefinition(array $set, string $basePath): SetDefinition
                 ...$set,
                 'settingsDefinitions' => $settingsDefinitions,
             ];
+            $setData['typoscript'] ??= $basePath;
             return new SetDefinition(...$setData);
         } catch (\Error $e) {
             throw new \Exception('Invalid set definition: ' . json_encode($set), 1170859526, $e);

typo3/sysext/core/Classes/TypoScript/IncludeTree/IncludeNode/SiteTemplateInclude.php

@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * The main node created for TypoScript from site sets
+ * and %configPath/sites/{constants,setup}.typoscript.
+ *
+ * @internal: Internal tree structure.
+ */
+final class SiteTemplateInclude extends AbstractInclude
+{
+    protected bool $root = true;
+    protected bool $clear = true;
+
+    public function isRoot(): bool
+    {
+        return true;
+    }
+
+    public function isClear(): bool
+    {
+        return true;
+    }
+}

typo3/sysext/core/Classes/TypoScript/IncludeTree/SysTemplateTreeBuilder.php

@@ -26,6 +26,7 @@
 use TYPO3\CMS\Core\Package\PackageManager;
 use TYPO3\CMS\Core\Site\Entity\Site;
 use TYPO3\CMS\Core\Site\Entity\SiteInterface;
+use TYPO3\CMS\Core\Site\Set\SetRegistry;
 use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\DefaultTypoScriptInclude;
 use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\DefaultTypoScriptMagicKeyInclude;
 use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\ExtensionStaticInclude;
@@ -35,6 +36,7 @@
 use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeStaticFileFileInclude;
 use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
 use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SiteInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SiteTemplateInclude;
 use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SysTemplateInclude;
 use TYPO3\CMS\Core\TypoScript\Tokenizer\TokenizerInterface;
 use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
@@ -55,7 +57,7 @@
  * attached sys_template records, gets their content and various sub includes and takes care
  * of correct include order.
  *
- * This class together with TreeFromTokenLineStreamBuilder also takes care of conditions and
+ * This class together with TreeFromLineStreamBuilder also takes care of conditions and
  * imports ("@import" and "<INCLUDE_TYPOSCRIPT:"): Those create child nodes in the tree. To
  * evaluate conditions, the tree is later traversed, condition verdicts (true / false) are
  * determined, to see if condition's child nodes should be considered in AST.
@@ -89,6 +91,7 @@ public function __construct(
         private readonly PackageManager $packageManager,
         private readonly Context $context,
         private readonly TreeFromLineStreamBuilder $treeFromTokenStreamBuilder,
+        private readonly SetRegistry $setRegistry,
     ) {}
 
     /**
@@ -110,6 +113,15 @@ public function getTreeBySysTemplateRowsAndSite(
         $this->includedSysTemplateUids = [];
 
         $rootNode = new RootInclude();
+
+        $siteIsTypoScriptRoot = $site instanceof Site ? $site->isTypoScriptRoot() : false;
+        if ($siteIsTypoScriptRoot) {
+            $cacheIdentifier = 'site-template-' . $this->type . '-' . $site->getIdentifier();
+            $includeNode = $this->cache?->require($cacheIdentifier) ?: null;
+            $includeNode ??= $this->createSiteTemplateInclude($site, $cacheIdentifier);
+            $rootNode->addChild($includeNode);
+        }
+
         if (empty($sysTemplateRows)) {
             return $rootNode;
         }
@@ -121,13 +133,14 @@ public function getTreeBySysTemplateRowsAndSite(
         // sys_template records if the flag is set somewhere and if not, actively sets it dynamically for the
         // first templates. As a result, integrators do not need to think about the 'clear' flags at all for
         // simple instances, it 'just works'.
-        $atLeastOneSysTemplateRowHasClearFlag = false;
-        foreach ($sysTemplateRows as $sysTemplateRow) {
-            if (($this->type === 'constants' && $sysTemplateRow['clear'] & 1) || ($this->type === 'setup' && $sysTemplateRow['clear'] & 2)) {
-                $atLeastOneSysTemplateRowHasClearFlag = true;
-            }
-        }
+        $atLeastOneSysTemplateRowHasClearFlag = $siteIsTypoScriptRoot;
         if (!$atLeastOneSysTemplateRowHasClearFlag) {
+            foreach ($sysTemplateRows as $sysTemplateRow) {
+                if (($this->type === 'constants' && $sysTemplateRow['clear'] & 1) || ($this->type === 'setup' && $sysTemplateRow['clear'] & 2)) {
+                    $atLeastOneSysTemplateRowHasClearFlag = true;
+                    break;
+                }
+            }
             $firstRow = reset($sysTemplateRows);
             $firstRow['clear'] = $this->type === 'constants' ? 1 : 2;
             $sysTemplateRows[array_key_first($sysTemplateRows)] = $firstRow;
@@ -168,6 +181,84 @@ public function getTreeBySysTemplateRowsAndSite(
         return $rootNode;
     }
 
+    private function createSiteTemplateInclude(
+        Site $site,
+        string $cacheIdentifier
+    ): SiteTemplateInclude {
+        $includeNode = new SiteTemplateInclude();
+        $includeNode->setRoot(true);
+        $includeNode->setClear(true);
+
+        $this->addDefaultTypoScriptFromGlobals($includeNode);
+
+        $sets = $this->setRegistry->getSets(...$site->getSets());
+        if (count($sets) > 0) {
+            $includeSetInclude = new IncludeStaticFileFileInclude();
+            $includeSetInclude->setName('site:' . $site->getIdentifier() . ':sets');
+            $includeSetInclude->setPath('site:' . $site->getIdentifier() . '/');
+            foreach ($sets as $set) {
+                $this->handleSetInclude($includeSetInclude, rtrim($set->typoscript, '/') . '/', 'set:' . $set->name);
+                $this->addStaticMagicFromGlobals($includeSetInclude, 'set:' . $set->name);
+            }
+            $includeNode->addChild($includeSetInclude);
+        }
+
+        if ($this->type === 'constants') {
+            $this->addDefaultTypoScriptConstantsFromSite($includeNode, $site);
+        }
+
+        $siteTypoScript = $site->getTypoScript();
+        $content = $this->type === 'constants' ? $siteTypoScript?->constants : $siteTypoScript?->setup;
+        if ($content !== null) {
+            $includeNode->setLineStream($this->tokenizer->tokenize($content));
+            $this->treeFromTokenStreamBuilder->buildTree($includeNode, $this->type, $this->tokenizer);
+        }
+
+        $includeNode->setName(sprintf(
+            '[site:%s%s] %s',
+            $site->getIdentifier(),
+            $content === null ? '' : '/' . $this->type . '.typoscript',
+            $site->getConfiguration()['websiteTitle'] ?? ''
+        ));
+
+        $this->cache?->set($cacheIdentifier, $this->prepareNodeForCache($includeNode));
+
+        return $includeNode;
+    }
+
+    private function handleSetInclude(IncludeInterface $parentNode, string $path, string $label): void
+    {
+        $path = GeneralUtility::getFileAbsFileName($path);
+
+        // '/.../my_extension/Configuration/TypoScript/MyStaticInclude/include_static_file.txt'
+        $includeStaticFileFileIncludePath = $path . 'include_static_file.txt';
+        if (file_exists($path . 'include_static_file.txt')) {
+            $includeStaticFileFileInclude = new IncludeStaticFileFileInclude();
+            //$name = 'EXT:' . $extensionKey . '/' . $pathSegmentWithAppendedSlash . 'include_static_file.txt';
+            //$includeStaticFileFileInclude->setName($name);
+            $includeStaticFileFileInclude->setName($label . ':include_static_file.txt');
+            $includeStaticFileFileInclude->setPath($path . 'include_static_file.txt');
+            $parentNode->addChild($includeStaticFileFileInclude);
+            $includeStaticFileFileIncludeContent = (string)file_get_contents($includeStaticFileFileIncludePath);
+            // @todo: There is no array_unique() for DB based include_static_file content?!
+            $includeStaticFileFileIncludeArray = array_unique(GeneralUtility::trimExplode(',', $includeStaticFileFileIncludeContent, true));
+            foreach ($includeStaticFileFileIncludeArray as $includeStaticFileFileIncludeString) {
+                $this->handleSingleIncludeStaticFile($includeStaticFileFileInclude, $includeStaticFileFileIncludeString);
+            }
+        }
+
+        $fileName = $path . $this->type . '.typoscript';
+        if (file_exists($fileName)) {
+            $fileContent = file_get_contents($fileName);
+            $fileNode = new FileInclude();
+            $fileNode->setName($label . ':' . $this->type . '.typoscript');
+            $fileNode->setPath($fileName);
+            $fileNode->setLineStream($this->tokenizer->tokenize($fileContent));
+            $this->treeFromTokenStreamBuilder->buildTree($fileNode, $this->type, $this->tokenizer);
+            $parentNode->addChild($fileNode);
+        }
+    }
+
     /**
      * Add includes defined in a sys_template record.
      */