Merge https://github.com/matomo-org/matomo into stylelint

CHANGELOG.md

@@ -7,13 +7,15 @@ The Product Changelog at **[matomo.org/changelog](https://matomo.org/changelog)*
 ## Matomo 3.9.0
 
 ### Breaking Changes
-
 * `Referrers.getKeywordsForPageUrl` and `Referrers.getKeywordsForPageTitle` APIs have been deprecated and will be removed in Matomo 4.0.0
+* By default, Matomo [application logs](https://matomo.org/faq/troubleshooting/faq_115/) will now be logged in `tmp/logs/matomo.log` instead of `tmp/logs/piwik.log`. This log file path can be edited in your config/config.ini.php in the INI setting `logger_file_path`.
 
 ### New Features
+* It is now possible to locate plugins in a custom directory by setting an environment variable `MATOMO_PLUGIN_DIRS` or a `$GLOBALS['MATOMO_PLUGIN_DIRS']` variable in `$MATOMO_ROOT/bootstrap.php`.
 * It is now possible to use monolog's FingersCrossedHandler which buffers all logs and logs all of them in case of warning or error.
 
 ### New APIs
+* New API methods `Piwik\Plugin\Manager::getPluginsDirectories()` and  `Piwik\Plugin\Manager::getPluginDirectory($pluginName)` have been added as it is now possible to locate Matomo plugins in different directories and it should be no longer assumed a plugin is located in the "/plugins" directory.
 * A new tracker method `disableQueueRequest` has been added to disable queued requests which may be useful when logs are imported.
 * The event `LanguageManager.getAvailableLanguages` has been deprecated. Use `LanguagesManager.getAvailableLanguages` instead.
 

config/global.ini.php

@@ -49,8 +49,8 @@
 host = localhost
 username = "@USERNAME@"
 password =
-dbname = piwik_tests
-tables_prefix = piwiktests_
+dbname = matomo_tests
+tables_prefix = matomotests_
 port = 3306
 adapter = PDO\MYSQL
 type = InnoDB
@@ -102,7 +102,7 @@
 ; log_level_file =
 
 ; if configured to log in a file, log entries will be made to this file
-logger_file_path = tmp/logs/piwik.log
+logger_file_path = tmp/logs/matomo.log
 
 [Cache]
 ; available backends are 'file', 'array', 'null', 'redis', 'chained'
@@ -442,7 +442,7 @@
 enable_framed_settings = 0
 
 ; language cookie name for session
-language_cookie_name = piwik_lang
+language_cookie_name = matomo_lang
 
 ; standard email address displayed when sending emails
 noreply_email_address = "noreply@{DOMAIN}"
@@ -744,10 +744,12 @@
 ; `_paq.push(['setSessionCookieTimeout', timeoutInSeconds=1800])`
 visit_standard_length = 1800
 
-; The window to look back for a previous visit by this current visitor. Defaults to visit_standard_length.
+; The amount of time in the past to match the current visitor to a known visitor via fingerprint. Defaults to visit_standard_length.
 ; If you are looking for higher accuracy of "returning visitors" metrics, you may set this value to 86400 or more.
 ; This is especially useful when you use the Tracking API where tracking Returning Visitors often depends on this setting.
-; The value window_look_back_for_visitor is used only if it is set to greater than visit_standard_length
+; The value window_look_back_for_visitor is used only if it is set to greater than visit_standard_length.
+; Note: visitors with visitor IDs will be matched by visitor ID from any point in time, this is only for recognizing visitors
+; by device fingerprint.
 window_look_back_for_visitor = 0
 
 ; visitors that stay on the website and view only one page will be considered as time on site of 0 second

config/global.php

@@ -79,6 +79,16 @@
 
     'observers.global' => array(),
 
+    /**
+     * By setting this option to false, the check that the DB schema version matches the version of the source code will be no longer performed.
+     * Thus it allows you to execute for example a newer version of Matomo with an older Matomo database version. Please note
+     * disabling this setting is not recommended because often an older DB version is not compatible with newer source code.
+     * If you disable this setting, make sure to execute the updates after updating the source code. The setting can be useful if
+     * you want to update Matomo without any outage when you know the current source code update will still run fine for a short time
+     * while in the background the database updates are running.
+     */
+    'EnableDbVersionCheck' => true,
+
     'fileintegrity.ignore' => DI\add(array(
         '*.htaccess',
         '*web.config',
@@ -101,6 +111,7 @@
         'misc/user/*png',
         'misc/user/*svg',
         'misc/user/*js',
+        'misc/user/*/config.ini.php',
         'misc/package',
         'misc/package/WebAppGallery/*.xml',
         'misc/package/WebAppGallery/install.sql',

core/API/Proxy.php

@@ -442,7 +442,7 @@ private function getRequestParametersArray($requiredParameters, $parametersReque
     private function includeApiFile($fileName)
     {
         $module = self::getModuleNameFromClassName($fileName);
-        $path = Manager::getPluginsDirectory() . $module . '/API.php';
+        $path = Manager::getPluginDirectory($module) . '/API.php';
 
         if (is_readable($path)) {
             require_once $path; // prefixed by PIWIK_INCLUDE_PATH

core/Application/Kernel/EnvironmentValidator.php

@@ -77,7 +77,7 @@ private function checkConfigFileExists($path, $startInstaller = false)
 
         $message = $this->getSpecificMessageWhetherFileExistsOrNot($path);
 
-        $exception = new \Exception($message);
+        $exception = new NotYetInstalledException($message);
 
         if ($startInstaller) {
             $this->startInstallation($exception);

core/AssetManager/UIAsset/OnDiskUIAsset.php

@@ -10,6 +10,7 @@
 
 use Exception;
 use Piwik\AssetManager\UIAsset;
+use Piwik\Common;
 use Piwik\Filesystem;
 
 class OnDiskUIAsset extends UIAsset
@@ -24,14 +25,27 @@ class OnDiskUIAsset extends UIAsset
      */
     private $relativeLocation;
 
+    /**
+     * @var string
+     */
+    private $relativeRootDir;
+
     /**
      * @param string $baseDirectory
      * @param string $fileLocation
      */
-    public function __construct($baseDirectory, $fileLocation)
+    public function __construct($baseDirectory, $fileLocation, $relativeRootDir = '')
     {
         $this->baseDirectory = $baseDirectory;
         $this->relativeLocation = $fileLocation;
+
+        if (!empty($relativeRootDir)
+            && is_string($relativeRootDir)
+            && !Common::stringEndsWith($relativeRootDir, '/')) {
+            $relativeRootDir .= '/';
+        }
+
+        $this->relativeRootDir = $relativeRootDir;
     }
 
     public function getAbsoluteLocation()
@@ -41,6 +55,9 @@ public function getAbsoluteLocation()
 
     public function getRelativeLocation()
     {
+        if (isset($this->relativeRootDir)) {
+            return $this->relativeRootDir . $this->relativeLocation;
+        }
         return $this->relativeLocation;
     }
 

core/AssetManager/UIAssetFetcher.php

@@ -9,6 +9,7 @@
 namespace Piwik\AssetManager;
 
 use Piwik\AssetManager\UIAsset\OnDiskUIAsset;
+use Piwik\Plugin\Manager;
 use Piwik\Theme;
 
 abstract class UIAssetFetcher
@@ -89,9 +90,39 @@ private function initCatalog()
 
     private function populateCatalog()
     {
+        $pluginBaseDir = Manager::getPluginsDirectory();
+        $pluginWebDirectories = Manager::getAlternativeWebRootDirectories();
+        $matomoRootDir = $this->getBaseDirectory();
+
         foreach ($this->fileLocations as $fileLocation) {
+            $fileAbsolute = $matomoRootDir . '/' . $fileLocation;
+
             $newUIAsset = new OnDiskUIAsset($this->getBaseDirectory(), $fileLocation);
-            $this->catalog->addUIAsset($newUIAsset);
+            if ($newUIAsset->exists()) {
+                $this->catalog->addUIAsset($newUIAsset);
+                continue;
+            }
+
+            $found = false;
+
+            if (strpos($fileAbsolute, $pluginBaseDir) === 0) {
+                // we iterate over all custom plugin directories only for plugin files, not libs files (not needed there)
+                foreach ($pluginWebDirectories as $pluginDirectory => $relative) {
+                    $fileTest = str_replace($pluginBaseDir, $pluginDirectory, $fileAbsolute);
+                    $newFileRelative = str_replace($pluginDirectory, '', $fileTest);
+                    $testAsset = new OnDiskUIAsset($pluginDirectory, $newFileRelative, $relative);
+                    if ($testAsset->exists()) {
+                        $this->catalog->addUIAsset($testAsset);
+                        $found = true;
+                        break;
+                    }
+                }
+            }
+
+            if (!$found) {
+                // we add it anyway so it'll trigger an error about the missing file
+                $this->catalog->addUIAsset($newUIAsset);
+            }
         }
     }
 

core/AssetManager/UIAssetMerger/StylesheetUIAssetMerger.php

@@ -15,6 +15,7 @@
 use Piwik\Common;
 use Piwik\Exception\StylesheetLessCompileException;
 use Piwik\Piwik;
+use Piwik\Plugin\Manager;
 
 class StylesheetUIAssetMerger extends UIAssetMerger
 {
@@ -191,8 +192,9 @@ private function rewriteCssImportPaths($content, $pathsRewriter)
     private function getCssPathsRewriter($uiAsset)
     {
         $baseDirectory = dirname($uiAsset->getRelativeLocation());
+        $webDirs = Manager::getAlternativeWebRootDirectories();
 
-        return function ($matches) use ($baseDirectory) {
+        return function ($matches) use ($baseDirectory, $webDirs) {
             $absolutePath = PIWIK_DOCUMENT_ROOT . "/$baseDirectory/" . $matches[2];
 
             // Allow to import extension less file
@@ -201,13 +203,39 @@ private function getCssPathsRewriter($uiAsset)
             }
 
             // Prevent from rewriting full path
-            $absolutePath = realpath($absolutePath);
-            if ($absolutePath) {
+            $absolutePathReal = realpath($absolutePath);
+            if ($absolutePathReal) {
                 $relativePath = $baseDirectory . "/" . $matches[2];
                 $relativePath = str_replace('\\', '/', $relativePath);
                 $publicPath   = $matches[1] . $relativePath;
             } else {
-                $publicPath   = $matches[1] . $matches[2];
+                foreach ($webDirs as $absPath => $relativePath) {
+                    if (strpos($baseDirectory, $relativePath) === 0) {
+                        if (strpos($matches[2], '.') === 0) {
+                            // eg ../images/ok.png
+                            $fileRelative = $baseDirectory . '/' . $matches[2];
+                            $fileAbsolute = $absPath . str_replace($relativePath, '', $fileRelative);
+                            if (file_exists($fileAbsolute)) {
+                                return $matches[1] . $fileRelative;
+                            }
+                        } elseif (strpos($matches[2], 'plugins/') === 0) {
+                            // eg plugins/Foo/images/ok.png
+                            $fileRelative = substr($matches[2], strlen('plugins/'));
+                            $fileAbsolute = $absPath . $fileRelative;
+                            if (file_exists($fileAbsolute)) {
+                                return $matches[1] . $relativePath . $fileRelative;
+                            }
+                        } elseif ($matches[1] === '@import "') {
+                            $fileRelative = $baseDirectory . '/' . $matches[2];
+                            $fileAbsolute = $absPath . str_replace($relativePath, '', $fileRelative);
+                            if (file_exists($fileAbsolute)) {
+                                return $matches[1] . $baseDirectory . '/' . $matches[2];
+                            }
+                        }
+                    }
+                }
+
+                $publicPath = $matches[1] . $matches[2];
             }
 
             return $publicPath;

core/Columns/Updater.php

@@ -342,11 +342,13 @@ public function onNoUpdateAvailable($versionsThatWereChecked)
 
     private static function getCurrentDimensionFileChanges()
     {
-        $files = Filesystem::globr(Manager::getPluginsDirectory() . '*/Columns', '*.php');
-
         $times = array();
-        foreach ($files as $file) {
-            $times[$file] = filemtime($file);
+        foreach (Manager::getPluginsDirectories() as $pluginsDir) {
+            $files = Filesystem::globr($pluginsDir . '*/Columns', '*.php');
+
+            foreach ($files as $file) {
+                $times[$file] = filemtime($file);
+            }
         }
 
         return $times;

core/Container/ContainerFactory.php

@@ -121,7 +121,7 @@ private function addEnvironmentConfig(ContainerBuilder $builder, $environment)
         // add plugin environment configs
         $plugins = $this->pluginList->getActivatedPlugins();
         foreach ($plugins as $plugin) {
-            $baseDir = Manager::getPluginsDirectory() . $plugin;
+            $baseDir = Manager::getPluginDirectory($plugin);
 
             $environmentFile = $baseDir . '/config/' . $environment . '.php';
             if (file_exists($environmentFile)) {
@@ -135,7 +135,7 @@ private function addPluginConfigs(ContainerBuilder $builder)
         $plugins = $this->pluginList->getActivatedPlugins();
 
         foreach ($plugins as $plugin) {
-            $baseDir = Manager::getPluginsDirectory() . $plugin;
+            $baseDir = Manager::getPluginDirectory($plugin);
 
             $file = $baseDir . '/config/config.php';
             if (file_exists($file)) {

core/DataAccess/LogAggregator.php

@@ -805,15 +805,15 @@ protected function getActionsMetricFields()
      *                                 clause. These can be aggregate expressions, eg, `SUM(somecol)`.
      * @return \Zend_Db_Statement
      */
-    public function queryConversionsByDimension($dimensions = array(), $where = false, $additionalSelects = array())
+    public function queryConversionsByDimension($dimensions = array(), $where = false, $additionalSelects = array(), $extraFrom = [])
     {
         $dimensions = array_merge(array(self::IDGOAL_FIELD), $dimensions);
         $tableName  = self::LOG_CONVERSION_TABLE;
         $availableMetrics = $this->getConversionsMetricFields();
 
         $select = $this->getSelectStatement($dimensions, $tableName, $additionalSelects, $availableMetrics);
 
-        $from    = array($tableName);
+        $from    = array_merge([$tableName], $extraFrom);
         $where   = $this->getWhereStatement($tableName, self::CONVERSION_DATETIME_FIELD, $where);
         $groupBy = $this->getGroupByStatement($dimensions, $tableName);
         $orderBy = false;

core/FrontController.php

@@ -626,6 +626,10 @@ private function throwIfPiwikVersionIsOlderThanDBSchema()
             return;
         }
 
+        if (!StaticContainer::get('EnableDbVersionCheck')) {
+            return;
+        }
+
         $updater = new Updater();
 
         $dbSchemaVersion = $updater->getCurrentComponentVersion('core');

core/Plugin.php

@@ -355,9 +355,9 @@ public function findComponent($componentName, $expectedSubclass)
 
         $cacheId = 'Plugin' . $this->pluginName . $componentName . $expectedSubclass;
 
-        $pluginsDir = Manager::getPluginsDirectory();
+        $pluginsDir = Manager::getPluginDirectory($this->pluginName);
 
-        $componentFile = sprintf('%s%s/%s.php', $pluginsDir, $this->pluginName, $componentName);
+        $componentFile = sprintf('%s/%s.php', $pluginsDir, $componentName);
 
         if ($this->cache->contains($cacheId)) {
             $classname = $this->cache->fetch($cacheId);
@@ -537,8 +537,9 @@ private function doFindMultipleComponents($directoryWithinPlugin, $expectedSubcl
     {
         $components = array();
 
-        $pluginsDir = Manager::getPluginsDirectory();
-        $baseDir = $pluginsDir . $this->pluginName . '/' . $directoryWithinPlugin;
+        $pluginsDir = Manager::getPluginDirectory($this->pluginName);
+        $baseDir = $pluginsDir . '/' . $directoryWithinPlugin;
+
         $files   = Filesystem::globr($baseDir, '*.php');
 
         foreach ($files as $file) {

core/Plugin/ArchivedMetric.php

@@ -64,6 +64,11 @@ public function __construct(Dimension $dimension, $aggregation = false)
         $this->aggregation = $aggregation;
     }
 
+    public function getAggregation()
+    {
+        return $this->aggregation;
+    }
+
     public function setDimension($dimension)
     {
         $this->dimension = $dimension;