Disable features that are inaccurate if visit data is not profilable

config/environment/test.php

@@ -135,5 +135,6 @@
         })),
     )),
 
+    'tests.isProfilableCheckDisabled' => true,
     'test.vars.forceCliMultiViaCurl' => false,
 );

core/API/DataTablePostProcessor.php

@@ -12,16 +12,20 @@
 use Piwik\API\DataTableManipulator\Flattener;
 use Piwik\API\DataTableManipulator\LabelFilter;
 use Piwik\API\DataTableManipulator\ReportTotalsCalculator;
+use Piwik\Archive\DataTableFactory;
 use Piwik\Common;
 use Piwik\DataTable;
 use Piwik\DataTable\DataTableInterface;
 use Piwik\DataTable\Filter\PivotByDimension;
+use Piwik\Metrics;
 use Piwik\Metrics\Formatter;
+use Piwik\Period;
 use Piwik\Piwik;
 use Piwik\Plugin\ProcessedMetric;
 use Piwik\Plugin\Report;
 use Piwik\Plugin\ReportsProvider;
 use Piwik\Plugins\API\Filter\DataComparisonFilter;
+use Piwik\Site;
 
 /**
  * Processes DataTables that should be served through Piwik's APIs. This processing handles
@@ -132,11 +136,53 @@ public function process(DataTableInterface $dataTable)
         $dataTable = $this->convertSegmentValueToSegment($dataTable);
         $dataTable = $this->applyQueuedFilters($dataTable);
         $dataTable = $this->applyRequestedColumnDeletion($dataTable);
+        $dataTable = $this->removeMetricsIfNotProfilable($dataTable);
         $dataTable = $this->applyLabelFilter($dataTable);
         $dataTable = $this->applyMetricsFormatting($dataTable);
         return $dataTable;
     }
 
+    private function removeMetricsIfNotProfilable(DataTableInterface $dataTable)
+    {
+        $dataTable->filter(function (DataTable $table) {
+            /** @var Site $site */
+            $site = $table->getMetadata('site');
+
+            /** @var Period $period */
+            $period = $table->getMetadata('period');
+
+            $segment = $table->getMetadata(DataTableFactory::TABLE_METADATA_SEGMENT_INDEX);
+
+            if (empty($site)
+                || empty($period)
+            ) {
+                return;
+            }
+
+            if (Request::isCurrentPeriodProfilable($site->getId(), $period->getLabel(), $period->getDateStart()->toString(), $segment)) {
+                return;
+            }
+
+            $metricsToRemove = [
+                Metrics::INDEX_NB_UNIQ_VISITORS,
+                Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS,
+                Metrics::INDEX_PAGE_ENTRY_NB_UNIQ_VISITORS,
+                Metrics::INDEX_PAGE_EXIT_NB_UNIQ_VISITORS,
+                Metrics::INDEX_PAGE_ENTRY_SUM_DAILY_NB_UNIQ_VISITORS,
+                Metrics::INDEX_PAGE_EXIT_SUM_DAILY_NB_UNIQ_VISITORS,
+            ];
+
+            $metricIdToNameMap = Metrics::getMappingFromIdToName();
+            foreach (array_values($metricsToRemove) as $indexMetric) {
+                $metricsToRemove[] = $metricIdToNameMap[$indexMetric];
+            }
+
+            $table->filter(DataTable\Filter\ColumnDelete::class, [$metricsToRemove]);
+        });
+
+        return $dataTable;
+    }
+
     private function convertSegmentValueToSegment(DataTableInterface $dataTable)
     {
         $dataTable->filter('AddSegmentBySegmentValue', array($this->report));

core/API/Request.php

@@ -9,6 +9,7 @@
 namespace Piwik\API;
 
 use Exception;
+use Matomo\Cache\Transient;
 use Piwik\Access;
 use Piwik\Cache;
 use Piwik\Common;
@@ -18,9 +19,11 @@
 use Piwik\DataTable;
 use Piwik\Exception\PluginDeactivatedException;
 use Piwik\IP;
+use Piwik\Period;
 use Piwik\Piwik;
 use Piwik\Plugin\Manager as PluginManager;
 use Piwik\Plugins\CoreHome\LoginAllowlist;
+use Piwik\Segment;
 use Piwik\SettingsServer;
 use Piwik\Url;
 use Piwik\UrlHelper;
@@ -701,4 +704,59 @@ private function shouldDisablePostProcessing()
 
         return $shouldDisable;
     }
+
+    public static function isCurrentPeriodProfilable($idSite = null, $period = null, $date = null, $segment = null)
+    {
+        if (self::isProfilableCheckDisabledInTests()) {
+            return true;
+        }
+
+        $idSite = $idSite ?: Common::getRequestVar('idSite', $default = false);
+        $period = $period ?: Common::getRequestVar('period', $default = false);
+        $date = $date ?: Common::getRequestVar('date', $default = false);
+        $segment = $segment ?: Request::getRawSegmentFromRequest();
+
+        if ($idSite === false
+            || $period === false
+            || $date === false
+            || !is_numeric($idSite)
+            || Period::isMultiplePeriod($date, $period)
+        ) {
+            return true;
+        }
+
+        $transientCache = StaticContainer::get(Transient::class);
+
+        $segmentObj = new Segment($segment, [$idSite]);
+
+        $cacheKey = "VisitsSummary.isProfilable.$idSite.$period.$date." . $segmentObj->getHash();
+        if (!$transientCache->contains($cacheKey)) {
+            $isProfilable = Request::processRequest('VisitsSummary.isProfilable', [
+                'idSite' => $idSite,
+                'period' => $period,
+                'date' => $date,
+                'segment' => $segment,
+            ]);
+
+            $transientCache->save($cacheKey, $isProfilable);
+        } else {
+            $isProfilable = (bool)$transientCache->fetch($cacheKey);
+        }
+
+        return $isProfilable;
+    }
+
+    private static function isProfilableCheckDisabledInTests()
+    {
+        if (!defined('PIWIK_TEST_MODE')) {
+            return false;
+        }
+
+        try {
+            $isDisabled = (bool) StaticContainer::get('tests.isProfilableCheckDisabled');
+            return $isDisabled;
+        } catch (\Exception $ex) {
+            return false;
+        }
+    }
 }

core/Archive.php

@@ -522,7 +522,7 @@ protected function get($archiveNames, $archiveDataType, $idSubtable = null)
      * query archive tables for IDs w/o launching archiving, or launch archiving and
      * get the idarchive from ArchiveProcessor instances.
      *
-     * @param string $archiveNames
+     * @param string[] $archiveNames
      * @return array
      */
     private function getArchiveIds($archiveNames)

core/Archive/DataCollection.php

@@ -112,6 +112,7 @@ public function __construct($dataNames, $dataType, $sitesId, $periods, $segment,
 
         if ($defaultRow === null) {
             $defaultRow = array_fill_keys($dataNames, 0);
+            unset($defaultRow['nb_profilable']); // for this special metric, we want to know when it was never present
         }
 
         $this->sitesId = $sitesId;

core/Columns/Dimension.php

@@ -192,6 +192,11 @@ abstract class Dimension
      */
     protected $metricId = '';
 
+    /**
+     * @var bool
+     */
+    private $isRequiresProfilableData = false;
+
     /**
      * To be implemented when a column references another column
      * @return Join|null
@@ -857,4 +862,17 @@ public function getVersion()
         return $this->columnType;
     }
 
+    /**
+     * Returns true if this dimension requires profilable visit data in order to be useful.
+     * Dimensions that require profilable data are hidden (as are their associated reports)
+     * when we find that a specific period contains a majority of un-profilable tracked data.
+     *
+     * Derived classes should set the $isRequiresProfilableData member if they are not profilable.
+     *
+     * @return mixed
+     */
+    public function isRequiresProfilableData()
+    {
+        return $this->isRequiresProfilableData;
+    }
 }

core/Columns/DimensionSegmentFactory.php

@@ -114,6 +114,8 @@ public function createSegment(Segment $segment = null)
             $segment->setRequiresRegisteredUser(true);
         }
 
+        $segment->setRequiresProfilableData($dimension->isRequiresProfilableData());
+
         return $segment;
     }
 

core/DataAccess/ArchiveWriter.php

@@ -211,8 +211,8 @@ private function batchInsertSpool($valueType)
 
         $valueSeen = false;
         foreach ($records as $record) {
-            // don't record zero
-            if (empty($record[1])) {
+            // don't record zero unless it's nb_profilable
+            if (empty($record[1]) && $record[0] != 'nb_profilable') {
                 continue;
             }
 
@@ -251,7 +251,8 @@ private function batchInsertSpool($valueType)
      */
     public function insertRecord($name, $value)
     {
-        if ($this->isRecordZero($value)) {
+        // NOTE: we allow nb_profilable to be there since periods before nb_profilable was archived will not have the data, and we want to know the difference
+        if ($this->isRecordZero($value) && $name != 'nb_profilable') {
             return false;
         }
 

core/DataAccess/LogAggregator.php

@@ -423,6 +423,7 @@ protected function getVisitsMetricFields()
             Metrics::INDEX_BOUNCE_COUNT                   => "sum(case " . self::LOG_VISIT_TABLE . ".visit_total_actions when 1 then 1 when 0 then 1 else 0 end)",
             Metrics::INDEX_NB_VISITS_CONVERTED            => "sum(case " . self::LOG_VISIT_TABLE . ".visit_goal_converted when 1 then 1 else 0 end)",
             Metrics::INDEX_NB_USERS                       => "count(distinct " . self::LOG_VISIT_TABLE . ".user_id)",
+            Metrics::INDEX_NB_PROFILABLE                  => "sum(" . self::LOG_VISIT_TABLE . ".profilable)",
         );
     }
 

core/DataTable/Filter/ColumnDelete.php

@@ -170,7 +170,6 @@ protected function removeColumnsFromTable(&$table)
             }
 
             foreach ($this->columnsToRemove as $column) {
-
                 if (is_array($row)) {
                     if (!array_key_exists($column, $row)) {
                         continue;

core/Metrics.php

@@ -40,6 +40,7 @@ class Metrics
     const INDEX_REVENUE = 9;
     const INDEX_GOALS = 10;
     const INDEX_SUM_DAILY_NB_UNIQ_VISITORS = 11;
+    const INDEX_NB_PROFILABLE = 44;
 
     // Specific to the Actions reports
     const INDEX_PAGE_NB_HITS = 12;
@@ -114,6 +115,7 @@ class Metrics
         Metrics::INDEX_GOALS                                 => 'goals',
         Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS            => 'sum_daily_nb_uniq_visitors',
         Metrics::INDEX_SUM_DAILY_NB_USERS                    => 'sum_daily_nb_users',
+        Metrics::INDEX_NB_PROFILABLE                         => 'nb_profilable',
 
         // Actions metrics
         Metrics::INDEX_PAGE_NB_HITS                          => 'nb_hits',
@@ -174,6 +176,7 @@ class Metrics
         Metrics::INDEX_SUM_VISIT_LENGTH,
         Metrics::INDEX_BOUNCE_COUNT,
         Metrics::INDEX_NB_VISITS_CONVERTED,
+        Metrics::INDEX_NB_PROFILABLE,
     );
 
     public static function getMappingFromIdToName()

core/Plugin/Report.php

@@ -15,6 +15,7 @@
 use Piwik\DataTable;
 use Piwik\DataTable\Filter\Sort;
 use Piwik\Metrics;
+use Piwik\Period;
 use Piwik\Piwik;
 use Piwik\Plugins\CoreVisualizations\Visualizations\HtmlTable;
 use Piwik\ViewDataTable\Factory as ViewDataTableFactory;
@@ -231,11 +232,23 @@ protected function init()
      * might depend on a setting (such as Ecommerce) of a site. In such a case you can perform any checks and then
      * return `true` or `false`. If your report is only available to users having super user access you can do the
      * following: `return Piwik::hasUserSuperUserAccess();`
+     *
+     * Classes that override this method must make sure to call the parent version as well.
+     *
      * @return bool
      * @api
      */
     public function isEnabled()
     {
+        if ($this->isRequiresProfilableData()
+            || (isset($this->dimension) && $this->dimension->isRequiresProfilableData())
+        ) {
+            $isProfilable = Request::isCurrentPeriodProfilable();
+            if (!$isProfilable) {
+                return false;
+            }
+        }
+
         return true;
     }
 
@@ -996,6 +1009,19 @@ public function getProcessedMetricsById()
         return $result;
     }
 
+    /**
+     * Returns `true` if this report requires visit data that is profilable (that is to say, visit data that accurately
+     * identifies visitors across visits, which generally requires using cookies or tracking user IDs). If a report requires
+     * this, and Matomo finds that existing data is not good enough, the reports will not be shown, since they will not be
+     * accurate.
+     *
+     * @api
+     */
+    public function isRequiresProfilableData()
+    {
+        return false;
+    }
+
     /**
      * Returns the Metrics that are displayed by a DataTable of a certain Report type.
      *

core/Plugin/Segment.php

@@ -64,6 +64,8 @@ class Segment
      */
     private $requiresRegisteredUser = false;
 
+    private $requiresProfilableData = false;
+
     /**
      * @ignore
      */
@@ -377,6 +379,8 @@ public function toArray()
             $segment['suggestedValuesApi'] = $this->suggestedValuesApi;
         }
 
+        $segment['requiresProfilableData'] = $this->requiresProfilableData;
+
         return $segment;
     }
 
@@ -436,4 +440,26 @@ private function check()
             throw new Exception(sprintf('The segment %s contains a union segment to itself', $this->name));
         }
     }
+
+    /**
+     * Returns true if this segment is only useful with profilable data, false if not.
+     *
+     * @return bool
+     */
+    public function isRequiresProfilableData(): bool
+    {
+        return $this->requiresProfilableData;
+    }
+
+    /**
+     * Set whether this segment is only useful with profilable data. If a segment requires
+     * profilable data and Matomo detects that this data is not present, the segment will be
+     * hidden in the UI. (It will still work in the API.)
+     *
+     * @param bool $requiresProfilableData
+     */
+    public function setRequiresProfilableData(bool $requiresProfilableData): void
+    {
+        $this->requiresProfilableData = $requiresProfilableData;
+    }
 }