Skip to content

Commit 991bed7

Browse files
ckandelerckandeler
committed
Extend scanner change tracking
Also keep track of properties read during script execution, as was already done for rules and commands. Change-Id: Ida56f0d2e15e08ac2b0fe64b86aa8d3f6e370d3b Reviewed-by: Ivan Komissarov <ABBAPOH@gmail.com>
1 parent 2bb405b commit 991bed7

File tree

18 files changed

+204
-30
lines changed

18 files changed

+204
-30
lines changed

‎src/lib/corelib/buildgraph/buildgraphloader.cpp‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -763,18 +763,19 @@ static bool dependenciesAreEqual(const ResolvedProductConstPtr &p1,
763763
bool BuildGraphLoader::checkProductForChanges(const ResolvedProductPtr &restoredProduct,
764764
const ResolvedProductPtr &newlyResolvedProduct)
765765
{
766-
// This check must come first, as it can prevent build data rescuing as a side effect.
766+
// These two checks must come first and run always, as they can have side effects.
767767
// TODO: Similar special checks must be done for Environment.getEnv() and File.exists() in
768768
// commands (or possibly it could be reasonable to just forbid such "dynamic" constructs
769769
// within commands).
770-
if (checkForPropertyChanges(restoredProduct, newlyResolvedProduct))
770+
const bool propertyChanges = checkForPropertyChanges(restoredProduct, newlyResolvedProduct);
771+
const bool scannerChanges = checkForScannerChanges(restoredProduct, newlyResolvedProduct);
772+
if (propertyChanges || scannerChanges)
771773
return true;
774+
772775
if (!ruleListsAreEqual(restoredProduct->rules, newlyResolvedProduct->rules))
773776
return true;
774777
if (!dependenciesAreEqual(restoredProduct, newlyResolvedProduct))
775778
return true;
776-
if (checkForScannerChanges(restoredProduct, newlyResolvedProduct))
777-
return true;
778779
return false;
779780
}
780781

@@ -834,11 +835,14 @@ bool BuildGraphLoader::checkForScannerChanges(
834835
const ResolvedProductPtr &restoredProduct, const ResolvedProductPtr &newlyResolvedProduct)
835836
{
836837
bool changed = false;
837-
for (const ResolvedScannerConstPtr &oldScanner : restoredProduct->scanners) {
838+
for (const ResolvedScannerPtr &oldScanner : restoredProduct->scanners) {
838839
bool found = false;
839-
for (const ResolvedScannerConstPtr &newScanner : newlyResolvedProduct->scanners) {
840+
for (const ResolvedScannerPtr &newScanner : newlyResolvedProduct->scanners) {
840841
if (*oldScanner == *newScanner) {
841842
found = true;
843+
newScanner->propertiesRequested = oldScanner->propertiesRequested;
844+
newScanner->propertiesRequestedFromArtifacts
845+
= oldScanner->propertiesRequestedFromArtifacts;
842846
break;
843847
}
844848
}

‎src/lib/corelib/buildgraph/depscanner.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ static QStringList collectProductBuildDirectories(const ResolvedProduct *product
7777
}
7878

7979
DependencyScanner::DependencyScanner(
80-
ResolvedScannerConstPtr scanner, ScriptEngine *engine, ScannerPlugin *plugin)
80+
ResolvedScannerPtr scanner, ScriptEngine *engine, ScannerPlugin *plugin)
8181
: m_scanner(std::move(scanner))
8282
, m_engine(engine)
8383
, m_global(engine->context(), JS_NewObjectProto(engine->context(), m_engine->globalObject()))
@@ -195,12 +195,17 @@ QStringList DependencyScanner::evaluate(
195195
m_engine->globalObject(),
196196
int(args.size()),
197197
args.data()));
198+
199+
m_scanner->propertiesRequested += m_engine->propertiesRequestedInScript();
200+
unite(m_scanner->propertiesRequestedFromArtifacts, m_engine->propertiesRequestedFromArtifact());
198201
m_engine->clearRequestedProperties();
202+
199203
if (m_engine->checkForJsError(script.location())) {
200204
ErrorInfo err = m_engine->getAndClearJsError();
201205
err.prepend(Tr::tr("Error evaluating scan script"));
202206
throw err;
203207
}
208+
204209
QStringList list;
205210
if (JS_IsArray(result)) {
206211
const int count = getJsIntProperty(

‎src/lib/corelib/buildgraph/depscanner.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ class DependencyScanner
6363
{
6464
public:
6565
DependencyScanner(
66-
ResolvedScannerConstPtr scanner, ScriptEngine *engine, ScannerPlugin *plugin = nullptr);
66+
ResolvedScannerPtr scanner, ScriptEngine *engine, ScannerPlugin *plugin = nullptr);
6767

6868
QString id() const;
6969

@@ -82,7 +82,7 @@ class DependencyScanner
8282
const FileResourceBase *fileToScan,
8383
const PrivateScriptFunction &script);
8484

85-
ResolvedScannerConstPtr m_scanner;
85+
ResolvedScannerPtr m_scanner;
8686
ScriptEngine *m_engine = nullptr;
8787
ScopedJsValue m_global;
8888
ResolvedProduct *m_product = nullptr;

‎src/lib/corelib/buildgraph/executor.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1325,10 +1325,37 @@ void Executor::prepareProducts()
13251325
{
13261326
ProductPrioritySetter prioritySetter(m_allProducts);
13271327
prioritySetter.apply();
1328+
Set<QString> scannersToInvalidate;
13281329
for (const ResolvedProductPtr &product : std::as_const(m_buildableProducts)) {
13291330
EnvironmentScriptRunner(product.get(), m_evalContext.get(), m_project->environment)
13301331
.setupForBuild();
1332+
bool scannersInvalidated = false;
1333+
for (const ResolvedScannerPtr &scanner : product->scanners) {
1334+
if (scannerNeedsInvalidation(
1335+
scanner.get(), product.get(), m_productsByName, m_projectsByName)) {
1336+
scannersToInvalidate.insert(
1337+
!scanner->pluginName.isEmpty() ? scanner->pluginName
1338+
: scanner->scanScript.sourceCode());
1339+
scanner->propertiesRequested.clear();
1340+
scanner->propertiesRequestedFromArtifacts.clear();
1341+
scannersInvalidated = true;
1342+
}
1343+
}
1344+
if (scannersInvalidated) {
1345+
for (Artifact * const artifact :
1346+
filterByType<Artifact>(product->buildData->allNodes())) {
1347+
if (artifact->transformer) {
1348+
qCDebug(lcBuildGraph())
1349+
<< "forcing transformer re-execution for output artifact"
1350+
<< artifact->filePath() << "due to scanner invalidation";
1351+
artifact->transformer->markedForRerun = true;
1352+
}
1353+
}
1354+
product->buildData->markRescuableArtifactsOutOfDate();
1355+
}
13311356
}
1357+
for (const QString &scannerId : scannersToInvalidate)
1358+
m_project->buildData->rawScanResults.invalidateResults(scannerId);
13321359
}
13331360

13341361
void Executor::setupRootNodes()

‎src/lib/corelib/buildgraph/inputartifactscanner.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ Set<DependencyScanner *> InputArtifactScanner::scannersForArtifact(const Artifac
223223
InputArtifactScannerContext::DependencyScannerCacheItem &cache = scannerCache[fileTag];
224224
if (!cache) {
225225
QList<DependencyScannerPtr> cacheScanners;
226-
for (const ResolvedScannerConstPtr &scanner : product->scanners) {
226+
for (const ResolvedScannerPtr &scanner : product->scanners) {
227227
if (scanner->inputs.contains(fileTag)) {
228228
ScannerPlugin *plugin = nullptr;
229229
if (!scanner->pluginName.isEmpty()) {

‎src/lib/corelib/buildgraph/productbuilddata.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,12 @@ void ProductBuildData::addRescuableArtifactData(
121121
m_rescuableArtifactData[filePath] = std::move(rad);
122122
}
123123

124+
void ProductBuildData::markRescuableArtifactsOutOfDate()
125+
{
126+
for (RescuableArtifactData &rad : m_rescuableArtifactData)
127+
rad.knownOutOfDate = true;
128+
}
129+
124130
bool ProductBuildData::checkAndSetJsArtifactsMapUpToDateFlag()
125131
{
126132
if (!m_jsArtifactsMapUpToDate) {

‎src/lib/corelib/buildgraph/productbuilddata.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ class QBS_AUTOTEST_EXPORT ProductBuildData
8181
void setRescuableArtifactData(const AllRescuableArtifactData &rad);
8282
RescuableArtifactData removeFromRescuableArtifactData(const QString &filePath);
8383
void addRescuableArtifactData(const QString &filePath, RescuableArtifactData &&rad);
84+
void markRescuableArtifactsOutOfDate();
8485

8586
unsigned int buildPriority() const { return m_buildPriority; }
8687
void setBuildPriority(unsigned int prio) { m_buildPriority = prio; }

‎src/lib/corelib/buildgraph/transformerchangetracking.cpp‎

Lines changed: 56 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,23 @@ namespace Internal {
4949
class TrafoChangeTracker
5050
{
5151
public:
52-
TrafoChangeTracker(const Transformer *transformer,
53-
const ResolvedProduct *product,
54-
const std::unordered_map<QString, const ResolvedProduct *> &productsByName,
55-
const std::unordered_map<QString, const ResolvedProject *> &projectsByName)
56-
: m_transformer(transformer),
57-
m_product(product),
58-
m_productsByName(productsByName),
59-
m_projectsByName(projectsByName)
52+
TrafoChangeTracker(
53+
const Transformer *transformer,
54+
const ResolvedScanner *scanner,
55+
const ResolvedProduct *product,
56+
const std::unordered_map<QString, const ResolvedProduct *> &productsByName,
57+
const std::unordered_map<QString, const ResolvedProject *> &projectsByName)
58+
: m_transformer(transformer)
59+
, m_scanner(scanner)
60+
, m_product(product)
61+
, m_productsByName(productsByName)
62+
, m_projectsByName(projectsByName)
6063
{
6164
}
6265

6366
bool prepareScriptNeedsRerun() const;
6467
bool commandsNeedRerun() const;
68+
bool scannerNeedsInvalidation() const;
6569

6670
private:
6771
QVariantMap propertyMapByKind(const Property &property) const;
@@ -77,6 +81,7 @@ class TrafoChangeTracker
7781
const ResolvedProduct *getProduct(const QString &name) const;
7882

7983
const Transformer * const m_transformer;
84+
const ResolvedScanner * const m_scanner;
8085
const ResolvedProduct * const m_product;
8186
const std::unordered_map<QString, const ResolvedProduct *> &m_productsByName;
8287
const std::unordered_map<QString, const ResolvedProject *> &m_projectsByName;
@@ -352,6 +357,35 @@ bool TrafoChangeTracker::commandsNeedRerun() const
352357
return false;
353358
}
354359

360+
bool TrafoChangeTracker::scannerNeedsInvalidation() const
361+
{
362+
for (const Property &property : std::as_const(m_scanner->propertiesRequested)) {
363+
if (checkForPropertyChange(property, propertyMapByKind(property)))
364+
return true;
365+
}
366+
367+
// TODO: imports, deps, artifacts map, exported modules? (See prepare script checker)
368+
369+
for (auto it = m_scanner->propertiesRequestedFromArtifacts.cbegin();
370+
it != m_scanner->propertiesRequestedFromArtifacts.cend();
371+
++it) {
372+
for (const Property &property : std::as_const(it.value())) {
373+
const Artifact * const artifact = getArtifact(it.key(), property.productName);
374+
if (!artifact)
375+
return true;
376+
if (property.kind == Property::PropertyInArtifact) {
377+
if (sorted(artifact->fileTags().toStringList()) != property.value.toStringList())
378+
return true;
379+
continue;
380+
}
381+
if (checkForPropertyChange(property, artifact->properties->value()))
382+
return true;
383+
}
384+
}
385+
386+
return false;
387+
}
388+
355389
bool prepareScriptNeedsRerun(
356390
Transformer *transformer, const ResolvedProduct *product,
357391
const std::unordered_map<QString, const ResolvedProduct *> &productsByName,
@@ -360,8 +394,8 @@ bool prepareScriptNeedsRerun(
360394
if (!transformer->prepareScriptNeedsChangeTracking)
361395
return false;
362396
transformer->prepareScriptNeedsChangeTracking = false;
363-
return TrafoChangeTracker(transformer, product, productsByName, projectsByName)
364-
.prepareScriptNeedsRerun();
397+
return TrafoChangeTracker(transformer, nullptr, product, productsByName, projectsByName)
398+
.prepareScriptNeedsRerun();
365399
}
366400

367401
bool commandsNeedRerun(Transformer *transformer, const ResolvedProduct *product,
@@ -371,8 +405,18 @@ bool commandsNeedRerun(Transformer *transformer, const ResolvedProduct *product,
371405
if (!transformer->commandsNeedChangeTracking)
372406
return false;
373407
transformer->commandsNeedChangeTracking = false;
374-
return TrafoChangeTracker(transformer, product, productsByName, projectsByName)
375-
.commandsNeedRerun();
408+
return TrafoChangeTracker(transformer, nullptr, product, productsByName, projectsByName)
409+
.commandsNeedRerun();
410+
}
411+
412+
bool scannerNeedsInvalidation(
413+
const ResolvedScanner *scanner,
414+
const ResolvedProduct *product,
415+
const std::unordered_map<QString, const ResolvedProduct *> &productsByName,
416+
const std::unordered_map<QString, const ResolvedProject *> &projectsByName)
417+
{
418+
return TrafoChangeTracker(nullptr, scanner, product, productsByName, projectsByName)
419+
.scannerNeedsInvalidation();
376420
}
377421

378422
} // namespace Internal

‎src/lib/corelib/buildgraph/transformerchangetracking.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ bool commandsNeedRerun(Transformer *transformer,
4747
const std::unordered_map<QString, const ResolvedProduct *> &productsByName,
4848
const std::unordered_map<QString, const ResolvedProject *> &projectsByName);
4949

50+
bool scannerNeedsInvalidation(
51+
const ResolvedScanner *scanner,
52+
const ResolvedProduct *product,
53+
const std::unordered_map<QString, const ResolvedProduct *> &productsByName,
54+
const std::unordered_map<QString, const ResolvedProject *> &projectsByName);
55+
5056
} // namespace Internal
5157
} // namespace qbs
5258

‎src/lib/corelib/language/language.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,12 @@
4444
#include "forward_decls.h"
4545
#include "jsimports.h"
4646
#include "moduleproviderinfo.h"
47+
#include "property.h"
4748
#include "propertydeclaration.h"
4849
#include "resolvedfilecontext.h"
4950

5051
#include <buildgraph/forward_decls.h>
5152
#include <tools/codelocation.h>
52-
#include <tools/fileinfo.h>
5353
#include <tools/filetime.h>
5454
#include <tools/joblimits.h>
5555
#include <tools/persistence.h>
@@ -499,6 +499,8 @@ class ResolvedScanner
499499
CodeLocation location;
500500
PrivateScriptFunction searchPathsScript;
501501
PrivateScriptFunction scanScript;
502+
PropertySet propertiesRequested;
503+
QHash<QString, PropertySet> propertiesRequestedFromArtifacts;
502504

503505
template<PersistentPool::OpType opType> void completeSerializationOp(PersistentPool &pool)
504506
{
@@ -510,7 +512,9 @@ class ResolvedScanner
510512
pluginName,
511513
location,
512514
searchPathsScript,
513-
scanScript);
515+
scanScript,
516+
propertiesRequested,
517+
propertiesRequestedFromArtifacts);
514518
}
515519

516520
private:
@@ -652,7 +656,7 @@ class QBS_AUTOTEST_EXPORT ResolvedProduct
652656
JobLimits jobLimits;
653657
std::vector<ResolvedModulePtr> modules;
654658
QHash<ResolvedModuleConstPtr, QVariantMap> moduleParameters;
655-
std::vector<ResolvedScannerConstPtr> scanners;
659+
std::vector<ResolvedScannerPtr> scanners;
656660
std::vector<GroupPtr> groups;
657661
std::vector<ProbeConstPtr> probes;
658662
std::vector<ArtifactPropertiesPtr> artifactProperties;

0 commit comments

Comments
 (0)