Skip to content

Commit

Permalink
fix: store last modified timestamp in db (#6271)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeremylong
jeremylong committed Dec 12, 2023
1 parent 41fc5a2 commit 2a388f1
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 22 deletions.
20 changes: 20 additions & 0 deletions core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java
View file
Expand Up @@ -28,6 +28,7 @@
import javax.annotation.concurrent.ThreadSafe;

import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.utils.DateUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -72,6 +73,14 @@ public class DatabaseProperties {
* The key for the last check time for the Known Exploited Vulnerabilities.
*/
public static final String KEV_LAST_CHECKED = "kev.checked";
/**
* The key for the last check time for the Retire JS repository.
*/
public static final String RETIRE_LAST_CHECKED = "retirejs.checked";
/**
* The key for the last check time for the hosted suppression file.
*/
public static final String HOSTED_SUPPRESSION_LAST_CHECKED = "hosted.suppression.checked";
/**
* The key for the version the Known Exploited Vulnerabilities.
*/
Expand Down Expand Up @@ -214,5 +223,16 @@ public static ZonedDateTime getTimestamp(Properties properties, String key) {
}
return null;
}

/**
* Returns the database property value in seconds.
*
* @param key the key to the property
* @return the property value in seconds
*/
public long getPropertyInSeconds(String key) {
final String value = getProperty(key, "0");
return DateUtil.getEpochValueInSeconds(value);
}

}
38 changes: 29 additions & 9 deletions core/src/main/java/org/owasp/dependencycheck/data/update/HostedSuppressionsDataSource.java
View file
Expand Up @@ -33,8 +33,9 @@
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;

public class HostedSuppressionsDataSource implements CachedWebDataSource {
public class HostedSuppressionsDataSource implements CachedWebDataSource {

/**
* Static logger.
Expand All @@ -45,7 +46,10 @@ public class HostedSuppressionsDataSource implements CachedWebDataSource {
* The configured settings.
*/
private Settings settings;

/**
* The properties obtained from the database.
*/
private DatabaseProperties dbProperties = null;
/**
* The default URL to the Hosted Suppressions file.
*/
Expand All @@ -55,12 +59,17 @@ public class HostedSuppressionsDataSource implements CachedWebDataSource {
* Downloads the current Hosted suppressions file.
*
* @param engine a reference to the ODC Engine
* @return returns false as no updates are made to the database, just web resources cached locally
* @return returns false as no updates are made to the database, just web
* resources cached locally
* @throws UpdateException thrown if the update encountered fatal errors
*/
@Override
public boolean update(Engine engine) throws UpdateException {
this.settings = engine.getSettings();
if (engine.getMode() != Engine.Mode.EVIDENCE_COLLECTION) {
//note this conditional is only to support test cases.
this.dbProperties = engine.getDatabase().getDatabaseProperties();
}
final String configuredUrl = settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_URL, DEFAULT_SUPPRESSIONS_URL);
final boolean autoupdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE, true);
final boolean forceupdate = settings.getBoolean(Settings.KEYS.HOSTED_SUPPRESSIONS_FORCEUPDATE, false);
Expand All @@ -76,11 +85,14 @@ public boolean update(Engine engine) throws UpdateException {
if (proceed) {
LOGGER.debug("Begin Hosted Suppressions file update");
fetchHostedSuppressions(settings, url, repoFile);
if (dbProperties != null) {
dbProperties.save(DatabaseProperties.HOSTED_SUPPRESSION_LAST_CHECKED, Long.toString(System.currentTimeMillis() / 1000));
}
}
} catch (UpdateException ex) {
// only emit a warning, DependencyCheck will continue without taking the latest hosted suppressions into account.
LOGGER.warn("Failed to update hosted suppressions file, results may contain false positives already resolved by the "
+ "DependencyCheck project", ex);
+ "DependencyCheck project", ex);
} catch (MalformedURLException ex) {
throw new UpdateException(String.format("Invalid URL for Hosted Suppressions file (%s)", configuredUrl), ex);
} catch (IOException ex) {
Expand All @@ -93,16 +105,23 @@ public boolean update(Engine engine) throws UpdateException {
* Determines if the we should update the Hosted Suppressions file.
*
* @param repo the Hosted Suppressions file.
* @return <code>true</code> if an update to the Hosted Suppressions file should
* be performed; otherwise <code>false</code>
* @return <code>true</code> if an update to the Hosted Suppressions file
* should be performed; otherwise <code>false</code>
* @throws NumberFormatException thrown if an invalid value is contained in
* the database properties
*/
protected boolean shouldUpdate(File repo) throws NumberFormatException {
boolean proceed = true;
if (repo != null && repo.isFile()) {
final int validForHours = settings.getInt(Settings.KEYS.HOSTED_SUPPRESSIONS_VALID_FOR_HOURS, 2);
final long lastUpdatedOn = repo.lastModified();
long lastUpdatedOn = 0;
if (dbProperties != null) {
lastUpdatedOn = dbProperties.getPropertyInSeconds(DatabaseProperties.HOSTED_SUPPRESSION_LAST_CHECKED);
}
if (lastUpdatedOn <= 0) {
//fall back on conversion from file last modified to storing in the db.
lastUpdatedOn = repo.lastModified();
}
final long now = System.currentTimeMillis();
LOGGER.debug("Last updated: {}", lastUpdatedOn);
LOGGER.debug("Now: {}", now);
Expand All @@ -120,7 +139,8 @@ protected boolean shouldUpdate(File repo) throws NumberFormatException {
*
* @param settings a reference to the dependency-check settings
* @param repoUrl the URL to the hosted suppressions file to use
* @param repoFile the local file where the hosted suppressions file is to be placed
* @param repoFile the local file where the hosted suppressions file is to
* be placed
* @throws UpdateException thrown if there is an exception during
* initialization
*/
Expand All @@ -144,7 +164,7 @@ public boolean purge(Engine engine) {
boolean result = true;
try {
final URL repoUrl = new URL(settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_URL,
DEFAULT_SUPPRESSIONS_URL));
DEFAULT_SUPPRESSIONS_URL));
final String filename = new File(repoUrl.getPath()).getName();
final File repo = new File(settings.getDataDirectory(), filename);
if (repo.exists()) {
Expand Down
13 changes: 1 addition & 12 deletions core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java
View file
Expand Up @@ -120,7 +120,7 @@ private boolean shouldUpdate() throws UpdateException {
if (cveDB.dataExists() && 0 < validForHours) {
// ms Valid = valid (hours) x 60 min/hour x 60 sec/min x 1000 ms/sec
final long validForSeconds = validForHours * 60L * 60L;
final long lastChecked = getPropertyInSeconds(DatabaseProperties.KEV_LAST_CHECKED);
final long lastChecked = dbProperties.getPropertyInSeconds(DatabaseProperties.KEV_LAST_CHECKED);
final long now = System.currentTimeMillis() / 1000;
proceed = (now - lastChecked) > validForSeconds;
if (!proceed) {
Expand All @@ -130,15 +130,4 @@ private boolean shouldUpdate() throws UpdateException {
return proceed;
}

/**
* Returns the database property value in seconds.
*
* @param key the key to the property
* @return the property value in seconds
*/
private long getPropertyInSeconds(String key) {
final String value = dbProperties.getProperty(key, "0");
return DateUtil.getEpochValueInSeconds(value);
}

}
13 changes: 12 additions & 1 deletion core/src/main/java/org/owasp/dependencycheck/data/update/RetireJSDataSource.java
View file
Expand Up @@ -24,6 +24,7 @@
import javax.annotation.concurrent.ThreadSafe;

import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.exception.WriteLockException;
import org.owasp.dependencycheck.utils.Downloader;
Expand Down Expand Up @@ -54,6 +55,10 @@ public class RetireJSDataSource implements CachedWebDataSource {
* The configured settings.
*/
private Settings settings;
/**
* The properties obtained from the database.
*/
private DatabaseProperties dbProperties = null;
/**
* The default URL to the RetireJS JavaScript repository.
*/
Expand All @@ -75,6 +80,7 @@ public RetireJSDataSource() {
@Override
public boolean update(Engine engine) throws UpdateException {
this.settings = engine.getSettings();
this.dbProperties = engine.getDatabase().getDatabaseProperties();
final String configuredUrl = settings.getString(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, DEFAULT_JS_URL);
final boolean autoupdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE, true);
final boolean forceupdate = settings.getBoolean(Settings.KEYS.ANALYZER_RETIREJS_FORCEUPDATE, false);
Expand All @@ -87,6 +93,7 @@ public boolean update(Engine engine) throws UpdateException {
if (proceed) {
LOGGER.debug("Begin RetireJS Update");
initializeRetireJsRepo(settings, url, repoFile);
dbProperties.save(DatabaseProperties.RETIRE_LAST_CHECKED, Long.toString(System.currentTimeMillis() / 1000));
}
} catch (MalformedURLException ex) {
throw new UpdateException(String.format("Invalid URL for RetireJS repository (%s)", configuredUrl), ex);
Expand All @@ -109,7 +116,11 @@ protected boolean shouldUpdate(File repo) throws NumberFormatException {
boolean proceed = true;
if (repo != null && repo.isFile()) {
final int validForHours = settings.getInt(Settings.KEYS.ANALYZER_RETIREJS_REPO_VALID_FOR_HOURS, 0);
final long lastUpdatedOn = repo.lastModified();
long lastUpdatedOn = dbProperties.getPropertyInSeconds(DatabaseProperties.RETIRE_LAST_CHECKED);
if (lastUpdatedOn <= 0) {
//fall back on conversion from file last modified to storing in the db.
lastUpdatedOn = repo.lastModified();
}
final long now = System.currentTimeMillis();
LOGGER.debug("Last updated: {}", lastUpdatedOn);
LOGGER.debug("Now: {}", now);
Expand Down

0 comments on commit 2a388f1

Please sign in to comment.