Skip to content

Commit

Permalink
Fixed bug where the incorrect category name was being displayed when …
Browse files Browse the repository at this point in the history 
…grouping by category.

For speed, now stores hash set hits in memory.
Fixed bug where hash hit count would never update after a group was created, even if files were added/removed
Changed category zero count to be bases on other counts and not stored separately.
  • Loading branch information
@APriestman
APriestman committed May 22, 2015
1 parent 11394d0 commit 24bc7c1
Show file tree
Hide file tree
Showing 9 changed files with 144 additions and 79 deletions.
7 changes: 1 addition & 6 deletions ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/CategorizeAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,17 +100,12 @@ protected Object doInBackground() throws Exception {
//this is bad: treating tags as categories as long as their names start with prefix
//TODO: abandon using tags for categories and instead add a new column to DrawableDB
if (ct.getName().getDisplayName().startsWith(Category.CATEGORY_PREFIX)) {
LOGGER.log(Level.INFO, "removing old category from {0}", file.getName());
//LOGGER.log(Level.INFO, "removing old category from {0}", file.getName());
Case.getCurrentCase().getServices().getTagsManager().deleteContentTag(ct);
controller.getDatabase().decrementCategoryCount(Category.fromDisplayName(ct.getName().getDisplayName()));
hadExistingCategory = true;
}
}

// If the image was uncategorized, decrement the uncategorized count
if(! hadExistingCategory){
controller.getDatabase().decrementCategoryCount(Category.ZERO);
}

controller.getDatabase().incrementCategoryCount(Category.fromDisplayName(tagName.getDisplayName()));
if (tagName != Category.ZERO.getTagName()) { // no tags for cat-0
Expand Down
2 changes: 1 addition & 1 deletion ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/Category.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public enum Category implements Comparable<Category> {
private final static Set<CategoryListener> listeners = new HashSet<>();

public static void fireChange(Collection<Long> ids) {
Set<CategoryListener> listenersCopy = new HashSet<CategoryListener>(listeners);
Set<CategoryListener> listenersCopy = new HashSet<>();
synchronized (listeners) {
listenersCopy.addAll(listeners);
}
Expand Down
119 changes: 78 additions & 41 deletions ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@
import org.sleuthkit.autopsy.imagegallery.grouping.GroupSortBy;
import static org.sleuthkit.autopsy.imagegallery.grouping.GroupSortBy.GROUP_BY_VALUE;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.ContentTag;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TagName;
Expand Down Expand Up @@ -1119,6 +1121,46 @@ public MultipleTransactionException() {
}
}

@GuardedBy("hashSetMap")
private final Map<Long, Set<String>> hashSetMap = new HashMap<>();

@GuardedBy("hashSetMap")
public boolean isInHashSet(Long id){
if(! hashSetMap.containsKey(id)){
updateHashSetsForFile(id);
}
return (! hashSetMap.get(id).isEmpty());
}

@GuardedBy("hashSetMap")
public Set<String> getHashSetsForFile(Long id){
if(! isInHashSet(id)){
updateHashSetsForFile(id);
}
return hashSetMap.get(id);
}

@GuardedBy("hashSetMap")
public void updateHashSetsForFile(Long id){

try {
List<BlackboardArtifact> arts = ImageGalleryController.getDefault().getSleuthKitCase().getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT, id);
Set<String> hashNames = new HashSet<>();
for(BlackboardArtifact a:arts){
List<BlackboardAttribute> attrs = a.getAttributes();
for(BlackboardAttribute attr:attrs){
if(attr.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()){
hashNames.add(attr.getValueString());
}
}
}
hashSetMap.put(id, hashNames);
} catch (IllegalStateException | TskCoreException ex) {
LOGGER.log(Level.WARNING, "could not access case during updateHashSetsForFile()", ex);

}
}

/**
* For performance reasons, keep a list of all file IDs currently in the image database.
* Otherwise the database is queried many times to retrieve the same data.
Expand All @@ -1144,6 +1186,12 @@ public void removeImageFileFromList(Long id) {
}
}

public int getNumberOfImageFilesInList(){
synchronized (fileIDlist){
return fileIDlist.size();
}
}

public void initializeImageList(){
synchronized (fileIDlist){
dbReadLock();
Expand All @@ -1169,63 +1217,52 @@ public void initializeImageList(){
private final Map<Category, Integer> categoryCounts = new HashMap<>();

public void incrementCategoryCount(Category cat) throws TskCoreException{
synchronized(categoryCounts){
int count = getCategoryCount(cat);
count++;
categoryCounts.put(cat, count);
if(cat != Category.ZERO){
synchronized(categoryCounts){
int count = getCategoryCount(cat);
count++;
categoryCounts.put(cat, count);
}
}
}

public void decrementCategoryCount(Category cat) throws TskCoreException{
synchronized(categoryCounts){
int count = getCategoryCount(cat);
count--;
categoryCounts.put(cat, count);
if(cat != Category.ZERO){
synchronized(categoryCounts){
int count = getCategoryCount(cat);
count--;
categoryCounts.put(cat, count);
}
}
}

public int getCategoryCount(Category cat) throws TskCoreException{
synchronized(categoryCounts){
if(categoryCounts.containsKey(cat)){
if(cat == Category.ZERO){
// Keeping track of the uncategorized files is a bit tricky while ingest
// is going on, so always use the list of file IDs we already have along with the
// other category counts instead of trying to track it separately.
int allOtherCatCount = getCategoryCount(Category.ONE) + getCategoryCount(Category.TWO) + getCategoryCount(Category.THREE) +
getCategoryCount(Category.FOUR) + getCategoryCount(Category.FIVE);
return getNumberOfImageFilesInList() - allOtherCatCount;
}
else if(categoryCounts.containsKey(cat)){
return categoryCounts.get(cat);
}
else{
try {
if (cat == Category.ZERO) {

// Category Zero (Uncategorized) files will not be tagged as such -
// this is really just the default setting. So we count the number of files
// tagged with the other categories and subtract from the total.
int allOtherCatCount = 0;
TagName[] tns = {Category.FOUR.getTagName(), Category.THREE.getTagName(), Category.TWO.getTagName(), Category.ONE.getTagName(), Category.FIVE.getTagName()};
for (TagName tn : tns) {
List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(tn);
for (ContentTag ct : contentTags) {
if(ct.getContent() instanceof AbstractFile){
AbstractFile f = (AbstractFile)ct.getContent();
if(this.isImageFile(f.getId())){
allOtherCatCount++;
}
}
}
}
categoryCounts.put(cat, this.countAllFiles() - allOtherCatCount);
return categoryCounts.get(cat);
} else {

int fileCount = 0;
List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(cat.getTagName());
for (ContentTag ct : contentTags) {
if(ct.getContent() instanceof AbstractFile){
AbstractFile f = (AbstractFile)ct.getContent();
if(this.isImageFile(f.getId())){
fileCount++;
}
int fileCount = 0;
List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(cat.getTagName());
for (ContentTag ct : contentTags) {
if(ct.getContent() instanceof AbstractFile){
AbstractFile f = (AbstractFile)ct.getContent();
if(this.isImageFile(f.getId())){
fileCount++;
}
}
categoryCounts.put(cat, fileCount);
return fileCount;
}
categoryCounts.put(cat, fileCount);
return fileCount;
} catch(IllegalStateException ex){
throw new TskCoreException("Case closed while getting files");
}
Expand Down
12 changes: 3 additions & 9 deletions ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.commons.lang3.text.WordUtils;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.imagegallery.ImageGalleryController;
import org.sleuthkit.autopsy.imagegallery.ImageGalleryModule;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
Expand Down Expand Up @@ -97,8 +98,6 @@ public static DrawableFile<?> create(Long id, boolean analyzed) throws TskCoreEx

private final SimpleObjectProperty<Category> category = new SimpleObjectProperty<>(null);

private Collection<String> hashHitSetNames;

private String make;

private String model;
Expand All @@ -116,16 +115,11 @@ protected DrawableFile(T file, Boolean analyzed) {

public abstract boolean isVideo();

public Collection<String> getHashHitSetNames() {
updateHashSets();
synchronized public Collection<String> getHashHitSetNames() {
Collection<String> hashHitSetNames = ImageGalleryController.getDefault().getDatabase().getHashSetsForFile(getId());
return hashHitSetNames;
}

@SuppressWarnings("unchecked")
private void updateHashSets() {
hashHitSetNames = (Collection<String>) getValuesOfBBAttribute(BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME);
}

@Override
public boolean isRoot() {
return false;
Expand Down
20 changes: 15 additions & 5 deletions ImageGallery/src/org/sleuthkit/autopsy/imagegallery/grouping/DrawableGroup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,26 @@ public double getHashHitDensity() {
return getFilesWithHashSetHitsCount() / (double) getSize();
}

synchronized public int getFilesWithHashSetHitsCount() {
/**
* Call to indicate that an image has been added or removed from the group,
* so the hash counts may not longer be accurate.
*/
synchronized public void invalidateHashSetHitsCount(){
filesWithHashSetHitsCount = -1;
}

synchronized public int getFilesWithHashSetHitsCount() {
//TODO: use the drawable db for this ? -jm
if (filesWithHashSetHitsCount < 0) {
filesWithHashSetHitsCount = 0;
for (Long fileID : fileIds()) {

try {
long artcount = ImageGalleryController.getDefault().getSleuthKitCase().getBlackboardArtifactsCount(BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT, fileID);
if (artcount > 0) {
if(ImageGalleryController.getDefault().getDatabase().isInHashSet(fileID)){
filesWithHashSetHitsCount++;
}
} catch (IllegalStateException | TskCoreException ex) {
LOGGER.log(Level.WARNING, "could not access case during getFilesWithHashSetHitsCount()", ex);
} catch (IllegalStateException | NullPointerException ex) {
LOGGER.log(Level.WARNING, "could not access case during getFilesWithHashSetHitsCount()");
break;
}
}
Expand Down Expand Up @@ -112,10 +120,12 @@ synchronized public void addFile(Long f) {
if (fileIDs.contains(f) == false) {
fileIDs.add(f);
}
invalidateHashSetHitsCount();
}

synchronized public void removeFile(Long f) {
fileIDs.removeAll(f);
invalidateHashSetHitsCount();
}

// By default, sort by group key name
Expand Down
38 changes: 24 additions & 14 deletions ImageGallery/src/org/sleuthkit/autopsy/imagegallery/grouping/GroupManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,18 +281,21 @@ public synchronized void removeFromGroup(GroupKey<?> groupKey, final Long fileID
final DrawableGroup group = getGroupForKey(groupKey);
if (group != null) {
group.removeFile(fileID);
if (group.fileIds().isEmpty()) {
synchronized (groupMap) {
groupMap.remove(groupKey, group);
}
Platform.runLater(() -> {
analyzedGroups.remove(group);
synchronized (unSeenGroups) {
unSeenGroups.remove(group);

// If we're grouping by category, we don't want to remove empty groups.
if(! group.groupKey.getValueDisplayName().startsWith("CAT-")){
if (group.fileIds().isEmpty()) {
synchronized (groupMap) {
groupMap.remove(groupKey, group);
}
});
Platform.runLater(() -> {
analyzedGroups.remove(group);
synchronized (unSeenGroups) {
unSeenGroups.remove(group);
}
});
}
}

}
}

Expand Down Expand Up @@ -487,7 +490,7 @@ public List<Long> getFileIDsWithCategory(Category category) throws TskCoreExcept
for (TagName tn : tns) {
List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(tn);
for (ContentTag ct : contentTags) {
if (ct.getContent() instanceof AbstractFile && ImageGalleryModule.isSupportedAndNotKnown((AbstractFile) ct.getContent())) {
if (ct.getContent() instanceof AbstractFile && db.isImageFile(((AbstractFile) ct.getContent()).getId())) {
files.add(ct.getContent().getId());
}
}
Expand All @@ -499,7 +502,8 @@ public List<Long> getFileIDsWithCategory(Category category) throws TskCoreExcept
List<Long> files = new ArrayList<>();
List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(category.getTagName());
for (ContentTag ct : contentTags) {
if (ct.getContent() instanceof AbstractFile && ImageGalleryModule.isSupportedAndNotKnown((AbstractFile) ct.getContent())) {
if (ct.getContent() instanceof AbstractFile && db.isImageFile(((AbstractFile) ct.getContent()).getId())) {

files.add(ct.getContent().getId());
}
}
Expand Down Expand Up @@ -529,7 +533,8 @@ public List<Long> getFileIDsWithTag(TagName tagName) throws TskCoreException {
List<Long> files = new ArrayList<>();
List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(tagName);
for (ContentTag ct : contentTags) {
if (ct.getContent() instanceof AbstractFile && ImageGalleryModule.isSupportedAndNotKnown((AbstractFile) ct.getContent())) {
if (ct.getContent() instanceof AbstractFile && db.isImageFile(((AbstractFile) ct.getContent()).getId())) {

files.add(ct.getContent().getId());
}
}
Expand Down Expand Up @@ -649,7 +654,10 @@ synchronized public void handleFileUpdate(FileUpdateEvent evt) {
if (checkAnalyzed != null) { // => the group is analyzed, so add it to the ui
populateAnalyzedGroup(gk, checkAnalyzed);
}
}
}
else{
g.invalidateHashSetHitsCount();
}
}
}

Expand All @@ -668,6 +676,8 @@ synchronized public void handleFileUpdate(FileUpdateEvent evt) {
* innertask? -jm
*/
for (final long fileId : fileIDs) {

db.updateHashSetsForFile(fileId);

//get grouping(s) this file would be in
Set<GroupKey<?>> groupsForFile = getGroupKeysForFileID(fileId);
Expand Down
11 changes: 10 additions & 1 deletion ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DrawableView.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.sleuthkit.autopsy.imagegallery.gui;

import java.util.Collection;
import java.util.logging.Level;
import javafx.application.Platform;
import javafx.scene.layout.Border;
import javafx.scene.layout.BorderStroke;
Expand All @@ -9,6 +10,7 @@
import javafx.scene.layout.CornerRadii;
import javafx.scene.layout.Region;
import javafx.scene.paint.Color;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ThreadConfined;
import org.sleuthkit.autopsy.imagegallery.TagUtils;
import org.sleuthkit.autopsy.imagegallery.datamodel.Category;
Expand Down Expand Up @@ -56,7 +58,14 @@ public interface DrawableView extends Category.CategoryListener, TagUtils.TagLis
void handleTagsChanged(Collection<Long> ids);

default boolean hasHashHit() {
return getFile().getHashHitSetNames().isEmpty() == false;
try{
return getFile().getHashHitSetNames().isEmpty() == false;
} catch (NullPointerException ex){
// I think this happens when we're in the process of removing images from the view while
// also trying to update it?
Logger.getLogger(DrawableView.class.getName()).log(Level.WARNING, "Error looking up hash set hits");
return false;
}
}

static Border getCategoryBorder(Category category) {
Expand Down
2 changes: 1 addition & 1 deletion ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/SingleDrawableViewBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,7 @@ public void setFile(final Long fileID) {
} else {
Category.registerListener(this);
TagUtils.registerListener(this);

getFile();
updateSelectionState();
updateCategoryBorder();
Expand Down
Loading

0 comments on commit 24bc7c1

Please sign in to comment.