Skip to content

Commit

Permalink
fix #1126 thumbnail purger checks if doc exists
Browse files Browse the repository at this point in the history
  • Loading branch information
@marevol
marevol committed Jun 29, 2017
1 parent 6464277 commit 0950397
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 9 deletions.
26 changes: 26 additions & 0 deletions src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -743,6 +743,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. 100 */
String PAGE_THUMBNAIL_QUEUE_MAX_FETCH_SIZE = "page.thumbnail.queue.max.fetch.size";

/** The key of the configuration. e.g. 100 */
String PAGE_THUMBNAIL_PURGE_MAX_FETCH_SIZE = "page.thumbnail.purge.max.fetch.size";

/** The key of the configuration. e.g. 0 */
String PAGING_SEARCH_PAGE_START = "paging.search.page.start";

Expand Down Expand Up @@ -3631,6 +3634,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
Integer getPageThumbnailQueueMaxFetchSizeAsInteger();

/**
* Get the value for the key 'page.thumbnail.purge.max.fetch.size'. <br>
* The value is, e.g. 100 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getPageThumbnailPurgeMaxFetchSize();

/**
* Get the value for the key 'page.thumbnail.purge.max.fetch.size' as {@link Integer}. <br>
* The value is, e.g. 100 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getPageThumbnailPurgeMaxFetchSizeAsInteger();

/**
* Get the value for the key 'paging.search.page.start'. <br>
* The value is, e.g. 0 <br>
Expand Down Expand Up @@ -6405,6 +6423,14 @@ public Integer getPageThumbnailQueueMaxFetchSizeAsInteger() {
return getAsInteger(FessConfig.PAGE_THUMBNAIL_QUEUE_MAX_FETCH_SIZE);
}

public String getPageThumbnailPurgeMaxFetchSize() {
return get(FessConfig.PAGE_THUMBNAIL_PURGE_MAX_FETCH_SIZE);
}

public Integer getPageThumbnailPurgeMaxFetchSizeAsInteger() {
return getAsInteger(FessConfig.PAGE_THUMBNAIL_PURGE_MAX_FETCH_SIZE);
}

public String getPagingSearchPageStart() {
return get(FessConfig.PAGING_SEARCH_PAGE_START);
}
Expand Down
102 changes: 93 additions & 9 deletions src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Tuple4;
import org.codelibs.fess.Constants;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv;
import org.codelibs.fess.es.config.exentity.ThumbnailQueue;
import org.codelibs.fess.exception.FessSystemException;
Expand All @@ -47,6 +48,7 @@
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.codelibs.fess.util.ResourceUtil;
import org.elasticsearch.index.query.QueryBuilders;
import org.lastaflute.web.util.LaRequestUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -60,6 +62,8 @@ public class ThumbnailManager {

private static final String NOIMAGE_FILE_SUFFIX = ".txt";

protected static final String THUMBNAILS_DIR_NAME = "thumbnails";

private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class);

protected File baseDir;
Expand Down Expand Up @@ -94,7 +98,7 @@ public void init() {
} else {
final String varPath = System.getProperty(FESS_VAR_PATH);
if (varPath != null) {
baseDir = new File(varPath, "thumbnails");
baseDir = new File(varPath, THUMBNAILS_DIR_NAME);
} else {
baseDir = ResourceUtil.getThumbnailPath().toFile();
}
Expand Down Expand Up @@ -331,25 +335,103 @@ public long purge(final long expiry) {
return 0;
}
try {
final FilePurgeVisitor visitor = new FilePurgeVisitor(expiry);
final FilePurgeVisitor visitor = new FilePurgeVisitor(baseDir.toPath(), imageExtention, expiry);
Files.walkFileTree(baseDir.toPath(), visitor);
return visitor.getCount();
} catch (final Exception e) {
throw new JobProcessingException(e);
}
}

private static class FilePurgeVisitor implements FileVisitor<Path> {
protected static class FilePurgeVisitor implements FileVisitor<Path> {

protected final long expiry;

protected long count;

protected final int maxPurgeSize;

protected final List<Path> deletedFileList = new ArrayList<>();

protected final Path basePath;

protected final String imageExtention;

private final long expiry;
protected final FessEsClient fessEsClient;

private long count;
protected final FessConfig fessConfig;

FilePurgeVisitor(final long expiry) {
FilePurgeVisitor(final Path basePath, final String imageExtention, final long expiry) {
this.basePath = basePath;
this.imageExtention = imageExtention;
this.expiry = expiry;
this.fessConfig = ComponentUtil.getFessConfig();
this.maxPurgeSize = fessConfig.getPageThumbnailPurgeMaxFetchSizeAsInteger();
this.fessEsClient = ComponentUtil.getFessEsClient();
}

protected void deleteFiles() {
final Map<String, Path> deleteFileMap = new HashMap<>();
for (final Path path : deletedFileList) {
final String docId = getDocId(path);
if (StringUtil.isBlank(docId) || deleteFileMap.containsKey(docId)) {
deleteFile(path);
} else {
deleteFileMap.put(docId, path);
}
}
deletedFileList.clear();

if (!deleteFileMap.isEmpty()) {
final String docIdField = fessConfig.getIndexFieldDocId();
fessEsClient.getDocumentList(
fessConfig.getIndexDocumentSearchIndex(),
fessConfig.getIndexDocumentType(),
searchRequestBuilder -> {
searchRequestBuilder.setQuery(QueryBuilders.termsQuery(docIdField,
deleteFileMap.keySet().toArray(new String[deleteFileMap.size()])));
searchRequestBuilder.setFetchSource(new String[] { docIdField }, StringUtil.EMPTY_STRINGS);
return true;
}).forEach(m -> {
final Object docId = m.get(docIdField);
if (docId != null) {
deleteFileMap.remove(docId);
if (logger.isDebugEnabled()) {
logger.debug("Keep thumbnail: " + docId);
}
}
});
;
deleteFileMap.values().forEach(v -> deleteFile(v));
count += deleteFileMap.size();
}
}

protected void deleteFile(final Path path) {
try {
Files.delete(path);
if (logger.isDebugEnabled()) {
logger.debug("Delete " + path);
}
} catch (IOException e) {
logger.warn("Failed to delete " + path, e);
}
}

protected String getDocId(final Path file) {
final String s = file.toUri().toString();
final String b = basePath.toUri().toString();
final String id = s.replace(b, StringUtil.EMPTY).replace("." + imageExtention, StringUtil.EMPTY).replace("/", StringUtil.EMPTY);
if (logger.isDebugEnabled()) {
logger.debug("Base: " + b + " File: " + s + " DocId: " + id);
}
return id;
}

public long getCount() {
if (!deletedFileList.isEmpty()) {
deleteFiles();
}
return count;
}

Expand All @@ -361,8 +443,10 @@ public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttribut
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
if (System.currentTimeMillis() - Files.getLastModifiedTime(file).toMillis() > expiry) {
Files.delete(file);
count++;
deletedFileList.add(file);
if (deletedFileList.size() > maxPurgeSize) {
deleteFiles();
}
}
return FileVisitResult.CONTINUE;
}
Expand All @@ -380,7 +464,7 @@ public FileVisitResult postVisitDirectory(final Path dir, final IOException e) t
if (e != null) {
logger.warn("I/O exception on " + dir, e);
}
if (dir.toFile().list().length == 0) {
if (dir.toFile().list().length == 0 && !dir.toFile().getName().equals(THUMBNAILS_DIR_NAME)) {
Files.delete(dir);
}
return FileVisitResult.CONTINUE;
Expand Down
1 change: 1 addition & 0 deletions src/main/resources/fess_config.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,7 @@ page.elevate.word.max.fetch.size=1000
page.bad.word.max.fetch.size=1000
page.dictionary.max.fetch.size=1000
page.thumbnail.queue.max.fetch.size=100
page.thumbnail.purge.max.fetch.size=100

# search page
paging.search.page.start=0
Expand Down

0 comments on commit 0950397

Please sign in to comment.