Skip to content
Permalink
 
 
Cannot retrieve contributors at this time
414 lines (369 sloc) 11.7 KB
package pl.djvuhtml5.client;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.gwt.core.shared.GWT;
import com.google.gwt.typedarrays.shared.TypedArrays;
import com.google.gwt.typedarrays.shared.Uint8Array;
import com.google.gwt.xhr.client.ReadyStateChangeHandler;
import com.google.gwt.xhr.client.XMLHttpRequest;
import com.google.gwt.xhr.client.XMLHttpRequest.ResponseType;
import com.lizardtech.djvu.CachedInputStream;
import com.lizardtech.djvu.DataSource;
import com.lizardtech.djvu.DjVmDir;
import com.lizardtech.djvu.DjVuInfo;
import com.lizardtech.djvu.DjVuPage;
import com.lizardtech.djvu.Document;
import com.lizardtech.djvu.IFFEnumeration;
import com.lizardtech.djvu.URLInputStream;
import com.lizardtech.djvu.text.DjVuText;
public class PageDecoder implements DataSource {
private static class FileItem {
public Uint8Array data;
public int dataSize;
public final List<ReadyListener> listeners = new ArrayList<>();
public boolean downloadStarted;
}
private class PageItem implements Comparable<PageItem> {
public final int pageNum;
public DjVuPage page;
public DjVuInfo info;
public DjVuText text;
public boolean isDecoded;
public int memoryUsage;
public int rank = 10000;
public PageItem(int pageNum) {
this.pageNum = pageNum;
}
@Override
public int compareTo(PageItem o) {
int result = this.rank - o.rank;
if (result == 0)
result = Math.abs(lastRequestedPage - o.pageNum) - Math.abs(lastRequestedPage - this.pageNum);
return -result;
}
public void setInfo(DjVuInfo info) {
this.info = info;
context.setPageInfo(pageNum, info);
}
public void setText(DjVuText text) {
if (text == null)
text = new DjVuText();
this.text = text;
if (text.length() > 0)
textAvailable = true;
context.setText(pageNum, text);
}
}
private final ProcessingContext context;
private Document document;
private final HashMap<String, FileItem> fileCache = new HashMap<>();
/** Most recently used files are at the beginning */
private final List<FileItem> filesByMRU = new ArrayList<>();
private long filesMemoryUsage = 0;
private int downloadsInProgress = 0;
private boolean textDownloadInProgress = false;
private boolean textAvailable = false;
private List<PageItem> pages;
/** The most important pages are at the beginning. */
private List<PageItem> pagesByRank;
private int pagesMemoryUsage = 0;
private int lastRequestedPage = 0;
public PageDecoder(final ProcessingContext context, final String url) {
this.context = context;
DjvuContext.addViewChangeListener(this::pageChanged);
URLInputStream.dataSource = this;
Uint8Array data = getData(url, () -> init(url));
if (data != null)
init(url);
}
private void init(String url) {
try {
document = new Document();
document.read(url);
int pageCount = document.getDjVmDir().get_pages_num();
context.setPageCount(pageCount);
pages = new ArrayList<>(pageCount);
for (int i = 0; i < pageCount; i++)
pages.add(new PageItem(i));
pagesByRank = new ArrayList<>(pages);
Collections.sort(pagesByRank);
context.startProcessing();
} catch (IOException e) {
Logger.getGlobal().log(Level.SEVERE, "Could not parse document", e);
}
}
public boolean decodeCurrentPage() {
PageItem currentPageItem = pages.get(lastRequestedPage);
context.setStatus(currentPageItem.isDecoded ? null : ProcessingContext.STATUS_LOADING);
if (currentPageItem.isDecoded)
return false;
cleanCacheOverflow(0);
return decodePage(currentPageItem);
}
public boolean decodePages() {
int memoryLimit = DjvuContext.getPageCacheSize();
int totalMemory = 0;
int fetchIndex = 0;
while (fetchIndex < pagesByRank.size() && totalMemory < memoryLimit) {
PageItem pageItem = pagesByRank.get(fetchIndex);
if (!pageItem.isDecoded)
break;
totalMemory += pageItem.memoryUsage;
fetchIndex++;
}
if (fetchIndex == pagesByRank.size())
return false; // all is decoded
cleanCacheOverflow(fetchIndex + 1);
PageItem pageToDecode = pagesByRank.get(fetchIndex);
if (pagesMemoryUsage + pageToDecode.memoryUsage > memoryLimit)
return false; // all the best pages are in memory
return decodePage(pageToDecode);
}
public boolean decodeTexts() {
PageItem firstMissing = null;
DjVmDir dir = document.getDjVmDir();
for (PageItem page : pagesByRank) {
if (page.text != null)
continue;
CachedInputStream stream;
if (dir.is_bundled()) {
try {
stream = document.get_data(page.pageNum);
} catch (IOException e) {
GWT.log("Error while decoding text in page " + page.pageNum, e);
return false;
}
} else {
FileItem file = getCachedFile(dir.page_to_url(page.pageNum));
if (file.data != null) {
stream = new CachedInputStream().init(new URLInputStream().init(file.data));
} else {
if (firstMissing == null)
firstMissing = page;
continue;
}
}
extractInfoAndText(page, stream);
}
if (firstMissing == null || downloadsInProgress > 0 || textDownloadInProgress || !textAvailable)
return false;
// download missing page specifically to extract text (bypass file cache)
final String url = dir.page_to_url(firstMissing.pageNum);
final PageItem page = firstMissing;
XMLHttpRequest request = XMLHttpRequest.create();
request.open("GET", url);
request.setResponseType(ResponseType.ArrayBuffer);
request.setOnReadyStateChange(xhr -> {
if (xhr.getReadyState() != XMLHttpRequest.DONE)
return;
textDownloadInProgress = false;
if (xhr.getStatus() == 200) {
Uint8Array data = TypedArrays.createUint8Array(xhr.getResponseArrayBuffer());
extractInfoAndText(page, new CachedInputStream().init(new URLInputStream().init(data)));
context.startProcessing();
} else {
GWT.log("Error downloading " + url);
GWT.log("response status: " + xhr.getStatus() + " " + xhr.getStatusText());
}
});
request.send();
textDownloadInProgress = true;
return true;
}
private void extractInfoAndText(PageItem page, CachedInputStream input) {
try {
IFFEnumeration chunks = input.getIFFChunks();
while (chunks.hasMoreElements() && (page.info == null || page.text == null)) {
CachedInputStream chunk = chunks.nextElement();
String chunkName = chunk.getName();
if (chunkName.startsWith("FORM:")) {
extractInfoAndText(page, chunk);
} else if ("INFO".equals(chunkName)) {
DjVuInfo info = new DjVuInfo();
info.decode(chunk);
page.setInfo(info);
} else if ("TXTa".equals(chunkName) || "TXTz".equals(chunkName)) {
page.setText(new DjVuText().init(chunk));
}
}
if (page.text == null)
page.setText(new DjVuText());
} catch (IOException e) {
GWT.log("Error while decoding text in page " + page.pageNum, e);
page.setText(null);
}
}
/**
* @param cutoffIndex
* pages in ranking from first to this index will not be removed.
* Not inclusive (the page at this index can be removed).
*/
private void cleanCacheOverflow(int cutoffIndex) {
int memoryLimit = DjvuContext.getPageCacheSize();
for (int i = pagesByRank.size() - 1; pagesMemoryUsage > memoryLimit && i >= cutoffIndex; i--) {
PageItem pageItem = pagesByRank.get(i);
if (pageItem.pageNum == lastRequestedPage)
continue;
if (pageItem.isDecoded) {
pagesMemoryUsage -= pageItem.memoryUsage;
pageItem.isDecoded = false;
}
pageItem.page = null;
}
}
private boolean decodePage(PageItem pageItem) {
DjVuPage page = pageItem.page;
try {
if (page == null) {
page = pageItem.page = document.getPage(pageItem.pageNum);
if (page == null)
return true; // not downloaded yet
GWT.log("Decoding page " + pageItem.pageNum);
}
if (page.decodeStep()) {
pageItem.isDecoded = true;
if (pageItem.info == null) {
pageItem.setInfo(page.getInfo());
pageItem.setText(page.getText());
}
pageItem.memoryUsage = page.getMemoryUsage();
pagesMemoryUsage += pageItem.memoryUsage;
}
return true;
} catch (IOException e) {
GWT.log("Error while decoding page " + pageItem.pageNum, e);
return false;
}
}
public int getPageCount() {
return pages == null ? 0 : pages.size();
}
private void pageChanged() {
int number = DjvuContext.getPage();
if (pages == null) {
lastRequestedPage = number;
return;
}
if (lastRequestedPage != number) {
lastRequestedPage = number;
updateRanks();
Collections.sort(pagesByRank);
}
context.startProcessing();
}
private void updateRanks() {
int points = 0;
for (PageItem item : pages) {
int d = item.rank / 10;
points += d;
item.rank -= d;
}
int[] dd = { 1, -1 };
int i = 0;
while (points > 0) {
for (int d : dd) {
int index = lastRequestedPage + d * (i % pages.size());
if (index < 0 || index >= pages.size())
continue;
int p = points / 10 + 1;
points -= p;
pages.get(index).rank += p;
if (points <= 0)
break;
}
i++;
}
}
public DjVuPage getPage(int number) {
PageItem pageItem = pages.get(number);
return pageItem.isDecoded ? pageItem.page : null;
}
@Override
public Uint8Array getData(String url, ReadyListener listener) {
FileItem entry = getCachedFile(url);
if (!entry.downloadStarted) {
downloadFile(url);
}
if (entry.data == null && listener != null)
entry.listeners.add(listener);
filesByMRU.remove(entry);
filesByMRU.add(0, entry);
return entry.data;
}
private void downloadFile(final String url) {
XMLHttpRequest request = XMLHttpRequest.create();
request.open("GET", url);
request.setResponseType(ResponseType.ArrayBuffer);
request.setOnReadyStateChange(new ReadyStateChangeHandler() {
@Override
public void onReadyStateChange(XMLHttpRequest xhr) {
if (xhr.getReadyState() == XMLHttpRequest.DONE) {
downloadsInProgress--;
if (xhr.getStatus() == 200) {
FileItem entry = getCachedFile(url);
entry.data = TypedArrays.createUint8Array(xhr.getResponseArrayBuffer());
entry.dataSize = entry.data.byteLength();
filesMemoryUsage += entry.dataSize;
checkFilesMemory();
context.startProcessing();
fireReady(url);
continueDownload();
} else {
GWT.log("Error downloading " + url);
GWT.log("response status: " + xhr.getStatus() + " " + xhr.getStatusText());
context.setStatus(ProcessingContext.STATUS_ERROR);
fileCache.get(url).downloadStarted = false;
}
}
}
});
request.send();
fileCache.get(url).downloadStarted = true;
downloadsInProgress++;
}
protected void checkFilesMemory() {
int limit = DjvuContext.getFileCacheSize();
for (int i = filesByMRU.size() - 1; filesMemoryUsage > limit && i > 4; i--) {
FileItem item = filesByMRU.remove(i);
if (item.data == null)
continue;
filesMemoryUsage -= item.dataSize;
item.data = null;
item.downloadStarted = false;
}
}
protected void fireReady(String url) {
FileItem entry = fileCache.get(url);
if (entry == null)
return;
for (ReadyListener listener : entry.listeners)
listener.dataReady();
entry.listeners.clear();
}
protected void continueDownload() {
DjVmDir dir = document.getDjVmDir();
if (downloadsInProgress > 0 || dir.is_bundled() || filesMemoryUsage > DjvuContext.getFileCacheSize())
return;
for (PageItem page : pagesByRank) {
String url = dir.page_to_url(page.pageNum);
FileItem file = getCachedFile(url);
if (file.data == null && !file.downloadStarted) {
if (filesMemoryUsage + file.dataSize < DjvuContext.getFileCacheSize()) {
downloadFile(url);
}
break;
}
}
}
private FileItem getCachedFile(String url) {
FileItem file = fileCache.get(url);
if (file == null)
fileCache.put(url, file = new FileItem());
return file;
}
}
You can’t perform that action at this time.